def emit_capability(ctx, m, fd):
amp = "&" if ctx.opts.capa_entity else "&"
ns = m.search_one('namespace')
if ns is None:
return
s = ns.arg + "?module=" + m.i_modulename
latest_rev = util.get_latest_revision(m)
if latest_rev != "unknown":
s = s + amp + "revision=" + latest_rev
if m.i_modulename in ctx.features:
if len(ctx.features[m.i_modulename]) > 0:
s = s + amp + "features=" + ",".join(ctx.features[m.i_modulename])
else:
# do not report any features from the module
pass
else:
# report all features defined in the module
fs = [x.arg for x in m.search('feature')]
s = s + amp + "features=" + ",".join(fs)
devs = []
for d in ctx.deviation_modules:
# check if this deviation module deviates anything in our module
for dev in d.search('deviation'):
if (dev.i_target_node is not None and
dev.i_target_node.i_module.i_modulename == m.i_modulename):
devs.append(d.i_modulename)
break
if len(devs) > 0:
s = s + amp + "deviations=" + ",".join(devs)
fd.write(s + '\n')
def emit(self, ctx, modules):
res = {}
for module in modules:
res['revision'] = util.get_latest_revision(module)
res['name'] = module.arg
for statement in ['namespace', 'prefix', 'revision', 'module-version']:
stmt = module.search_one(statement)
if stmt:
res[stmt.keyword] = stmt.arg
istmts = module.search('import')
if istmts:
if 'dependencies' not in res:
res['dependencies'] = {}
res['dependencies']['required-module'] = []
for istmt in istmts:
# res['dependencies']['required-module'].append(istmt.arg)
# TODO: Need to ask draft authors to add revision-date to YANG
revision = 'unknown'
r = istmt.search_one('revision-date')
if r is not None:
revision = r.arg
res['dependencies']['required-module'].append({'module-name': istmt.arg,'module-revision': revision})
if module.keyword == 'submodule':
if 'module-hierarchy' not in res:
res['module-hierarchy'] = {}
res['module-hierarchy']['module-hierarchy-level'] = 2
belongs = module.search_one('belongs-to')
res['module-hierarchy']['module-parent'] = belongs.arg
return res
def emit_capability(ctx, m, fd):
amp = "&" if ctx.opts.capa_entity else "&"
ns = m.search_one('namespace')
if ns is None:
return
s = ns.arg + "?module=" + m.i_modulename
latest_rev = util.get_latest_revision(m)
if latest_rev != "unknown":
s = s + amp + "revision=" + latest_rev
if m.i_modulename in ctx.features:
if len(ctx.features[m.i_modulename]) > 0:
s = s + amp + "features=" + ",".join(ctx.features[m.i_modulename])
else:
# do not report any features from the module
pass
else:
# report all features defined in the module
fs = [x.arg for x in m.search('feature')]
s = s + amp + "features=" + ",".join(fs)
devs = []
for d in ctx.deviation_modules:
# check if this deviation module deviates anything in our module
for dev in d.search('deviation'):
if (dev.i_target_node is not None and
dev.i_target_node.i_module.i_modulename == m.i_modulename):
devs.append(d.i_modulename)
break
if len(devs) > 0:
s = s + amp + "deviations=" + ",".join(devs)
fd.write(s + '\n')
def index_printer(stmt):
global _yang_catalog_index_values
global _values
global _ctx
if stmt.arg is None:
return
skey = flatten_keyword(stmt.keyword)
module = stmt.i_module
rev = get_latest_revision(module)
revision = ''
path = statements.mk_path_str(stmt, True)
descr = stmt.search_one('description')
dstr = ''
if descr:
dstr = descr.arg
dstr = dstr.replace("'", "''")
subs = []
if rev == 'unknown':
revision = '1970-01-01'
else:
revision = rev
for i in stmt.substmts:
a = i.arg
k = i.keyword
k = flatten_keyword(k)
if i.keyword not in statements.data_definition_keywords:
subs.append(index_get_other(i))
else:
has_children = hasattr(i, 'i_children') and len(i.i_children) > 0
if not a:
a = ''
else:
a = index_escape_json(a)
subs.append({
k: {
'value': a,
'has_children': has_children,
'children': []
}
})
vals = {}
vals['module'] = module.arg
vals['revision'] = revision
vals['organization'] = resolve_organization(module)
vals['path'] = path
vals['statement'] = skey
vals['argument'] = stmt.arg
vals['description'] = dstr
vals['properties'] = json.dumps(subs)
_values['yindex'].append(vals)
def emit_dot(ctx, modules, fd):
seen = []
fd.write("digraph \"modules-graph\" {\n")
for module in modules:
modname = module.arg
modrev = util.get_latest_revision(module)
modid = "%s@%s" % (modname, modrev)
if modid in seen:
# Ignoring duplicate modules
continue
else:
seen.append(modid)
fd.write("\t\"%s\"\n" % (modid))
imps = module.search('import')
if imps:
for imp in imps:
if imp.search('revision-date'):
date = imp.search_one('revision-date').arg
fd.write(
"\t\"%s\" -> \"%s\" [label=\"imports by revision: %s\"]\n"
% (modname, imp.arg, date))
else:
fd.write("\t\"%s\" -> \"%s\" [label=imports]\n" %
(modname, imp.arg))
subs = module.search('include')
if subs:
for sub in subs:
if sub.search('revision-date'):
date = sub.search_one('revision-date').arg
fd.write(
"\t\"%s\" -> \"%s\" [label=\"includes by revision: %s\"]\n"
% (modname, sub.arg, date))
else:
fd.write("\t\"%s\" -> \"%s\" [label=includes]\n" %
(modname, sub.arg))
fd.write("}")
def emit_dot(ctx, modules, fd):
seen = []
fd.write("digraph \"modules-graph\" {\n")
for module in modules:
modname = module.arg
modrev = util.get_latest_revision(module)
modid = "%s@%s" % (modname, modrev)
if modid in seen:
# Ignoring duplicate modules
continue
else:
seen.append(modid)
fd.write("\t\"%s\"\n" % (modid))
imps = module.search('import')
if imps:
for imp in imps:
if imp.search('revision-date'):
date = imp.search_one('revision-date').arg
fd.write("\t\"%s\" -> \"%s\" [label=\"imports by revision: %s\"]\n" % (modname, imp.arg, date))
else:
fd.write("\t\"%s\" -> \"%s\" [label=imports]\n" % (modname, imp.arg))
subs = module.search('include')
if subs:
for sub in subs:
if sub.search('revision-date'):
date = sub.search_one('revision-date').arg
fd.write("\t\"%s\" -> \"%s\" [label=\"includes by revision: %s\"]\n" % (modname, sub.arg, date))
else:
fd.write("\t\"%s\" -> \"%s\" [label=includes]\n" % (modname, sub.arg))
fd.write("}")
def process_sid_file(self, module):
self.module_name = module.i_modulename
self.module_revision = util.get_latest_revision(module)
self.output_file_name = '%s@%s.sid' % (self.module_name, self.module_revision)
if self.range is not None:
if self.range == 'count':
self.count = True
else:
self.set_sid_range(self.range)
if self.input_file_name is not None:
if self.input_file_name[-4:] != ".sid":
raise SidParsingError("File '%s' is not a .sid file" % self.input_file_name)
with open(self.input_file_name) as f:
self.content = json.load(f, object_pairs_hook=collections.OrderedDict)
self.upgrade_sid_file_format() # This function can be removed after a reasonable transition period.
self.validate_key_and_value()
self.validate_ovelaping_ranges()
self.validate_sid()
if self.extra_range is not None:
if self.extra_range == 'count':
self.count = True
else:
self.set_sid_range(self.extra_range)
self.validate_ovelaping_ranges()
self.set_module_information()
self.collect_module_items(module)
if self.range == 'count':
number_of_unassigned_yang_items = self.number_of_unassigned_yang_items()
print("\nThis YANG module requires %d SIDs." % number_of_unassigned_yang_items)
return
if self.extra_range == 'count':
number_of_SIDs_allocated = self.number_of_SIDs_allocated()
number_of_SIDs_used = self.number_of_SIDs_used()
number_of_SIDs_available = number_of_SIDs_allocated - number_of_SIDs_used
number_of_unassigned_yang_items = self.number_of_unassigned_yang_items()
print("\nNumber of SIDs allocated to this module: %d" % number_of_SIDs_allocated)
print("Number of SIDs required by this version: %d" % (number_of_SIDs_used + number_of_unassigned_yang_items))
if (number_of_unassigned_yang_items > number_of_SIDs_available):
print("\nAn extra range of at least %d SIDs is required to perform this update." % (number_of_unassigned_yang_items - number_of_SIDs_available))
else:
print("\nThe update of the .sid file can be performed using the currently available SIDs.")
return
self.sort_items()
self.assign_sid()
if self.list_content:
self.list_all_items()
else:
self.list_deleted_items()
if self.check_consistency:
if self.is_consistent:
if self.sid_registration_info:
self.print_registration_information(module)
else:
print("\nCheck completed successfully")
else:
print("\nThe .sid file need to be updated.")
else:
if self.is_consistent:
print("No .sid file generated, the current .sid file is already up to date.")
else:
self.generate_file()
if self.sid_file_created:
print("\nFile %s created" % self.output_file_name)
else:
print("\nFile %s updated" % self.output_file_name)
print ("Number of SIDs available : %d" % self.number_of_SIDs_allocated())
print ("Number of SIDs used : %d" % self.number_of_SIDs_used())
def index_printer(stmt):
global _yang_catalog_index_values
global _values
global _ctx
if stmt.arg is None:
return
skey = flatten_keyword(stmt.keyword)
module = stmt.i_module
rev = get_latest_revision(module)
revision = ''
path = mk_path_str(stmt, True)
descr = stmt.search_one('description')
dstr = ''
if descr:
dstr = descr.arg
dstr = dstr.replace("'", "''")
subs = []
if rev == 'unknown':
revision = '1970-01-01'
else:
revision = rev
try:
dateutil.parser.parse(revision)
except Exception as e:
if revision[-2:] == '29' and revision[-5:-3] == '02':
revision = revision.replace('02-29', '02-28')
else:
revision = '1970-01-01'
rev_parts = revision.split('-')
revision = datetime(int(rev_parts[0]), int(rev_parts[1]), int(rev_parts[2])).date().isoformat()
for i in stmt.substmts:
a = i.arg
k = i.keyword
k = flatten_keyword(k)
if i.keyword not in statements.data_definition_keywords:
subs.append(index_get_other(i))
else:
has_children = hasattr(i, 'i_children') and len(i.i_children) > 0
if not a:
a = ''
else:
a = index_escape_json(a)
subs.append(
{k: {'value': a, 'has_children': has_children, 'children': []}})
vals = {}
vals['module'] = module.arg
vals['revision'] = revision
vals['organization'] = resolve_organization(module)
vals['path'] = path
vals['statement'] = skey
vals['argument'] = stmt.arg
vals['description'] = dstr
vals['properties'] = json.dumps(subs)
text = '{} {} {} {} {} {} {} {}'.format(vals['module'], vals['revision'], vals['organization'],
vals['path'], vals['statement'], vals['argument'],
vals['description'], vals['properties'])
vals['sort-hash-id'] = hashlib.sha256(text.encode('utf-8')).hexdigest()
_values['yindex'].append(vals)
def build_and_populate(lock_exception, lock_write_rest, lock_remove_list,
lock_remove_elk, lock_index_elk, es, modules, LOGGER,
save_file_dir, threads, log_file, failed_changes_dir,
temp_dir, ytree_dir, process_name):
x = 0
for module in modules:
try:
with lock_remove_list:
modules_copy.remove(module)
x += 1
LOGGER.info(
'yindex on module {}. module {} out of {}, process {}'.format(
module.split('/')[-1], x, len(modules), process_name))
# split to module with path and organization [email protected]:cisco
m_parts = module.split(":")
m = m_parts[0]
plugin.init([])
ctx = create_context('{}'.format(save_file_dir))
ctx.opts.lint_namespace_prefixes = []
ctx.opts.lint_modulename_prefixes = []
for p in plugin.plugins:
p.setup_ctx(ctx)
with open(m, 'r') as f:
parsed_module = ctx.add_module(m, f.read())
ctx.validate()
if parsed_module is None:
raise Exception('Unable to pyang parse module')
f = io.StringIO()
ctx.opts.print_revision = True
emit_name(ctx, [parsed_module], f)
name_revision = f.getvalue().strip()
mods = [parsed_module]
find_submodules(ctx, mods, parsed_module)
f = io.StringIO()
ctx.opts.yang_index_make_module_table = True
ctx.opts.yang_index_no_schema = True
indexerPlugin = IndexerPlugin()
indexerPlugin.emit(ctx, [parsed_module], f)
yindexes = json.loads(f.getvalue())
name_revision = name_revision.split('@')
if len(name_revision) > 1:
name = name_revision[0]
revision = name_revision[1].split(' ')[0]
else:
name = name_revision[0]
revision = '1970-01-01'
if 'belongs-to' in name:
name = name.split(' ')[0]
try:
dateutil.parser.parse(revision)
except Exception as e:
if revision[-2:] == '29' and revision[-5:-3] == '02':
revision = revision.replace('02-29', '02-28')
else:
revision = '1970-01-01'
rev_parts = revision.split('-')
try:
revision = datetime(int(rev_parts[0]), int(rev_parts[1]),
int(rev_parts[2])).date().isoformat()
except Exception:
revision = '1970-01-01'
retry = 3
while retry > 0:
try:
for m in mods:
n = m.arg
rev = get_latest_revision(m)
if rev == 'unknown':
r = '1970-01-01'
else:
r = rev
try:
dateutil.parser.parse(r)
except Exception as e:
if r[-2:] == '29' and r[-5:-3] == '02':
r = r.replace('02-29', '02-28')
else:
r = '1970-01-01'
rev_parts = r.split('-')
r = datetime(int(rev_parts[0]), int(rev_parts[1]),
int(rev_parts[2])).date().isoformat()
query = \
{
"query": {
"bool": {
"must": [{
"match_phrase": {
"module.keyword": {
"query": n
}
}
}, {
"match_phrase": {
"revision": {
"query": r
}
}
}]
}
}
}
while lock_index_elk.locked():
sleep(1)
with lock_remove_elk:
try:
LOGGER.debug(
'deleting data from yindex index, process {}'
.format(process_name))
es.delete_by_query(index='yindex',
body=query,
doc_type='modules',
conflicts='proceed',
request_timeout=40)
LOGGER.debug(
'deleting data from modules index, process {}'
.format(process_name))
total = es.delete_by_query(
index='modules',
body=query,
doc_type='modules',
conflicts='proceed',
request_timeout=40)['deleted']
if total > 1:
LOGGER.info('{}@{}, process {}'.format(
name, revision, process_name))
except NotFoundError as e:
pass
while lock_remove_elk.locked():
sleep(1)
if not lock_index_elk.locked():
lock_index_elk.acquire()
for key in yindexes:
j = -1
for j in range(0, int(len(yindexes[key]) / 30)):
LOGGER.debug(
'pushing new data to yindex {} of {} process {}'
.format(j, int(len(yindexes[key]) / 30),
process_name))
for success, info in parallel_bulk(
es,
yindexes[key][j * 30:(j * 30) + 30],
thread_count=int(threads),
index='yindex',
doc_type='modules',
request_timeout=40):
if not success:
LOGGER.error(
'A elasticsearch document failed with info: {}, process {}'
.format(info, process_name))
LOGGER.debug('pushing new data to yindex last one')
for success, info in parallel_bulk(
es,
yindexes[key][(j * 30) + 30:],
thread_count=int(threads),
index='yindex',
doc_type='modules',
request_timeout=40):
if not success:
LOGGER.error(
'A elasticsearch document failed with info: {}, process {}'
.format(info, process_name))
query = {}
query['module'] = name
query['organization'] = resolve_organization(parsed_module)
query['revision'] = revision
query['dir'] = parsed_module.pos.ref
LOGGER.debug(
'pushing data to modules index, process {}'.format(
process_name))
es.index(index='modules',
doc_type='modules',
body=query,
request_timeout=40)
break
except (ConnectionTimeout, ConnectionError) as e:
retry = retry - 1
if retry > 0:
LOGGER.warning(
'module {}@{} timed out, process {}'.format(
name, revision, process_name))
else:
LOGGER.error(
'module {}@{} timed out too many times failing, process'
.format(name, revision, process_name))
raise e
with open('{}/{}@{}.json'.format(ytree_dir, name, revision),
'w') as f:
try:
emit_tree([parsed_module], f, ctx)
except:
# create empty file so we still have access to that
f.write("")
with lock_write_rest:
with open('{}/rest-of-elk-data.json'.format(temp_dir),
'w') as f:
json.dump(list(modules_copy), f)
except Exception as e:
with lock_exception:
with open(log_file, 'a') as f:
traceback.print_exc(file=f)
m_parts = module.split(":")
key = '{}/{}'.format(m_parts[0].split('/')[-1][:-5],
m_parts[1])
LOGGER.warning("Exception while adding {}, process".format(
key, process_name))
val = m_parts[0]
with open(failed_changes_dir, 'r') as f:
failed_mods = json.load(f)
if key not in failed_mods:
failed_mods[key] = val
with open(failed_changes_dir, 'w') as f:
json.dump(failed_mods, f)
def generate_from(self, module):
"""Generates classes, schema file and pkginfo files for module,
according to options set in self.ctx. The attributes self.directory
and self.d are used to determine where to generate the files.
module -- Module statement to generate files from
"""
if module in self.done:
return
self.done.add(module)
subpkg = util.camelize(module.arg)
if self.ctx.rootpkg:
fullpkg = '.'.join([self.ctx.rootpkg, subpkg])
else:
fullpkg = subpkg
d = os.sep.join(self.d + [subpkg])
if not self.ctx.opts.no_classes:
# Generate Java classes
src = ('module "' + module.arg + '", revision: "' +
putil.get_latest_revision(module) + '".')
generator = ClassGenerator(module,
path=os.sep.join(
[self.ctx.opts.directory, subpkg]),
package=fullpkg,
src=src,
ctx=self.ctx)
generator.generate()
if not self.ctx.opts.no_schema:
# Generate external schema
schema_nodes = ['<schema>']
stmts = util.search(module, context.node_stmts)
module_root = SchemaNode(self.ctx, module, '/')
schema_nodes.extend(module_root.as_list())
if self.ctx.opts.verbose:
print('Generating schema node "/"...')
schema_generator = SchemaGenerator(stmts, '/', self.ctx, fullpkg)
schema_nodes.extend(schema_generator.schema_nodes())
for i in range(1, len(schema_nodes)):
# Indent all but the first and last line
if schema_nodes[i] in ('<node>', '</node>'):
schema_nodes[i] = ' ' * 4 + schema_nodes[i]
else:
schema_nodes[i] = ' ' * 8 + schema_nodes[i]
schema_nodes.append('</schema>')
name = util.normalize(util.search_one(module, 'prefix').arg)
# Replace schema files store path
s = d
if self.ctx.opts.classpath_schema_loading:
s = self.ctx.opts.classpath_schema_loading
util.write_file(s, name + '.xml', '\n'.join(schema_nodes),
self.ctx)
print("- Visiting module '" + name + "'")
if not self.ctx.opts.no_pkginfo:
# Generate package-info.java for javadoc
pkginfo_generator = PackageInfoGenerator(d, module, self.ctx)
pkginfo_generator.generate_package_info()
if self.ctx.opts.debug or self.ctx.opts.verbose:
print('pkg ' + fullpkg + ' generated')
def build_yindex(ytree_dir, modules, LOGGER, save_file_dir, es_host, es_port,
es_aws, elk_credentials, threads, log_file,
failed_changes_dir, temp_dir, processes):
if es_aws:
es = Elasticsearch([es_host],
http_auth=(elk_credentials[0], elk_credentials[1]),
scheme="https",
port=443)
else:
es = Elasticsearch([{'host': '{}'.format(es_host), 'port': es_port}])
initialize_body_yindex = json.load(
open('json/initialize_yindex_elasticsearch.json', 'r'))
initialize_body_modules = json.load(
open('json/initialize_module_elasticsearch.json', 'r'))
logging.getLogger('elasticsearch').setLevel(logging.ERROR)
for i in range(0, 5, 1):
try:
es.indices.create(index='yindex',
body=initialize_body_yindex,
ignore=400)
es.indices.create(index='modules',
body=initialize_body_modules,
ignore=400)
except ConnectionError:
import time
LOGGER.warning(
"Could not connect to elasticsearch waiting 30 seconds")
time.sleep(30)
# it must be able to connect in here
es.ping()
x = 0
modules_copy = modules.copy()
for module in modules:
try:
modules_copy.remove(module)
x += 1
LOGGER.info('yindex on module {}. module {} out of {}'.format(
module.split('/')[-1], x, len(modules)))
# split to module with path and organization
m_parts = module.split(":")
m = m_parts[0]
plugin.init([])
ctx = create_context('{}'.format(save_file_dir))
ctx.opts.lint_namespace_prefixes = []
ctx.opts.lint_modulename_prefixes = []
for p in plugin.plugins:
p.setup_ctx(ctx)
with open(m, 'r') as f:
parsed_module = ctx.add_module(m, f.read())
ctx.validate()
if parsed_module is None:
raise Exception('Unable to pyang parse module')
f = io.StringIO()
ctx.opts.print_revision = True
emit_name(ctx, [parsed_module], f)
name_revision = f.getvalue().strip()
mods = [parsed_module]
find_submodules(ctx, mods, parsed_module)
f = io.StringIO()
ctx.opts.yang_index_make_module_table = True
ctx.opts.yang_index_no_schema = True
indexerPlugin = IndexerPlugin()
indexerPlugin.emit(ctx, [parsed_module], f)
yindexes = json.loads(f.getvalue())
name_revision = name_revision.split('@')
if len(name_revision) > 1:
name = name_revision[0]
revision = name_revision[1].split(' ')[0]
else:
name = name_revision[0]
revision = '1970-01-01'
if 'belongs-to' in name:
name = name.split(' ')[0]
try:
dateutil.parser.parse(revision)
except Exception as e:
if revision[-2:] == '29' and revision[-5:-3] == '02':
revision = revision.replace('02-29', '02-28')
else:
revision = '1970-01-01'
rev_parts = revision.split('-')
try:
revision = datetime(int(rev_parts[0]), int(rev_parts[1]),
int(rev_parts[2])).date().isoformat()
except Exception:
revision = '1970-01-01'
retry = 3
while retry > 0:
try:
for m in mods:
n = m.arg
rev = get_latest_revision(m)
if rev == 'unknown':
r = '1970-01-01'
else:
r = rev
try:
dateutil.parser.parse(r)
except Exception as e:
if r[-2:] == '29' and r[-5:-3] == '02':
r = r.replace('02-29', '02-28')
else:
r = '1970-01-01'
rev_parts = r.split('-')
r = datetime(int(rev_parts[0]), int(rev_parts[1]),
int(rev_parts[2])).date().isoformat()
try:
query = \
{
"query": {
"bool": {
"must": [{
"match_phrase": {
"module.keyword": {
"query": n
}
}
}, {
"match_phrase": {
"revision": {
"query": r
}
}
}]
}
}
}
LOGGER.debug('deleting data from yindex')
es.delete_by_query(index='yindex',
body=query,
doc_type='modules',
conflicts='proceed',
request_timeout=40)
except NotFoundError as e:
pass
for key in yindexes:
j = -1
for j in range(0, int(len(yindexes[key]) / 30)):
LOGGER.debug(
'pushing new data to yindex {} of {}'.format(
j, int(len(yindexes[key]) / 30)))
for success, info in parallel_bulk(
es,
yindexes[key][j * 30:(j * 30) + 30],
thread_count=int(threads),
index='yindex',
doc_type='modules',
request_timeout=40):
if not success:
LOGGER.error(
'A elasticsearch document failed with info: {}'
.format(info))
LOGGER.debug('pushing rest of data to yindex')
for success, info in parallel_bulk(
es,
yindexes[key][(j * 30) + 30:],
thread_count=int(threads),
index='yindex',
doc_type='modules',
request_timeout=40):
if not success:
LOGGER.error(
'A elasticsearch document failed with info: {}'
.format(info))
rev = get_latest_revision(parsed_module)
if rev == 'unknown':
revision = '1970-01-01'
else:
revision = rev
try:
dateutil.parser.parse(revision)
except Exception as e:
if revision[-2:] == '29' and revision[-5:-3] == '02':
revision = revision.replace('02-29', '02-28')
else:
revision = '1970-01-01'
rev_parts = revision.split('-')
revision = datetime(int(rev_parts[0]), int(rev_parts[1]),
int(rev_parts[2])).date().isoformat()
query = \
{
"query": {
"bool": {
"must": [{
"match_phrase": {
"module.keyword": {
"query": name
}
}
}, {
"match_phrase": {
"revision": {
"query": revision
}
}
}]
}
}
}
LOGGER.debug('deleting data from modules index')
total = es.delete_by_query(index='modules',
body=query,
doc_type='modules',
conflicts='proceed',
request_timeout=40)['deleted']
if total > 1:
LOGGER.info('{}@{}'.format(name, revision))
query = {}
query['module'] = name
query['organization'] = resolve_organization(parsed_module)
query['revision'] = revision
query['dir'] = parsed_module.pos.ref
LOGGER.debug('pushing data to modules index')
es.index(index='modules',
doc_type='modules',
body=query,
request_timeout=40)
break
except (ConnectionTimeout, ConnectionError) as e:
retry = retry - 1
if retry > 0:
LOGGER.warning('module {}@{} timed out'.format(
name, revision))
else:
LOGGER.error(
'module {}@{} timed out too many times failing'.
format(name, revision))
raise e
with open('{}/{}@{}.json'.format(ytree_dir, name, revision),
'w') as f:
try:
emit_tree([parsed_module], f, ctx)
except Exception as e:
# create empty file so we still have access to that
LOGGER.warning(
'unable to create ytree for module {}@{} creating empty file'
)
f.write("")
with open('{}/rest-of-elk-data.json'.format(temp_dir), 'w') as f:
json.dump(modules_copy, f)
except Exception as e:
with open(log_file, 'a') as f:
traceback.print_exc(file=f)
m_parts = module.split(":")
key = '{}/{}'.format(m_parts[0].split('/')[-1][:-5], m_parts[1])
val = m_parts[0]
with open(failed_changes_dir, 'r') as f:
failed_mods = json.load(f)
if key not in failed_mods:
failed_mods[key] = val
with open(failed_changes_dir, 'w') as f:
json.dump(failed_mods, f)