def emit_capability(ctx, m, fd): amp = "&" if ctx.opts.capa_entity else "&" ns = m.search_one('namespace') if ns is None: return s = ns.arg + "?module=" + m.i_modulename latest_rev = util.get_latest_revision(m) if latest_rev != "unknown": s = s + amp + "revision=" + latest_rev if m.i_modulename in ctx.features: if len(ctx.features[m.i_modulename]) > 0: s = s + amp + "features=" + ",".join(ctx.features[m.i_modulename]) else: # do not report any features from the module pass else: # report all features defined in the module fs = [x.arg for x in m.search('feature')] s = s + amp + "features=" + ",".join(fs) devs = [] for d in ctx.deviation_modules: # check if this deviation module deviates anything in our module for dev in d.search('deviation'): if (dev.i_target_node is not None and dev.i_target_node.i_module.i_modulename == m.i_modulename): devs.append(d.i_modulename) break if len(devs) > 0: s = s + amp + "deviations=" + ",".join(devs) fd.write(s + '\n')
def emit(self, ctx, modules): res = {} for module in modules: res['revision'] = util.get_latest_revision(module) res['name'] = module.arg for statement in ['namespace', 'prefix', 'revision', 'module-version']: stmt = module.search_one(statement) if stmt: res[stmt.keyword] = stmt.arg istmts = module.search('import') if istmts: if 'dependencies' not in res: res['dependencies'] = {} res['dependencies']['required-module'] = [] for istmt in istmts: # res['dependencies']['required-module'].append(istmt.arg) # TODO: Need to ask draft authors to add revision-date to YANG revision = 'unknown' r = istmt.search_one('revision-date') if r is not None: revision = r.arg res['dependencies']['required-module'].append({'module-name': istmt.arg,'module-revision': revision}) if module.keyword == 'submodule': if 'module-hierarchy' not in res: res['module-hierarchy'] = {} res['module-hierarchy']['module-hierarchy-level'] = 2 belongs = module.search_one('belongs-to') res['module-hierarchy']['module-parent'] = belongs.arg return res
def emit_capability(ctx, m, fd): amp = "&" if ctx.opts.capa_entity else "&" ns = m.search_one('namespace') if ns is None: return s = ns.arg + "?module=" + m.i_modulename latest_rev = util.get_latest_revision(m) if latest_rev != "unknown": s = s + amp + "revision=" + latest_rev if m.i_modulename in ctx.features: if len(ctx.features[m.i_modulename]) > 0: s = s + amp + "features=" + ",".join(ctx.features[m.i_modulename]) else: # do not report any features from the module pass else: # report all features defined in the module fs = [x.arg for x in m.search('feature')] s = s + amp + "features=" + ",".join(fs) devs = [] for d in ctx.deviation_modules: # check if this deviation module deviates anything in our module for dev in d.search('deviation'): if (dev.i_target_node is not None and dev.i_target_node.i_module.i_modulename == m.i_modulename): devs.append(d.i_modulename) break if len(devs) > 0: s = s + amp + "deviations=" + ",".join(devs) fd.write(s + '\n')
def index_printer(stmt): global _yang_catalog_index_values global _values global _ctx if stmt.arg is None: return skey = flatten_keyword(stmt.keyword) module = stmt.i_module rev = get_latest_revision(module) revision = '' path = statements.mk_path_str(stmt, True) descr = stmt.search_one('description') dstr = '' if descr: dstr = descr.arg dstr = dstr.replace("'", "''") subs = [] if rev == 'unknown': revision = '1970-01-01' else: revision = rev for i in stmt.substmts: a = i.arg k = i.keyword k = flatten_keyword(k) if i.keyword not in statements.data_definition_keywords: subs.append(index_get_other(i)) else: has_children = hasattr(i, 'i_children') and len(i.i_children) > 0 if not a: a = '' else: a = index_escape_json(a) subs.append({ k: { 'value': a, 'has_children': has_children, 'children': [] } }) vals = {} vals['module'] = module.arg vals['revision'] = revision vals['organization'] = resolve_organization(module) vals['path'] = path vals['statement'] = skey vals['argument'] = stmt.arg vals['description'] = dstr vals['properties'] = json.dumps(subs) _values['yindex'].append(vals)
def emit_dot(ctx, modules, fd): seen = [] fd.write("digraph \"modules-graph\" {\n") for module in modules: modname = module.arg modrev = util.get_latest_revision(module) modid = "%s@%s" % (modname, modrev) if modid in seen: # Ignoring duplicate modules continue else: seen.append(modid) fd.write("\t\"%s\"\n" % (modid)) imps = module.search('import') if imps: for imp in imps: if imp.search('revision-date'): date = imp.search_one('revision-date').arg fd.write( "\t\"%s\" -> \"%s\" [label=\"imports by revision: %s\"]\n" % (modname, imp.arg, date)) else: fd.write("\t\"%s\" -> \"%s\" [label=imports]\n" % (modname, imp.arg)) subs = module.search('include') if subs: for sub in subs: if sub.search('revision-date'): date = sub.search_one('revision-date').arg fd.write( "\t\"%s\" -> \"%s\" [label=\"includes by revision: %s\"]\n" % (modname, sub.arg, date)) else: fd.write("\t\"%s\" -> \"%s\" [label=includes]\n" % (modname, sub.arg)) fd.write("}")
def emit_dot(ctx, modules, fd): seen = [] fd.write("digraph \"modules-graph\" {\n") for module in modules: modname = module.arg modrev = util.get_latest_revision(module) modid = "%s@%s" % (modname, modrev) if modid in seen: # Ignoring duplicate modules continue else: seen.append(modid) fd.write("\t\"%s\"\n" % (modid)) imps = module.search('import') if imps: for imp in imps: if imp.search('revision-date'): date = imp.search_one('revision-date').arg fd.write("\t\"%s\" -> \"%s\" [label=\"imports by revision: %s\"]\n" % (modname, imp.arg, date)) else: fd.write("\t\"%s\" -> \"%s\" [label=imports]\n" % (modname, imp.arg)) subs = module.search('include') if subs: for sub in subs: if sub.search('revision-date'): date = sub.search_one('revision-date').arg fd.write("\t\"%s\" -> \"%s\" [label=\"includes by revision: %s\"]\n" % (modname, sub.arg, date)) else: fd.write("\t\"%s\" -> \"%s\" [label=includes]\n" % (modname, sub.arg)) fd.write("}")
def process_sid_file(self, module): self.module_name = module.i_modulename self.module_revision = util.get_latest_revision(module) self.output_file_name = '%s@%s.sid' % (self.module_name, self.module_revision) if self.range is not None: if self.range == 'count': self.count = True else: self.set_sid_range(self.range) if self.input_file_name is not None: if self.input_file_name[-4:] != ".sid": raise SidParsingError("File '%s' is not a .sid file" % self.input_file_name) with open(self.input_file_name) as f: self.content = json.load(f, object_pairs_hook=collections.OrderedDict) self.upgrade_sid_file_format() # This function can be removed after a reasonable transition period. self.validate_key_and_value() self.validate_ovelaping_ranges() self.validate_sid() if self.extra_range is not None: if self.extra_range == 'count': self.count = True else: self.set_sid_range(self.extra_range) self.validate_ovelaping_ranges() self.set_module_information() self.collect_module_items(module) if self.range == 'count': number_of_unassigned_yang_items = self.number_of_unassigned_yang_items() print("\nThis YANG module requires %d SIDs." % number_of_unassigned_yang_items) return if self.extra_range == 'count': number_of_SIDs_allocated = self.number_of_SIDs_allocated() number_of_SIDs_used = self.number_of_SIDs_used() number_of_SIDs_available = number_of_SIDs_allocated - number_of_SIDs_used number_of_unassigned_yang_items = self.number_of_unassigned_yang_items() print("\nNumber of SIDs allocated to this module: %d" % number_of_SIDs_allocated) print("Number of SIDs required by this version: %d" % (number_of_SIDs_used + number_of_unassigned_yang_items)) if (number_of_unassigned_yang_items > number_of_SIDs_available): print("\nAn extra range of at least %d SIDs is required to perform this update." % (number_of_unassigned_yang_items - number_of_SIDs_available)) else: print("\nThe update of the .sid file can be performed using the currently available SIDs.") return self.sort_items() self.assign_sid() if self.list_content: self.list_all_items() else: self.list_deleted_items() if self.check_consistency: if self.is_consistent: if self.sid_registration_info: self.print_registration_information(module) else: print("\nCheck completed successfully") else: print("\nThe .sid file need to be updated.") else: if self.is_consistent: print("No .sid file generated, the current .sid file is already up to date.") else: self.generate_file() if self.sid_file_created: print("\nFile %s created" % self.output_file_name) else: print("\nFile %s updated" % self.output_file_name) print ("Number of SIDs available : %d" % self.number_of_SIDs_allocated()) print ("Number of SIDs used : %d" % self.number_of_SIDs_used())
def index_printer(stmt): global _yang_catalog_index_values global _values global _ctx if stmt.arg is None: return skey = flatten_keyword(stmt.keyword) module = stmt.i_module rev = get_latest_revision(module) revision = '' path = mk_path_str(stmt, True) descr = stmt.search_one('description') dstr = '' if descr: dstr = descr.arg dstr = dstr.replace("'", "''") subs = [] if rev == 'unknown': revision = '1970-01-01' else: revision = rev try: dateutil.parser.parse(revision) except Exception as e: if revision[-2:] == '29' and revision[-5:-3] == '02': revision = revision.replace('02-29', '02-28') else: revision = '1970-01-01' rev_parts = revision.split('-') revision = datetime(int(rev_parts[0]), int(rev_parts[1]), int(rev_parts[2])).date().isoformat() for i in stmt.substmts: a = i.arg k = i.keyword k = flatten_keyword(k) if i.keyword not in statements.data_definition_keywords: subs.append(index_get_other(i)) else: has_children = hasattr(i, 'i_children') and len(i.i_children) > 0 if not a: a = '' else: a = index_escape_json(a) subs.append( {k: {'value': a, 'has_children': has_children, 'children': []}}) vals = {} vals['module'] = module.arg vals['revision'] = revision vals['organization'] = resolve_organization(module) vals['path'] = path vals['statement'] = skey vals['argument'] = stmt.arg vals['description'] = dstr vals['properties'] = json.dumps(subs) text = '{} {} {} {} {} {} {} {}'.format(vals['module'], vals['revision'], vals['organization'], vals['path'], vals['statement'], vals['argument'], vals['description'], vals['properties']) vals['sort-hash-id'] = hashlib.sha256(text.encode('utf-8')).hexdigest() _values['yindex'].append(vals)
def build_and_populate(lock_exception, lock_write_rest, lock_remove_list, lock_remove_elk, lock_index_elk, es, modules, LOGGER, save_file_dir, threads, log_file, failed_changes_dir, temp_dir, ytree_dir, process_name): x = 0 for module in modules: try: with lock_remove_list: modules_copy.remove(module) x += 1 LOGGER.info( 'yindex on module {}. module {} out of {}, process {}'.format( module.split('/')[-1], x, len(modules), process_name)) # split to module with path and organization [email protected]:cisco m_parts = module.split(":") m = m_parts[0] plugin.init([]) ctx = create_context('{}'.format(save_file_dir)) ctx.opts.lint_namespace_prefixes = [] ctx.opts.lint_modulename_prefixes = [] for p in plugin.plugins: p.setup_ctx(ctx) with open(m, 'r') as f: parsed_module = ctx.add_module(m, f.read()) ctx.validate() if parsed_module is None: raise Exception('Unable to pyang parse module') f = io.StringIO() ctx.opts.print_revision = True emit_name(ctx, [parsed_module], f) name_revision = f.getvalue().strip() mods = [parsed_module] find_submodules(ctx, mods, parsed_module) f = io.StringIO() ctx.opts.yang_index_make_module_table = True ctx.opts.yang_index_no_schema = True indexerPlugin = IndexerPlugin() indexerPlugin.emit(ctx, [parsed_module], f) yindexes = json.loads(f.getvalue()) name_revision = name_revision.split('@') if len(name_revision) > 1: name = name_revision[0] revision = name_revision[1].split(' ')[0] else: name = name_revision[0] revision = '1970-01-01' if 'belongs-to' in name: name = name.split(' ')[0] try: dateutil.parser.parse(revision) except Exception as e: if revision[-2:] == '29' and revision[-5:-3] == '02': revision = revision.replace('02-29', '02-28') else: revision = '1970-01-01' rev_parts = revision.split('-') try: revision = datetime(int(rev_parts[0]), int(rev_parts[1]), int(rev_parts[2])).date().isoformat() except Exception: revision = '1970-01-01' retry = 3 while retry > 0: try: for m in mods: n = m.arg rev = get_latest_revision(m) if rev == 'unknown': r = '1970-01-01' else: r = rev try: dateutil.parser.parse(r) except Exception as e: if r[-2:] == '29' and r[-5:-3] == '02': r = r.replace('02-29', '02-28') else: r = '1970-01-01' rev_parts = r.split('-') r = datetime(int(rev_parts[0]), int(rev_parts[1]), int(rev_parts[2])).date().isoformat() query = \ { "query": { "bool": { "must": [{ "match_phrase": { "module.keyword": { "query": n } } }, { "match_phrase": { "revision": { "query": r } } }] } } } while lock_index_elk.locked(): sleep(1) with lock_remove_elk: try: LOGGER.debug( 'deleting data from yindex index, process {}' .format(process_name)) es.delete_by_query(index='yindex', body=query, doc_type='modules', conflicts='proceed', request_timeout=40) LOGGER.debug( 'deleting data from modules index, process {}' .format(process_name)) total = es.delete_by_query( index='modules', body=query, doc_type='modules', conflicts='proceed', request_timeout=40)['deleted'] if total > 1: LOGGER.info('{}@{}, process {}'.format( name, revision, process_name)) except NotFoundError as e: pass while lock_remove_elk.locked(): sleep(1) if not lock_index_elk.locked(): lock_index_elk.acquire() for key in yindexes: j = -1 for j in range(0, int(len(yindexes[key]) / 30)): LOGGER.debug( 'pushing new data to yindex {} of {} process {}' .format(j, int(len(yindexes[key]) / 30), process_name)) for success, info in parallel_bulk( es, yindexes[key][j * 30:(j * 30) + 30], thread_count=int(threads), index='yindex', doc_type='modules', request_timeout=40): if not success: LOGGER.error( 'A elasticsearch document failed with info: {}, process {}' .format(info, process_name)) LOGGER.debug('pushing new data to yindex last one') for success, info in parallel_bulk( es, yindexes[key][(j * 30) + 30:], thread_count=int(threads), index='yindex', doc_type='modules', request_timeout=40): if not success: LOGGER.error( 'A elasticsearch document failed with info: {}, process {}' .format(info, process_name)) query = {} query['module'] = name query['organization'] = resolve_organization(parsed_module) query['revision'] = revision query['dir'] = parsed_module.pos.ref LOGGER.debug( 'pushing data to modules index, process {}'.format( process_name)) es.index(index='modules', doc_type='modules', body=query, request_timeout=40) break except (ConnectionTimeout, ConnectionError) as e: retry = retry - 1 if retry > 0: LOGGER.warning( 'module {}@{} timed out, process {}'.format( name, revision, process_name)) else: LOGGER.error( 'module {}@{} timed out too many times failing, process' .format(name, revision, process_name)) raise e with open('{}/{}@{}.json'.format(ytree_dir, name, revision), 'w') as f: try: emit_tree([parsed_module], f, ctx) except: # create empty file so we still have access to that f.write("") with lock_write_rest: with open('{}/rest-of-elk-data.json'.format(temp_dir), 'w') as f: json.dump(list(modules_copy), f) except Exception as e: with lock_exception: with open(log_file, 'a') as f: traceback.print_exc(file=f) m_parts = module.split(":") key = '{}/{}'.format(m_parts[0].split('/')[-1][:-5], m_parts[1]) LOGGER.warning("Exception while adding {}, process".format( key, process_name)) val = m_parts[0] with open(failed_changes_dir, 'r') as f: failed_mods = json.load(f) if key not in failed_mods: failed_mods[key] = val with open(failed_changes_dir, 'w') as f: json.dump(failed_mods, f)
def generate_from(self, module): """Generates classes, schema file and pkginfo files for module, according to options set in self.ctx. The attributes self.directory and self.d are used to determine where to generate the files. module -- Module statement to generate files from """ if module in self.done: return self.done.add(module) subpkg = util.camelize(module.arg) if self.ctx.rootpkg: fullpkg = '.'.join([self.ctx.rootpkg, subpkg]) else: fullpkg = subpkg d = os.sep.join(self.d + [subpkg]) if not self.ctx.opts.no_classes: # Generate Java classes src = ('module "' + module.arg + '", revision: "' + putil.get_latest_revision(module) + '".') generator = ClassGenerator(module, path=os.sep.join( [self.ctx.opts.directory, subpkg]), package=fullpkg, src=src, ctx=self.ctx) generator.generate() if not self.ctx.opts.no_schema: # Generate external schema schema_nodes = ['<schema>'] stmts = util.search(module, context.node_stmts) module_root = SchemaNode(self.ctx, module, '/') schema_nodes.extend(module_root.as_list()) if self.ctx.opts.verbose: print('Generating schema node "/"...') schema_generator = SchemaGenerator(stmts, '/', self.ctx, fullpkg) schema_nodes.extend(schema_generator.schema_nodes()) for i in range(1, len(schema_nodes)): # Indent all but the first and last line if schema_nodes[i] in ('<node>', '</node>'): schema_nodes[i] = ' ' * 4 + schema_nodes[i] else: schema_nodes[i] = ' ' * 8 + schema_nodes[i] schema_nodes.append('</schema>') name = util.normalize(util.search_one(module, 'prefix').arg) # Replace schema files store path s = d if self.ctx.opts.classpath_schema_loading: s = self.ctx.opts.classpath_schema_loading util.write_file(s, name + '.xml', '\n'.join(schema_nodes), self.ctx) print("- Visiting module '" + name + "'") if not self.ctx.opts.no_pkginfo: # Generate package-info.java for javadoc pkginfo_generator = PackageInfoGenerator(d, module, self.ctx) pkginfo_generator.generate_package_info() if self.ctx.opts.debug or self.ctx.opts.verbose: print('pkg ' + fullpkg + ' generated')
def build_yindex(ytree_dir, modules, LOGGER, save_file_dir, es_host, es_port, es_aws, elk_credentials, threads, log_file, failed_changes_dir, temp_dir, processes): if es_aws: es = Elasticsearch([es_host], http_auth=(elk_credentials[0], elk_credentials[1]), scheme="https", port=443) else: es = Elasticsearch([{'host': '{}'.format(es_host), 'port': es_port}]) initialize_body_yindex = json.load( open('json/initialize_yindex_elasticsearch.json', 'r')) initialize_body_modules = json.load( open('json/initialize_module_elasticsearch.json', 'r')) logging.getLogger('elasticsearch').setLevel(logging.ERROR) for i in range(0, 5, 1): try: es.indices.create(index='yindex', body=initialize_body_yindex, ignore=400) es.indices.create(index='modules', body=initialize_body_modules, ignore=400) except ConnectionError: import time LOGGER.warning( "Could not connect to elasticsearch waiting 30 seconds") time.sleep(30) # it must be able to connect in here es.ping() x = 0 modules_copy = modules.copy() for module in modules: try: modules_copy.remove(module) x += 1 LOGGER.info('yindex on module {}. module {} out of {}'.format( module.split('/')[-1], x, len(modules))) # split to module with path and organization m_parts = module.split(":") m = m_parts[0] plugin.init([]) ctx = create_context('{}'.format(save_file_dir)) ctx.opts.lint_namespace_prefixes = [] ctx.opts.lint_modulename_prefixes = [] for p in plugin.plugins: p.setup_ctx(ctx) with open(m, 'r') as f: parsed_module = ctx.add_module(m, f.read()) ctx.validate() if parsed_module is None: raise Exception('Unable to pyang parse module') f = io.StringIO() ctx.opts.print_revision = True emit_name(ctx, [parsed_module], f) name_revision = f.getvalue().strip() mods = [parsed_module] find_submodules(ctx, mods, parsed_module) f = io.StringIO() ctx.opts.yang_index_make_module_table = True ctx.opts.yang_index_no_schema = True indexerPlugin = IndexerPlugin() indexerPlugin.emit(ctx, [parsed_module], f) yindexes = json.loads(f.getvalue()) name_revision = name_revision.split('@') if len(name_revision) > 1: name = name_revision[0] revision = name_revision[1].split(' ')[0] else: name = name_revision[0] revision = '1970-01-01' if 'belongs-to' in name: name = name.split(' ')[0] try: dateutil.parser.parse(revision) except Exception as e: if revision[-2:] == '29' and revision[-5:-3] == '02': revision = revision.replace('02-29', '02-28') else: revision = '1970-01-01' rev_parts = revision.split('-') try: revision = datetime(int(rev_parts[0]), int(rev_parts[1]), int(rev_parts[2])).date().isoformat() except Exception: revision = '1970-01-01' retry = 3 while retry > 0: try: for m in mods: n = m.arg rev = get_latest_revision(m) if rev == 'unknown': r = '1970-01-01' else: r = rev try: dateutil.parser.parse(r) except Exception as e: if r[-2:] == '29' and r[-5:-3] == '02': r = r.replace('02-29', '02-28') else: r = '1970-01-01' rev_parts = r.split('-') r = datetime(int(rev_parts[0]), int(rev_parts[1]), int(rev_parts[2])).date().isoformat() try: query = \ { "query": { "bool": { "must": [{ "match_phrase": { "module.keyword": { "query": n } } }, { "match_phrase": { "revision": { "query": r } } }] } } } LOGGER.debug('deleting data from yindex') es.delete_by_query(index='yindex', body=query, doc_type='modules', conflicts='proceed', request_timeout=40) except NotFoundError as e: pass for key in yindexes: j = -1 for j in range(0, int(len(yindexes[key]) / 30)): LOGGER.debug( 'pushing new data to yindex {} of {}'.format( j, int(len(yindexes[key]) / 30))) for success, info in parallel_bulk( es, yindexes[key][j * 30:(j * 30) + 30], thread_count=int(threads), index='yindex', doc_type='modules', request_timeout=40): if not success: LOGGER.error( 'A elasticsearch document failed with info: {}' .format(info)) LOGGER.debug('pushing rest of data to yindex') for success, info in parallel_bulk( es, yindexes[key][(j * 30) + 30:], thread_count=int(threads), index='yindex', doc_type='modules', request_timeout=40): if not success: LOGGER.error( 'A elasticsearch document failed with info: {}' .format(info)) rev = get_latest_revision(parsed_module) if rev == 'unknown': revision = '1970-01-01' else: revision = rev try: dateutil.parser.parse(revision) except Exception as e: if revision[-2:] == '29' and revision[-5:-3] == '02': revision = revision.replace('02-29', '02-28') else: revision = '1970-01-01' rev_parts = revision.split('-') revision = datetime(int(rev_parts[0]), int(rev_parts[1]), int(rev_parts[2])).date().isoformat() query = \ { "query": { "bool": { "must": [{ "match_phrase": { "module.keyword": { "query": name } } }, { "match_phrase": { "revision": { "query": revision } } }] } } } LOGGER.debug('deleting data from modules index') total = es.delete_by_query(index='modules', body=query, doc_type='modules', conflicts='proceed', request_timeout=40)['deleted'] if total > 1: LOGGER.info('{}@{}'.format(name, revision)) query = {} query['module'] = name query['organization'] = resolve_organization(parsed_module) query['revision'] = revision query['dir'] = parsed_module.pos.ref LOGGER.debug('pushing data to modules index') es.index(index='modules', doc_type='modules', body=query, request_timeout=40) break except (ConnectionTimeout, ConnectionError) as e: retry = retry - 1 if retry > 0: LOGGER.warning('module {}@{} timed out'.format( name, revision)) else: LOGGER.error( 'module {}@{} timed out too many times failing'. format(name, revision)) raise e with open('{}/{}@{}.json'.format(ytree_dir, name, revision), 'w') as f: try: emit_tree([parsed_module], f, ctx) except Exception as e: # create empty file so we still have access to that LOGGER.warning( 'unable to create ytree for module {}@{} creating empty file' ) f.write("") with open('{}/rest-of-elk-data.json'.format(temp_dir), 'w') as f: json.dump(modules_copy, f) except Exception as e: with open(log_file, 'a') as f: traceback.print_exc(file=f) m_parts = module.split(":") key = '{}/{}'.format(m_parts[0].split('/')[-1][:-5], m_parts[1]) val = m_parts[0] with open(failed_changes_dir, 'r') as f: failed_mods = json.load(f) if key not in failed_mods: failed_mods[key] = val with open(failed_changes_dir, 'w') as f: json.dump(failed_mods, f)