def prepare_for_es_removal(yc_api_prefix: str, modules_to_delete: list, save_file_dir: str, LOGGER: logging.Logger): for mod in modules_to_delete: name, revision_organization = mod.split('@') revision = revision_organization.split('/')[0] path_to_delete_local = '{}/{}@{}.yang'.format(save_file_dir, name, revision) data = {'input': {'dependents': [{'name': name}]}} response = requests.post('{}search-filter'.format(yc_api_prefix), json=data) if response.status_code == 200: data = response.json() modules = data['yang-catalog:modules']['module'] for mod in modules: redis_key = '{}@{}/{}'.format(mod['name'], mod['revision'], mod['organization']) redisConnection = RedisConnection() redisConnection.delete_dependent(redis_key, name) if os.path.exists(path_to_delete_local): os.remove(path_to_delete_local) post_body = {} if modules_to_delete: post_body = {'modules-to-delete': modules_to_delete} LOGGER.debug('Modules to delete:\n{}'.format(json.dumps(post_body, indent=2))) mf = messageFactory.MessageFactory() mf.send_removed_yang_files(json.dumps(post_body, indent=4)) return post_body
def prepare_for_es_indexing(yc_api_prefix: str, modules_to_index: str, LOGGER: logging.Logger, save_file_dir: str, force_indexing: bool = False): """ Sends the POST request which will activate indexing script for modules which will help to speed up process of searching. It will create a json body of all the modules containing module name and path where the module can be found if we are adding new modules. Arguments: :param yc_api_prefix (str) prefix for sending request to API :param modules_to_index (str) path to the prepare.json file generated while parsing :param LOOGER (logging.Logger) formated logger with the specified name :param save_file_dir (str) path to the directory where all the yang files will be saved :param force_indexing (bool) Whether or not we should force indexing even if module exists in cache. """ mf = messageFactory.MessageFactory() es_manager = ESManager() with open(modules_to_index, 'r') as f: sdos_json = json.load(f) LOGGER.debug('{} modules loaded from prepare.json'.format(len(sdos_json.get('module', [])))) post_body = {} load_new_files_to_github = False for module in sdos_json.get('module', []): url = '{}search/modules/{},{},{}'.format(yc_api_prefix, module['name'], module['revision'], module['organization']) response = requests.get(url, headers=json_headers) code = response.status_code in_es = False in_redis = code == 200 or code == 201 or code == 204 if in_redis: in_es = es_manager.document_exists(ESIndices.MODULES, module) else: load_new_files_to_github = True if force_indexing or not in_es or not in_redis: path = '{}/{}@{}.yang'.format(save_file_dir, module.get('name'), module.get('revision')) key = '{}@{}/{}'.format(module['name'], module['revision'], module['organization']) post_body[key] = path if post_body: post_body = {'modules-to-index': post_body} LOGGER.debug('Modules to index:\n{}'.format(json.dumps(post_body, indent=2))) mf.send_added_new_yang_files(json.dumps(post_body, indent=4)) if load_new_files_to_github: try: LOGGER.info('Calling draftPull.py script') module = __import__('ietfYangDraftPull', fromlist=['draftPull']) submodule = getattr(module, 'draftPull') submodule.main() except Exception: LOGGER.exception('Error occurred while running draftPull.py script') return post_body
def trigger_populate(): app.logger.info('Trigger populate if necessary') repoutil.pull(ac.d_yang_models_dir) try: assert request.json commits = request.json.get('commits') if request.is_json else None paths = set() new = [] mod = [] if commits: for commit in commits: added = commit.get('added') if added: for add in added: if 'platform-metadata.json' in add: paths.add('/'.join(add.split('/')[:-1])) new.append('/'.join(add.split('/')[:-1])) modified = commit.get('modified') if modified: for m in modified: if 'platform-metadata.json' in m: paths.add('/'.join(m.split('/')[:-1])) mod.append('/'.join(m.split('/')[:-1])) if len(paths) > 0: mf = messageFactory.MessageFactory() mf.send_new_modified_platform_metadata(new, mod) app.logger.info('Forking the repo') try: populate_path = os.path.join(os.environ['BACKEND'], 'parseAndPopulate/populate.py') arguments = ['python', populate_path, '--api-protocol', ac.g_protocol_api, '--api-port', ac.w_api_port, '--api-ip', ac.w_ip, '--result-html-dir', ac.w_result_html_dir, '--credentials', ac.s_confd_credentials[0], ac.s_confd_credentials[1], '--save-file-dir', ac.d_save_file_dir, 'repoLocalDir'] arguments = arguments + list(paths) + [ac.d_yang_models_dir, 'github'] ac.sender.send('#'.join(arguments)) except: app.logger.exception('Could not populate after git push') except Exception as e: app.logger.error('Automated github webhook failure - {}'.format(e)) return {'info': 'Success'}
def _patch(self, data: list, type: str, log_file: str) -> bool: errors = False chunk_size = 500 failed_data = {} chunks = [data[i:i + chunk_size] for i in range(0, len(data), chunk_size)] path = '{}/restconf/data/yang-catalog:catalog/{}/'.format(self.confd_prefix, type) self.LOGGER.debug('Sending PATCH request to patch multiple {}'.format(type)) for i, chunk in enumerate(chunks, start=1): self.LOGGER.debug('Processing chunk {} out of {}'.format(i, len(chunks))) patch_data = {type: {type.rstrip('s'): chunk}} patch_json = json.dumps(patch_data) response = requests.patch(path, patch_json, auth=(self.credentials[0], self.credentials[1]), headers=confd_headers) if response.status_code == 400: self.LOGGER.warning('Failed to batch patch {}, falling back to patching individually'.format(type)) for datum in chunk: patch_data = {type: {type.rstrip('s'): [datum]}} patch_json = json.dumps(patch_data) response = requests.patch(path, patch_json, auth=(self.credentials[0], self.credentials[1]), headers=confd_headers) if response.status_code == 400: errors = True with open(os.path.join(self.log_directory, log_file), 'a') as f: if type == 'modules': name_revision = '{}@{}'.format(datum['name'], datum['revision']) self.LOGGER.error('Failed to patch {} {}'.format(type.rstrip('s'), name_revision)) try: failed_data[name_revision] = json.loads(response.text) except json.decoder.JSONDecodeError: self.LOGGER.exception('No test in response') f.write('{}@{} error: {}\n'.format(datum['name'], datum['revision'], response.text)) elif type == 'vendors': platform_name = datum['platforms']['platform'][0]['name'] vendor_platform = '{} {}'.format(datum['name'], platform_name) self.LOGGER.error('Failed to patch {} {}'.format(type.rstrip('s'), vendor_platform)) try: failed_data[vendor_platform] = json.loads(response.text) except json.decoder.JSONDecodeError: self.LOGGER.exception('No test in response') f.write('{} {} error: {}\n'.format(datum['name'], platform_name, response.text)) if failed_data: mf = messageFactory.MessageFactory() mf.send_confd_writing_failures(type, failed_data) return errors
def check_local(): """Authorize sender if it is Travis, if travis job was sent from yang-catalog repository and job passed fine and Travis run a job on pushed patch, create a pull request to YangModules repository. If the job passed on this pull request, merge the pull request and remove the repository at yang-catalog repository :return response to the request """ app.logger.info('Starting pull request job') body = json.loads(request.form['payload']) app.logger.info('Body of travis {}'.format(json.dumps(body))) app.logger.info('type of job {}'.format(body['type'])) try: check_authorized(request.headers['SIGNATURE'], request.form['payload']) app.logger.info('Authorization successful') except: app.logger.exception('Authorization failed. Request did not come from Travis') mf = messageFactory.MessageFactory() mf.send_travis_auth_failed() abort(401) github_repos_url = '{}/repos'.format(github_api) yang_models_url = '{}/YangModels/yang'.format(github_repos_url) verify_commit = False app.logger.info('Checking commit SHA if it is the commit sent by yang-catalog user.') if body['repository']['owner_name'] == 'yang-catalog': commit_sha = body['commit'] else: commit_sha = body['head_commit'] try: with open(ac.d_commit_dir, 'r') as commit_file: for line in commit_file: if commit_sha in line: verify_commit = True break except: abort(404) token_header_value = 'token {}'.format(ac.s_yang_catalog_token) if verify_commit: app.logger.info('commit verified') if body['repository']['owner_name'] == 'yang-catalog': if body['result_message'] == 'Passed': if body['type'] in ['push', 'api']: # After build was successful only locally json_body = json.loads(json.dumps({ 'title': 'Cronjob - every day pull and update of ietf draft yang files.', 'body': 'ietf extracted yang modules', 'head': 'yang-catalog:main', 'base': 'main' })) url = '{}/pulls'.format(yang_models_url) r = requests.post(url, json=json_body, headers={'Authorization': token_header_value}) if r.status_code == 201: app.logger.info('Pull request created successfully') return ({'info': 'Success'}, 201) else: app.logger.error('Could not create a pull request {}'.format(r.status_code)) abort(400) else: app.logger.warning('Travis job did not pass.') return ({'info': 'Failed'}, 406) elif body['repository']['owner_name'] == 'YangModels': if body['result_message'] == 'Passed': if body['type'] == 'pull_request': # If build was successful on pull request admin_token_header_value = 'token {}'.format(ac.s_admin_token) pull_number = body['pull_request_number'] app.logger.info('Pull request was successful {}. sending review.'.format(repr(pull_number))) url = '{}/repos/YangModels/yang/pulls/{}/reviews'.format(github_api, repr(pull_number)) data = json.dumps({ 'body': 'AUTOMATED YANG CATALOG APPROVAL', 'event': 'APPROVE' }) response = requests.post(url, data, headers={'Authorization': admin_token_header_value}) app.logger.info('review response code {}. Merge response {}.'.format( response.status_code, response.text)) data = json.dumps({'commit-title': 'Travis job passed', 'sha': body['head_commit']}) response = requests.put('{}/repos/YangModels/yang/pulls/{}/merge'.format(github_api, repr(pull_number)), data, headers={'Authorization': admin_token_header_value}) app.logger.info('Merge response code {}. Merge response {}.'.format(response.status_code, response.text)) return ({'info': 'Success'}, 201) else: app.logger.warning('Travis job did not pass. Removing pull request') pull_number = body['pull_request_number'] json_body = json.loads(json.dumps({ 'title': 'Cron job - every day pull and update of ietf draft yang files.', 'body': 'ietf extracted yang modules', 'state': 'closed', 'base': 'main' })) requests.patch('{}/repos/YangModels/yang/pulls/{}'.format(github_api, pull_number), json=json_body, headers={'Authorization': token_header_value}) app.logger.warning('Travis job did not pass.') return ({'info': 'Failed'}, 406) else: app.logger.warning('Owner name verification failed. Owner -> {}'.format(body['repository']['owner_name'])) return ({'Error': 'Owner verfication failed'}, 401) else: app.logger.info('Commit verification failed. Commit sent by someone else.' 'Not doing anything.') return ({'Error': 'Fails'}, 500)
def main(scriptConf=None): start_time = int(time.time()) if scriptConf is None: scriptConf = ScriptConfig() args = scriptConf.args config_path = args.config_path config = create_config(config_path) token = config.get('Secrets-Section', 'yang-catalog-token') username = config.get('General-Section', 'repository-username') commit_dir = config.get('Directory-Section', 'commit-dir') config_name = config.get('General-Section', 'repo-config-name') config_email = config.get('General-Section', 'repo-config-email') log_directory = config.get('Directory-Section', 'logs') temp_dir = config.get('Directory-Section', 'temp') exceptions = config.get('Directory-Section', 'exceptions') yang_models = config.get('Directory-Section', 'yang-models-dir') ietf_draft_url = config.get('Web-Section', 'ietf-draft-private-url') ietf_rfc_url = config.get('Web-Section', 'ietf-RFC-tar-private-url') is_production = config.get('General-Section', 'is-prod') is_production = is_production == 'True' LOGGER = log.get_logger('draftPull', '{}/jobs/draft-pull.log'.format(log_directory)) LOGGER.info('Starting Cron job IETF pull request') repo_name = 'yang' repourl = 'https://{}@github.com/{}/{}.git'.format(token, username, repo_name) commit_author = {'name': config_name, 'email': config_email} draftPullUtility.update_forked_repository(yang_models, LOGGER) repo = draftPullUtility.clone_forked_repository(repourl, commit_author, LOGGER) if not repo: error_message = 'Failed to clone repository {}/{}'.format( username, repo_name) job_log(start_time, temp_dir, error=error_message, status='Fail', filename=os.path.basename(__file__)) sys.exit() try: # Get rfc.tgz file response = requests.get(ietf_rfc_url) tgz_path = '{}/rfc.tgz'.format(repo.local_dir) extract_to = '{}/standard/ietf/RFCtemp'.format(repo.local_dir) with open(tgz_path, 'wb') as zfile: zfile.write(response.content) tar_opened = draftPullUtility.extract_rfc_tgz(tgz_path, extract_to, LOGGER) if tar_opened: diff_files = [] new_files = [] temp_rfc_yang_files = glob.glob( '{}/standard/ietf/RFCtemp/*.yang'.format(repo.local_dir)) for temp_rfc_yang_file in temp_rfc_yang_files: file_name = os.path.basename(temp_rfc_yang_file) rfc_yang_file = temp_rfc_yang_file.replace('RFCtemp', 'RFC') if not os.path.exists(rfc_yang_file): new_files.append(file_name) continue same = filecmp.cmp(rfc_yang_file, temp_rfc_yang_file) if not same: diff_files.append(file_name) shutil.rmtree('{}/standard/ietf/RFCtemp'.format(repo.local_dir)) with open(exceptions, 'r') as exceptions_file: remove_from_new = exceptions_file.read().split('\n') new_files = [ file_name for file_name in new_files if file_name not in remove_from_new ] if args.send_message: if new_files or diff_files: LOGGER.info( 'new or modified RFC files found. Sending an E-mail') mf = messageFactory.MessageFactory() mf.send_new_rfc_message(new_files, diff_files) # Experimental draft modules try: os.makedirs('{}/experimental/ietf-extracted-YANG-modules/'.format( repo.local_dir)) except OSError as e: # be happy if someone already created the path if e.errno != errno.EEXIST: raise experimental_path = '{}/experimental/ietf-extracted-YANG-modules'.format( repo.local_dir) LOGGER.info('Updating IETF drafts download links') draftPullUtility.get_draft_module_content(ietf_draft_url, experimental_path, LOGGER) LOGGER.info('Checking module filenames without revision in {}'.format( experimental_path)) draftPullUtility.check_name_no_revision_exist(experimental_path, LOGGER) LOGGER.info( 'Checking for early revision in {}'.format(experimental_path)) draftPullUtility.check_early_revisions(experimental_path, LOGGER) messages = [] try: # Add commit and push to the forked repository LOGGER.info('Adding all untracked files locally') untracked_files = repo.repo.untracked_files repo.add_untracked_remove_deleted() LOGGER.info('Committing all files locally') repo.commit_all( 'Cronjob - every day pull of ietf draft yang files.') LOGGER.info('Pushing files to forked repository') commit_hash = repo.repo.head.commit LOGGER.info('Commit hash {}'.format(commit_hash)) with open(commit_dir, 'w+') as f: f.write('{}\n'.format(commit_hash)) if is_production: LOGGER.info( 'Pushing untracked and modified files to remote repository' ) repo.push() else: LOGGER.info( 'DEV environment - not pushing changes into remote repository' ) LOGGER.debug( 'List of all untracked and modified files:\n{}'.format( '\n'.join(untracked_files))) except GitCommandError as e: message = 'Error while pushing procedure - git command error: \n {} \n git command out: \n {}'.format( e.stderr, e.stdout) if 'Your branch is up to date' in e.stdout: LOGGER.warning(message) messages = [{ 'label': 'Pull request created', 'message': 'False - branch is up to date' }] else: LOGGER.exception( 'Error while pushing procedure - Git command error') raise e except Exception as e: LOGGER.exception('Error while pushing procedure {}'.format( sys.exc_info()[0])) raise type(e)('Error while pushing procedure') except Exception as e: LOGGER.exception('Exception found while running draftPull script') job_log(start_time, temp_dir, error=str(e), status='Fail', filename=os.path.basename(__file__)) raise e if len(messages) == 0: messages = [{ 'label': 'Pull request created', 'message': 'True - {}'.format(commit_hash) } # pyright: ignore ] job_log(start_time, temp_dir, messages=messages, status='Success', filename=os.path.basename(__file__)) LOGGER.info('Job finished successfully')
def parse_semver(self): def get_revision_datetime(module: dict): rev = module['revision'].split('-') try: date = datetime(int(rev[0]), int(rev[1]), int(rev[2])) except Exception: LOGGER.error( 'Failed to process revision for {}: (rev: {})'.format( module['name'], rev)) try: if int(rev[1]) == 2 and int(rev[2]) == 29: date = datetime(int(rev[0]), int(rev[1]), 28) else: date = datetime(1970, 1, 1) except Exception: date = datetime(1970, 1, 1) return date def increment_semver(old: str, significance: int): versions = old.split('.') versions = list(map(int, versions)) versions[significance] += 1 versions[significance + 1:] = [0] * len(versions[significance + 1:]) return '{}.{}.{}'.format(*versions) def update_semver(old_details: dict, new_module: dict, significance: int): upgraded_version = increment_semver(old_details['semver'], significance) new_module['derived-semantic-version'] = upgraded_version add_to_new_modules(new_module) def trees_match(new, old) -> bool: if type(new) != type(old): return False elif isinstance(new, dict): new.pop('description', None) old.pop('description', None) return new.keys() == old.keys() and all( (trees_match(new[i], old[i]) for i in new)) elif isinstance(new, list): return len(new) == len(old) and all( any((trees_match(i, j) for j in old)) for i in new) elif type(new) in (str, set, bool): return new == old else: assert False def get_trees(new: dict, old: dict): new_name_revision = '{}@{}'.format(new['name'], new['revision']) old_name_revision = '{}@{}'.format(old['name'], old['revision']) new_schema = '{}/{}.yang'.format(self._save_file_dir, new_name_revision) old_schema = '{}/{}.yang'.format(self._save_file_dir, old_name_revision) new_schema_exist = self._check_schema_file(new) old_schema_exist = self._check_schema_file(old) new_tree_path = '{}/{}.json'.format(self.json_ytree, new_name_revision) old_tree_path = '{}/{}.json'.format(self.json_ytree, old_name_revision) if old_schema_exist and new_schema_exist: ctx, new_schema_ctx = context_check_update_from( old_schema, new_schema, self._yang_models, self._save_file_dir) if len(ctx.errors) == 0: if os.path.exists(new_tree_path) and os.path.exists( old_tree_path): with open(new_tree_path) as nf, open( old_tree_path) as of: new_yang_tree = json.load(nf) old_yang_tree = json.load(of) else: with open(old_schema, 'r', errors='ignore') as f: old_schema_ctx = ctx.add_module( old_schema, f.read()) if ctx.opts.tree_path is not None: path = ctx.opts.tree_path.split('/') if path[0] == '': path = path[1:] else: path = None retry = 5 while retry: try: ctx.validate() break except Exception as e: retry -= 1 if retry == 0: raise e try: f = io.StringIO() emit_json_tree([new_schema_ctx], f, ctx) new_yang_tree = f.getvalue() with open(new_tree_path, 'w') as f: f.write(new_yang_tree) except: new_yang_tree = '' try: f = io.StringIO() emit_json_tree([old_schema_ctx], f, ctx) old_yang_tree = f.getvalue() with open(old_tree_path, 'w') as f: f.write(old_yang_tree) except: old_yang_tree = '2' return (new_yang_tree, old_yang_tree) else: raise Exception def add_to_new_modules(new_module: dict): name = new_module['name'] revision = new_module['revision'] if (revision not in self._existing_modules_dict[name] or self._existing_modules_dict[name][revision].get( 'derived-semantic-version') != new_module['derived-semantic-version']): LOGGER.info('semver {} vs {} for module {}@{}'.format( self._existing_modules_dict[name].get( revision, {}).get('derived-semantic-version'), new_module['derived-semantic-version'], name, revision)) if revision not in self.new_modules[name]: self.new_modules[name][revision] = new_module else: self.new_modules[name][revision][ 'derived-semantic-version'] = new_module[ 'derived-semantic-version'] z = 0 for new_module in self._all_modules.get('module', []): z += 1 name = new_module['name'] new_revision = new_module['revision'] name_revision = '{}@{}'.format(name, new_revision) data = defaultdict(dict) # Get all other available revisions of the module for m in self._existing_modules_dict[new_module['name']].values(): if m['revision'] != new_module['revision']: data[m['name']][m['revision']] = deepcopy(m) LOGGER.info('Searching semver for {}. {} out of {}'.format( name_revision, z, len(self._all_modules['module']))) if len(data) == 0: # If there is no other revision for this module new_module['derived-semantic-version'] = '1.0.0' add_to_new_modules(new_module) else: # If there is at least one revision for this module date = get_revision_datetime(new_module) module_temp = {} module_temp['name'] = name module_temp['revision'] = new_revision module_temp['organization'] = new_module['organization'] module_temp['compilation'] = new_module.get( 'compilation-status', 'PENDING') module_temp['date'] = date module_temp['schema'] = new_module['schema'] mod_details: t.List[t.Dict[str, str]] = [module_temp] # Loop through all other available revisions of the module revision = '' try: for mod in [ revision for name in data.values() for revision in name.values() ]: module_temp = {} revision = mod['revision'] if revision == new_module['revision']: continue module_temp['revision'] = revision module_temp['date'] = get_revision_datetime(mod) module_temp['name'] = name module_temp['compilation'] = mod.get( 'compilation-status', 'PENDING') module_temp['semver'] = mod['derived-semantic-version'] module_temp['organization'] = mod['organization'] module_temp['schema'] = mod['schema'] mod_details.append(module_temp) except KeyError as e: LOGGER.error( 'Existing module {}@{} is missing the {} field'.format( name, revision, e)) if str(e) == 'derived-semantic-version': LOGGER.error('Cannot resolve semver for {}@{}'.format( name, new_revision)) continue data[name][new_revision] = new_module mod_details = sorted(mod_details, key=lambda k: k['date']) # If we are adding a new module to the end (latest revision) of existing modules with this name # and all modules with this name have semver already assigned except for the last one if mod_details[-1]['date'] == date: if mod_details[-1]['compilation'] != 'passed': versions = mod_details[-2]['semver'].split('.') major_ver = int(versions[0]) major_ver += 1 upgraded_version = '{}.{}.{}'.format(major_ver, 0, 0) new_module[ 'derived-semantic-version'] = upgraded_version add_to_new_modules(new_module) else: if mod_details[-2]['compilation'] != 'passed': update_semver(mod_details[-2], new_module, 0) else: try: trees = get_trees(mod_details[-1], mod_details[-2]) # if schemas do not exist, trees will be None if trees: new_yang_tree, old_yang_tree = trees if trees_match(new_yang_tree, old_yang_tree): # yang trees are the same - update only the patch version update_semver(mod_details[-2], new_module, 2) else: # yang trees have changed - update minor version update_semver(mod_details[-2], new_module, 1) except: # pyang found an error - update major version update_semver(mod_details[-2], new_module, 0) # If we are adding new module in the middle (between two revisions) of existing modules with this name else: name = mod_details[0]['name'] revision = mod_details[0]['revision'] mod_details[0]['semver'] = '1.0.0' response = data[name][revision] response['derived-semantic-version'] = '1.0.0' add_to_new_modules(response) for x in range(1, len(mod_details)): name = mod_details[x]['name'] revision = mod_details[x]['revision'] module = data[name][revision] if mod_details[x]['compilation'] != 'passed': update_semver(mod_details[x - 1], module, 0) mod_details[x]['semver'] = increment_semver( mod_details[x - 1]['semver'], 0) else: # If the previous revision has the compilation status 'passed' if mod_details[x - 1]['compilation'] != 'passed': update_semver(mod_details[x - 1], module, 0) mod_details[x]['semver'] = increment_semver( mod_details[x - 1]['semver'], 0) else: # Both actual and previous revisions have the compilation status 'passed' try: trees = get_trees(mod_details[x], mod_details[x - 1]) # if schemas do not exist, trees will be None if trees: new_yang_tree, old_yang_tree = trees if trees_match(new_yang_tree, old_yang_tree): # yang trees are the same - update only the patch version update_semver( mod_details[x - 1], module, 2) mod_details[x][ 'semver'] = increment_semver( mod_details[x - 1]['semver'], 2) else: # yang trees have changed - update minor version update_semver( mod_details[x - 1], module, 1) mod_details[x][ 'semver'] = increment_semver( mod_details[x - 1]['semver'], 1) except: # pyang found an error - update major version update_semver(mod_details[x - 1], module, 0) mod_details[x][ 'semver'] = increment_semver( mod_details[x - 1]['semver'], 0) if len(self._unavailable_modules) != 0: mf = messageFactory.MessageFactory() mf.send_github_unavailable_schemas(self._unavailable_modules)
def on_request(ch, method, props, body): """Function called when something was sent from API sender. This function will process all the requests that would take too long to process for API. When the processing is done we will sent back the result of the request which can be either 'Failed' or 'Finished successfully' with corespondent correlation id. If the request 'Failed' it will sent back also a reason why it failed. Arguments: :param body: (str) String of arguments that need to be processed separated by '#'. """ try: if sys.version_info >= (3, 4): body = body.decode(encoding='utf-8', errors='strict') LOGGER.info('Received request with body {}'.format(body)) arguments = body.split('#') if body == 'run_ietf': LOGGER.info('Running all ietf and openconfig modules') final_response = run_ietf() elif 'github' == arguments[-1]: LOGGER.info('Github automated message starting to populate') paths_plus = arguments[arguments.index('repoLocalDir'):] LOGGER.info('paths plus {}'.format(paths_plus)) arguments = arguments[:arguments.index('repoLocalDir')] LOGGER.info('arguments {}'.format(arguments)) paths = paths_plus[1:-2] LOGGER.info('paths {}'.format(paths)) try: for path in paths: with open(temp_dir + "/log_trigger.txt", "w") as f: local_dir = paths_plus[-2] arguments = arguments + [ "--dir", local_dir + "/" + path ] subprocess.check_call(arguments, stderr=f) final_response = __response_type[1] except subprocess.CalledProcessError as e: final_response = __response_type[0] mf = messageFactory.MessageFactory() mf.send_automated_procedure_failed( arguments, temp_dir + "/log_no_sdo_api.txt") LOGGER.error( 'check log_trigger.txt Error calling process populate.py because {}\n\n with error {}' .format(e.stdout, e.stderr)) except: final_response = __response_type[0] LOGGER.error( "check log_trigger.txt failed to process github message with error {}" .format(sys.exc_info()[0])) else: global all_modules all_modules = None if arguments[-3] == 'DELETE': LOGGER.info('Deleting single module') if 'http' in arguments[0]: final_response = process_module_deletion(arguments) credentials = arguments[3:5] else: final_response = process_vendor_deletion(arguments) credentials = arguments[7:9] elif arguments[-3] == 'DELETE_MULTIPLE': LOGGER.info('Deleting multiple modules') final_response = process_module_deletion(arguments, True) credentials = arguments[3:5] elif '--sdo' in arguments[2]: final_response = process_sdo(arguments) credentials = arguments[11:13] direc = arguments[6] shutil.rmtree(direc) else: final_response = process_vendor(arguments) credentials = arguments[10:12] direc = arguments[5] shutil.rmtree(direc) if final_response.split('#split#')[0] == __response_type[1]: res = make_cache(credentials) if res.status_code != 201: final_response = __response_type[ 0] + '#split#Server error-> could not reload cache' if all_modules: complicatedAlgorithms = ModulesComplicatedAlgorithms( log_directory, yangcatalog_api_prefix, credentials, confd_protocol, confd_ip, confdPort, save_file_dir, None, all_modules, yang_models, temp_dir) complicatedAlgorithms.parse_non_requests() complicatedAlgorithms.parse_requests() complicatedAlgorithms.populate() except Exception as e: final_response = __response_type[0] LOGGER.error("receiver failed with message {}".format(e)) LOGGER.info('Receiver is done with id - {} and message = {}'.format( props.correlation_id, str(final_response))) f = open('{}/correlation_ids'.format(temp_dir), 'r') lines = f.readlines() f.close() with open('{}/correlation_ids'.format(temp_dir), 'w') as f: for line in lines: if props.correlation_id in line: new_line = '{} -- {} - {}\n'.format(datetime.now().ctime(), props.correlation_id, str(final_response)) f.write(new_line) else: f.write(line)
def prepare_to_indexing(yc_api_prefix, modules_to_index, credentials, apiIp=None, sdo_type=False, delete=False, from_api=True, force_indexing=True, LOOGER_temp=None, saveFilesDir=None, tempDir=None): """ Sends the POST request which will activate indexing script for modules which will help to speed up process of searching. It will create a json body of all the modules containing module name and path where the module can be found if we are adding new modules. Other situation can be if we need to delete module. In this case we are sending list of modules that need to be deleted. Arguments: :param apiIp: apiIp in case we can not use receiver s because other module is calling this method :param LOOGER_temp: LOGGER in case we can not use receiver s because other module is calling this method :param yc_api_prefix: (str) prefix for sending request to api :param modules_to_index: (json file) prepare.json file generated while parsing all the modules. This file is used to iterate through all the modules. :param credentials: (list) Basic authorization credentials - username, password respectively. :param sdo_type: (bool) Whether or not it is sdo that needs to be sent. :param delete: (bool) Whether or not we are deleting module. :param from_api: (bool) Whether or not api sent the request to index. :param set_key: (str) String containing key to confirm that it is receiver that sends data. This is is verified before indexing takes place. :param force_indexing: (bool) Whether or not we should force indexing even if module exists in cache. """ global api_ip global LOGGER global save_file_dir global temp_dir if tempDir is not None: temp_dir = tempDir if apiIp is not None: api_ip = apiIp if LOOGER_temp is not None: LOGGER = LOOGER_temp if saveFilesDir is not None: save_file_dir = saveFilesDir LOGGER.info('Sending data for indexing') mf = messageFactory.MessageFactory() if delete: body_to_send = json.dumps({'modules-to-delete': modules_to_index}, indent=4) mf.send_removed_yang_files(body_to_send) for mod in modules_to_index: name, revision_organization = mod.split('@') revision, organization = revision_organization.split('/') path_to_delete_local = "{}/{}@{}.yang".format( save_file_dir, name, revision) data = {'input': {'dependents': [{'name': name}]}} response = requests.post(yc_api_prefix + 'search-filter', auth=(credentials[0], credentials[1]), json={'input': data}) if response.status_code == 201: modules = response.json() for mod in modules: m_name = mod['name'] m_rev = mod['revision'] m_org = mod['organization'] url = ('{}://{}:{}/api/config/catalog/modules/module/' '{},{},{}/dependents/{}'.format( confd_protocol, confd_ip, confdPort, m_name, m_rev, m_org, name)) requests.delete(url, auth=(credentials[0], credentials[1]), headers={ 'Content-Type': 'application/vnd.yang.data+json' }) if os.path.exists(path_to_delete_local): os.remove(path_to_delete_local) else: with open(modules_to_index, 'r') as f: sdos_json = json.load(f) post_body = {} load_new_files_to_github = False if from_api: if sdo_type: prefix = 'sdo/' else: prefix = 'vendor/' for module in sdos_json['module']: url = '{}search/modules/{},{},{}'.format( yc_api_prefix, module['name'], module['revision'], module['organization']) response = requests.get(url, auth=(credentials[0], credentials[1]), headers={ 'Content-Type': 'application/vnd.yang.data+json', 'Accept': 'application/vnd.yang.data+json' }) code = response.status_code if force_indexing or (code != 200 and code != 201 and code != 204): if module.get('schema'): path = prefix + module['schema'].split( 'githubusercontent.com/')[1] path = os.path.abspath(temp_dir + '/' + path) else: path = 'module does not exist' post_body[module['name'] + '@' + module['revision'] + '/' + module['organization']] = path else: for module in sdos_json['module']: url = '{}search/modules/{},{},{}'.format( yc_api_prefix, module['name'], module['revision'], module['organization']) response = requests.get(url, auth=(credentials[0], credentials[1]), headers={ 'Content-Type': 'application/vnd.yang.data+json', 'Accept': 'application/vnd.yang.data+json' }) code = response.status_code if code != 200 and code != 201 and code != 204: load_new_files_to_github = True if force_indexing or (code != 200 and code != 201 and code != 204): path = '{}/{}@{}.yang'.format(save_file_dir, module.get('name'), module.get('revision')) post_body[module['name'] + '@' + module['revision'] + '/' + module['organization']] = path if len(post_body) == 0: body_to_send = '' else: body_to_send = json.dumps({'modules-to-index': post_body}, indent=4) if len(post_body) > 0 and not force_indexing: mf.send_added_new_yang_files(body_to_send) if load_new_files_to_github: LOGGER.info('Starting a new process to populate github') cmd = ['python', '../ietfYangDraftPull/draftPull.py'] proc = subprocess.Popen(cmd, close_fds=True) LOGGER.info('Populating github with process {}'.format(proc)) return body_to_send
def on_request_thread_safe(self, properties, body_raw: bytes): """Function called when something was sent from API sender. This function will process all the requests that would take too long to process for API. When the processing is done we will sent back the result of the request which can be either 'Failed' or 'Finished successfully' with corespondent correlation ID. If the request 'Failed' it will sent back also a reason why it failed. Arguments: :param body: (str) String of arguments that need to be processed separated by '#'. """ config_reloaded = False status: StatusMessage details: str = '' try: body = body_raw.decode() arguments = body.split('#') if body == 'run_ietf': self.LOGGER.info('Running all ietf and openconfig modules') status, details = self.run_ietf() elif body == 'reload_config': status = self.load_config() config_reloaded = True elif 'run_ping' == arguments[0]: status = self.run_ping(arguments[1]) elif 'run_script' == arguments[0]: status = self.run_script(arguments[1:]) elif 'github' == arguments[-1]: self.LOGGER.info( 'Github automated message starting to populate') paths_plus = arguments[arguments.index('repoLocalDir'):] self.LOGGER.info('paths plus {}'.format(paths_plus)) arguments = arguments[:arguments.index('repoLocalDir')] self.LOGGER.info('arguments {}'.format(arguments)) paths = paths_plus[1:-2] self.LOGGER.info('paths {}'.format(paths)) try: for path in paths: with open(self.temp_dir + '/log_trigger.txt', 'w') as f: local_dir = paths_plus[-2] arguments = arguments + [ '--dir', local_dir + '/' + path ] if self._notify_indexing: arguments.append('--notify-indexing') subprocess.check_call(arguments, stderr=f) status = StatusMessage.SUCCESS except subprocess.CalledProcessError as e: status = StatusMessage.FAIL mf = messageFactory.MessageFactory() mf.send_automated_procedure_failed( arguments, self.temp_dir + '/log_no_sdo_api.txt') self.LOGGER.error( 'check log_trigger.txt Error calling process populate.py because {}\n\n with error {}' .format(e.output, e.stderr)) except Exception: status = StatusMessage.FAIL self.LOGGER.error( 'check log_trigger.txt failed to process github message with error {}' .format(sys.exc_info()[0])) else: direc = '' if arguments[0] == 'DELETE-VENDORS': status = self.process_vendor_deletion(arguments) credentials = arguments[1:3] elif arguments[0] == 'DELETE-MODULES': status, details = self.process_module_deletion(arguments) credentials = arguments[1:3] elif arguments[0] == 'POPULATE-MODULES': status, details = self.process(arguments) i = arguments.index('--credentials') credentials = arguments[i + 1:i + 3] i = arguments.index('--dir') direc = arguments[i + 1] shutil.rmtree(direc) elif arguments[0] == 'POPULATE-VENDORS': status, details = self.process(arguments) i = arguments.index('--credentials') credentials = arguments[i + 1:i + 3] i = arguments.index('--dir') direc = arguments[i + 1] shutil.rmtree(direc) else: assert False, 'Invalid request type' if status == StatusMessage.SUCCESS: response = self.make_cache(credentials) code = response.status_code if code != 200 and code != 201 and code != 204: status = StatusMessage.FAIL details = 'Server error-> could not reload cache' except Exception: status = StatusMessage.FAIL self.LOGGER.exception('receiver.py failed') final_response = status.value if not details else '{}#split#{}'.format( status.value, details) self.LOGGER.info( 'Receiver is done with id - {} and message = {}'.format( properties.correlation_id, final_response)) f = open('{}/correlation_ids'.format(self.temp_dir), 'r') lines = f.readlines() f.close() with open('{}/correlation_ids'.format(self.temp_dir), 'w') as f: for line in lines: if properties.correlation_id in line: new_line = '{} -- {} - {}\n'.format( datetime.now().ctime(), properties.correlation_id, str(final_response)) f.write(new_line) else: f.write(line) if config_reloaded: assert self.channel, 'Should only be called from self.channel.start_consuming()' self.channel.stop_consuming()
if not same: diff_files.append(file_name) else: new_files.append(file_name) shutil.rmtree(repo.localdir + '/standard/ietf/RFCtemp') os.remove(repo.localdir + '/rfc.tgz') with open(exceptions, 'r') as exceptions_file: remove_from_new = exceptions_file.read().split('\n') for remove in remove_from_new: if remove in new_files: new_files.remove(remove) if len(new_files) > 0 or len(diff_files) > 0: LOGGER.warning('new or modified RFC files found. Sending an E-mail') mf = messageFactory.MessageFactory() mf.send_new_rfc_message(new_files, diff_files) for key in ietf_draft_json: yang_file = open( repo.localdir + '/experimental/ietf-extracted-YANG-modules/' + key, 'w+') yang_download_link = ietf_draft_json[key][2].split('href="')[1].split( '">Download')[0] yang_download_link = yang_download_link.replace( 'new.yangcatalog.org', 'yangcatalog.org') try: yang_raw = requests.get(yang_download_link).text yang_file.write(yang_raw) except: LOGGER.warning('{} - {}'.format(key, yang_download_link))
def __init__(self): self.__mf = messageFactory.MessageFactory() self.month = datetime.now().date().month self.day = datetime.now().date().day self.users = RedisUsersConnection()