def refresh(ecosystem, package, url): """Schedule refresh of GitHub data for given package.""" logger.info('Refreshing GitHub data for {e}/{p}: {u}'.format( e=ecosystem, p=package, u=url) ) node_args = { 'ecosystem': ecosystem, 'name': package, 'force': True } if not is_dry_run(): run_flow_selective('bayesianPackageFlow', TASK_NAMES, node_args, True, False)
def run_selinon_flow_selective(self, flow_name, task_names, node_args, follow_subflows, run_subsequent): """Connect to broker, if not connected, and run Selinon selective flow. :param flow_name: flow that should be run :param task_names: a list of tasks that should be executed :param node_args: flow arguments :param follow_subflows: follow subflows when resolving tasks to be executed :param run_subsequent: run tasks that follow after desired tasks stated in task_names """ if flow_name in ('bayesianFlow', 'bayesianAnalysisFlow'): task_names = list( set(task_names) | {'FinalizeTask', 'ResultCollector', 'GraphImporterTask'}) if flow_name in ('bayesianPackageFlow', 'bayesianPackageAnalysisFlow'): task_names = list( set(task_names) | { 'PackageFinalizeTask', 'PackageResultCollector', 'PackageGraphImporterTask' }) self._normalize_package_name(node_args) self.log.debug( "Scheduling selective Selinon flow '%s' with tasks '%s' and node_args: " "'%s', job '%s'", flow_name, task_names, node_args, self.job_id) return run_flow_selective(flow_name, task_names, node_args, follow_subflows, run_subsequent)
def run_flow_selective(self, flow_name, task_names, node_args=None, follow_subflows=False, run_subsequent=False): """Run only desired tasks in a flow. :param flow_name: name of the flow that should be run :param task_names: name of the tasks that should be run :param node_args: arguments that should be supplied to flow :param follow_subflows: if True, subflows will be followed and checked for nodes to be run :param run_subsequent: trigger run of all tasks that depend on the desired task :return: dispatcher id that is scheduled to run desired selective task flow :raises selinon.errors.SelectiveNoPathError: there was no way found to the desired task in the flow """ self._prepare() run_flow_selective(flow_name, task_names, node_args, follow_subflows, run_subsequent) self._executor_run()
def run_flow_selective(self, flow_name, task_names, node_args=None, follow_subflows=False, run_subsequent=False): """Run only desired tasks in a flow. :param flow_name: name of the flow that should be run :param task_names: name of the tasks that should be run :param node_args: arguments that should be supplied to flow :param follow_subflows: if True, subflows will be followed and checked for nodes to be run :param run_subsequent: trigger run of all tasks that depend on the desired task :return: dispatcher id that is scheduled to run desired selective task flow :raises selinon.errors.SelectiveNoPathError: there was no way found to the desired task in the flow """ self._prepare() run_flow_selective( flow_name, task_names, node_args, follow_subflows, run_subsequent ) self._executor_run()
def run_selinon_flow_selective(self, flow_name, task_names, node_args, follow_subflows, run_subsequent): """Connect to broker, if not connected, and run Selinon selective flow :param flow_name: flow that should be run :param task_names: a list of tasks that should be executed :param node_args: flow arguments :param follow_subflows: follow subflows when resolving tasks to be executed :param run_subsequent: run tasks that follow after desired tasks stated in task_names """ self.log.debug( "Scheduling selective Selinon flow '%s' with node_args: '%s'", flow_name, node_args) return run_flow_selective(flow_name, task_names, node_args, follow_subflows, run_subsequent)
def ingest_selective_epv_into_graph(epv_details): """Handle implementation of API for triggering selective ingestion for any flow. :param epv_details: A dictionary having list of packages/version/tasks as a nested object. Ex: { "ecosystem": "<ecosystem_name>", (*required) "packages": [ { "package": "<package_name_1>", (*required) "version": "<package_version_1>" (*required) }, { "package": "<package_name_2>", (*required) "version": "<package_version_2>" (optional) } ], "task_names": [ (*required) "TASK_1", "TASK_2", "TASK_3", "TASK_4" ], "force": false, (optional) "recursive_limit": 0 (optional) "follow_subflows": true, (optional) "run_subsequent": false, (optional) "source": "<Consumer_of_API>"(optional) } """ logger.info('graph_ingestion_:_ingest_selective_epv_into_graph is called.') input_data = epv_details.get('body', {}) # Check if worker flow activation is disabled. if not _INVOKE_API_WORKERS: logger.debug('Worker flows are disabled.') input_data['message'] = 'Worker flows are disabled.' return input_data, 201 ecosystem = input_data.get('ecosystem') package_list = input_data.get('packages') task_names = input_data.get('task_names') follow_subflows = input_data.get('follow_subflows', False) run_subsequent = input_data.get('run_subsequent', False) if input_data.get('source', '') == 'git-refresh': flow_name = 'newPackageAnalysisFlow' \ if ecosystem == 'golang' else 'bayesianPackageFlow' if 'flow_name' in input_data: flow_name = input_data['flow_name'] node_arguments = { "ecosystem": ecosystem, "force": input_data.get('force', True) } # Iterate through packages given for current ecosystem. for item in package_list: node_arguments["name"] = item.get('package') if 'url' in item: node_arguments['url'] = item['url'] if 'version' in item: node_arguments['version'] = item['version'] try: # Initiate Selinon flow for current EPV ingestion. dispacher_id = run_flow_selective(flow_name, task_names, node_arguments, follow_subflows, run_subsequent) item['dispacher_id'] = dispacher_id.id except Exception as e: logger.error('Exception while initiating the worker flow %s', e) return {'message': 'Failed to initiate worker flow.'}, 500 logger.info('A selective flow "%s" in initiated for ' 'eco: %s, pkg: %s, for task list: %s', flow_name, ecosystem, item['package'], task_names) return input_data, 201
def ingest_selective_epv_into_graph(epv_details): """Handle implementation of API for triggering selective ingestion for any flow. :param epv_details: A dictionary having list of packages/version/tasks as a nested object. Ex: { "ecosystem": "<ecosystem_name>", (*required) "packages": [ { "package": "<package_name_1>", (*required) "version": "<package_version_1>" (*required) }, { "package": "<package_name_2>", (*required) "version": "<package_version_2>" (optional) } ], "task_names": [ (*required) "TASK_1", "TASK_2", "TASK_3", "TASK_4" ], "force": false, (optional) "recursive_limit": 0 (optional) "follow_subflows": true, (optional) "run_subsequent": false, (optional) } """ try: logger.info( 'graph_ingestion_:_ingest_selective_epv_into_graph is called.') input_data = epv_details.get('body', {}) # Check if EPV ingestion is enabled. if _INVOKE_API_WORKERS: ecosystem = input_data.get('ecosystem') package_list = input_data.get('packages') flow_name = input_data.get('flow_name') task_names = input_data.get('task_names') force = input_data.get('force', True) follow_subflows = input_data.get('follow_subflows', False) run_subsequent = input_data.get('run_subsequent', False) input_data.pop('follow_subflows', None) input_data.pop('run_subsequent', None) input_data.pop('task_names') input_data.pop('force', None) # Iterate through packages given for current ecosystem. for item in package_list: node_arguments = { "ecosystem": ecosystem, "force": force, "name": item.get('package'), } if 'url' in item: node_arguments['url'] = item['url'] if 'version' in item: node_arguments['version'] = item['version'] # Initiate Selinon flow for current EPV ingestion. dispacher_id = run_flow_selective(flow_name, task_names, node_arguments, follow_subflows, run_subsequent) item['dispacher_id'] = dispacher_id.id logger.info('A selective flow "{}" in initiated for ' 'eco: {}, pkg: {}, for task list: {}'.format( flow_name, ecosystem, item['package'], task_names)) return input_data, 201 except Exception as e: logger.error('Exception while initiating the worker flow {}'.format(e)) return {'message': 'Failed to initiate worker flow.'}, 500