def refresh(ecosystem, package, url):
"""Schedule refresh of GitHub data for given package."""
logger.info('Refreshing GitHub data for {e}/{p}: {u}'.format(
e=ecosystem, p=package, u=url)
)
node_args = {
'ecosystem': ecosystem,
'name': package,
'force': True
}
if not is_dry_run():
run_flow_selective('bayesianPackageFlow', TASK_NAMES, node_args, True, False)
def run_selinon_flow_selective(self, flow_name, task_names, node_args,
follow_subflows, run_subsequent):
"""Connect to broker, if not connected, and run Selinon selective flow.
:param flow_name: flow that should be run
:param task_names: a list of tasks that should be executed
:param node_args: flow arguments
:param follow_subflows: follow subflows when resolving tasks to be executed
:param run_subsequent: run tasks that follow after desired tasks stated in task_names
"""
if flow_name in ('bayesianFlow', 'bayesianAnalysisFlow'):
task_names = list(
set(task_names)
| {'FinalizeTask', 'ResultCollector', 'GraphImporterTask'})
if flow_name in ('bayesianPackageFlow', 'bayesianPackageAnalysisFlow'):
task_names = list(
set(task_names) | {
'PackageFinalizeTask', 'PackageResultCollector',
'PackageGraphImporterTask'
})
self._normalize_package_name(node_args)
self.log.debug(
"Scheduling selective Selinon flow '%s' with tasks '%s' and node_args: "
"'%s', job '%s'", flow_name, task_names, node_args, self.job_id)
return run_flow_selective(flow_name, task_names, node_args,
follow_subflows, run_subsequent)
def run_flow_selective(self,
flow_name,
task_names,
node_args=None,
follow_subflows=False,
run_subsequent=False):
"""Run only desired tasks in a flow.
:param flow_name: name of the flow that should be run
:param task_names: name of the tasks that should be run
:param node_args: arguments that should be supplied to flow
:param follow_subflows: if True, subflows will be followed and checked for nodes to be run
:param run_subsequent: trigger run of all tasks that depend on the desired task
:return: dispatcher id that is scheduled to run desired selective task flow
:raises selinon.errors.SelectiveNoPathError: there was no way found to the desired task in the flow
"""
self._prepare()
run_flow_selective(flow_name, task_names, node_args, follow_subflows,
run_subsequent)
self._executor_run()
def run_flow_selective(self, flow_name, task_names, node_args=None, follow_subflows=False, run_subsequent=False):
"""Run only desired tasks in a flow.
:param flow_name: name of the flow that should be run
:param task_names: name of the tasks that should be run
:param node_args: arguments that should be supplied to flow
:param follow_subflows: if True, subflows will be followed and checked for nodes to be run
:param run_subsequent: trigger run of all tasks that depend on the desired task
:return: dispatcher id that is scheduled to run desired selective task flow
:raises selinon.errors.SelectiveNoPathError: there was no way found to the desired task in the flow
"""
self._prepare()
run_flow_selective(
flow_name,
task_names,
node_args,
follow_subflows,
run_subsequent
)
self._executor_run()
def run_selinon_flow_selective(self, flow_name, task_names, node_args,
follow_subflows, run_subsequent):
"""Connect to broker, if not connected, and run Selinon selective flow
:param flow_name: flow that should be run
:param task_names: a list of tasks that should be executed
:param node_args: flow arguments
:param follow_subflows: follow subflows when resolving tasks to be executed
:param run_subsequent: run tasks that follow after desired tasks stated in task_names
"""
self.log.debug(
"Scheduling selective Selinon flow '%s' with node_args: '%s'",
flow_name, node_args)
return run_flow_selective(flow_name, task_names, node_args,
follow_subflows, run_subsequent)
def ingest_selective_epv_into_graph(epv_details):
"""Handle implementation of API for triggering selective ingestion for any flow.
:param epv_details: A dictionary having list of packages/version/tasks as a nested object.
Ex:
{
"ecosystem": "<ecosystem_name>", (*required)
"packages": [
{
"package": "<package_name_1>", (*required)
"version": "<package_version_1>" (*required)
}, {
"package": "<package_name_2>", (*required)
"version": "<package_version_2>" (optional)
}
],
"task_names": [ (*required)
"TASK_1",
"TASK_2",
"TASK_3",
"TASK_4"
],
"force": false, (optional)
"recursive_limit": 0 (optional)
"follow_subflows": true, (optional)
"run_subsequent": false, (optional)
"source": "<Consumer_of_API>"(optional)
}
"""
logger.info('graph_ingestion_:_ingest_selective_epv_into_graph is called.')
input_data = epv_details.get('body', {})
# Check if worker flow activation is disabled.
if not _INVOKE_API_WORKERS:
logger.debug('Worker flows are disabled.')
input_data['message'] = 'Worker flows are disabled.'
return input_data, 201
ecosystem = input_data.get('ecosystem')
package_list = input_data.get('packages')
task_names = input_data.get('task_names')
follow_subflows = input_data.get('follow_subflows', False)
run_subsequent = input_data.get('run_subsequent', False)
if input_data.get('source', '') == 'git-refresh':
flow_name = 'newPackageAnalysisFlow' \
if ecosystem == 'golang' else 'bayesianPackageFlow'
if 'flow_name' in input_data:
flow_name = input_data['flow_name']
node_arguments = {
"ecosystem": ecosystem,
"force": input_data.get('force', True)
}
# Iterate through packages given for current ecosystem.
for item in package_list:
node_arguments["name"] = item.get('package')
if 'url' in item:
node_arguments['url'] = item['url']
if 'version' in item:
node_arguments['version'] = item['version']
try:
# Initiate Selinon flow for current EPV ingestion.
dispacher_id = run_flow_selective(flow_name,
task_names,
node_arguments,
follow_subflows,
run_subsequent)
item['dispacher_id'] = dispacher_id.id
except Exception as e:
logger.error('Exception while initiating the worker flow %s', e)
return {'message': 'Failed to initiate worker flow.'}, 500
logger.info('A selective flow "%s" in initiated for '
'eco: %s, pkg: %s, for task list: %s',
flow_name, ecosystem, item['package'], task_names)
return input_data, 201
def ingest_selective_epv_into_graph(epv_details):
"""Handle implementation of API for triggering selective ingestion for any flow.
:param epv_details: A dictionary having list of packages/version/tasks as a nested object.
Ex:
{
"ecosystem": "<ecosystem_name>", (*required)
"packages": [
{
"package": "<package_name_1>", (*required)
"version": "<package_version_1>" (*required)
}, {
"package": "<package_name_2>", (*required)
"version": "<package_version_2>" (optional)
}
],
"task_names": [ (*required)
"TASK_1",
"TASK_2",
"TASK_3",
"TASK_4"
],
"force": false, (optional)
"recursive_limit": 0 (optional)
"follow_subflows": true, (optional)
"run_subsequent": false, (optional)
}
"""
try:
logger.info(
'graph_ingestion_:_ingest_selective_epv_into_graph is called.')
input_data = epv_details.get('body', {})
# Check if EPV ingestion is enabled.
if _INVOKE_API_WORKERS:
ecosystem = input_data.get('ecosystem')
package_list = input_data.get('packages')
flow_name = input_data.get('flow_name')
task_names = input_data.get('task_names')
force = input_data.get('force', True)
follow_subflows = input_data.get('follow_subflows', False)
run_subsequent = input_data.get('run_subsequent', False)
input_data.pop('follow_subflows', None)
input_data.pop('run_subsequent', None)
input_data.pop('task_names')
input_data.pop('force', None)
# Iterate through packages given for current ecosystem.
for item in package_list:
node_arguments = {
"ecosystem": ecosystem,
"force": force,
"name": item.get('package'),
}
if 'url' in item:
node_arguments['url'] = item['url']
if 'version' in item:
node_arguments['version'] = item['version']
# Initiate Selinon flow for current EPV ingestion.
dispacher_id = run_flow_selective(flow_name, task_names,
node_arguments,
follow_subflows,
run_subsequent)
item['dispacher_id'] = dispacher_id.id
logger.info('A selective flow "{}" in initiated for '
'eco: {}, pkg: {}, for task list: {}'.format(
flow_name, ecosystem, item['package'],
task_names))
return input_data, 201
except Exception as e:
logger.error('Exception while initiating the worker flow {}'.format(e))
return {'message': 'Failed to initiate worker flow.'}, 500
