def initiate_unknown_package_ingestion(self):
"""Ingestion of Unknown dependencies."""
if Settings().disable_unknown_package_flow:
logger.warning('Skipping unknown flow %s',
self.get_all_unknown_packages())
return
ecosystem = self._normalized_packages.ecosystem
try:
for dep in self.get_all_unknown_packages():
server_create_analysis(ecosystem,
dep.name,
dep.version,
api_flow=True,
force=False,
force_graph_sync=True)
except Exception as e: # pylint:disable=W0703,C0103
logger.error('Ingestion failed for {%s, %s, %s}', ecosystem,
dep.name, dep.version)
logger.error(e)
def test_server_create_analysis():
"""Test server_create_analysis."""
rec_resp = server_create_analysis("npm", "hjhjhjh", "1.1")
assert rec_resp is None
def execute(aggregated=None, persist=True):
"""Task code."""
started_at = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%f")
stack_data = []
unknown_dep_list = []
show_transitive = aggregated.get('show_transitive')
external_request_id = aggregated.get('external_request_id')
# TODO multiple license file support
current_stack_license = aggregated.get('current_stack_license', {}).get('1', {})
for result in aggregated['result']:
resolved = result['details'][0]['_resolved']
ecosystem = result['details'][0]['ecosystem']
manifest = result['details'][0]['manifest_file']
manifest_file_path = result['details'][0]['manifest_file_path']
epv_set = create_dependency_data_set(resolved, ecosystem)
finished = get_dependency_data(epv_set)
""" Direct deps can have 0 transitives. This condition is added
so that in ext, we get to know if deps are 0 or if the transitive flag
is false """
if show_transitive == "true":
transitive_count = finished.get('transitive_count', 0)
else:
transitive_count = -1
if finished is not None:
output = aggregate_stack_data(finished, manifest, ecosystem.lower(), resolved,
manifest_file_path, persist, transitive_count)
if output and output.get('user_stack_info'):
output['user_stack_info']['license_analysis'].update({
"current_stack_license": current_stack_license
})
stack_data.append(output)
unknown_dep_list.extend(finished['unknown_deps'])
ended_at = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%f")
audit = {
'started_at': started_at,
'ended_at': ended_at,
'version': 'v1'
}
stack_data = {
'stack_data': stack_data,
'_audit': audit,
'_release': 'None:None:None'
}
if persist:
logger.info("Aggregation process completed for {}."
" Writing to RDS.".format(external_request_id))
persist_data_in_db(external_request_id=external_request_id,
task_result=stack_data, worker='stack_aggregator_v2',
started_at=started_at, ended_at=ended_at)
persiststatus = {'stack_aggregator': 'success',
'external_request_id': external_request_id,
'result': stack_data}
# Ingestion of Unknown dependencies
logger.info("Unknown ingestion flow process initiated.")
try:
for dep in unknown_dep_list:
server_create_analysis(ecosystem, dep['name'], dep['version'], api_flow=True,
force=False, force_graph_sync=True)
except Exception as e:
logger.error('Ingestion has been failed for ' + dep['name'])
logger.error(e)
pass
return persiststatus