def _base_file_content(old_file_content, result):
# remove entries we don't want to keep
result.pop('access_count', None)
# do not keep track of tasks that were used for Selinon's Dispatcher book-keeping
analyses_list = (set(old_file_content.get('analyses', [])) |
set(result.get('analyses', {}).keys()))
analyses_list = [a for a in analyses_list if not a[0].isupper()]
content = result
content['analyses'] = analyses_list
if 'finished_at' in content:
content['finished_at'] = json_serial(content['finished_at'])
if 'started_at' in content:
content['started_at'] = json_serial(content['started_at'])
return content
def run(self, node_args):
"""To be transparently called by Selinon.
Selinon transparently calls run(), which takes care of task audit and
some additional checks and calls execute().
"""
# SQS guarantees 'deliver at least once', so there could be multiple
# messages of a type, give up immediately
if self.storage and isinstance(self.storage,
(BayesianPostgres, PackagePostgres)):
if self.storage.get_worker_id_count(self.task_id) > 0:
raise TaskAlreadyExistsError("Task with ID '%s'"
" was already processed" %
self.task_id)
start = datetime.utcnow()
try:
result = self.execute(node_args)
finally:
# remove all files that were downloaded for this task
ObjectCache.wipe()
end = datetime.utcnow()
if result:
# Ensure result complies with the defined schema (if any) before saving
self.validate_result(result)
if result is None:
# Keep track of None results and add _audit and _release keys
result = {}
if self.add_audit_info:
# `_audit` key is added to every analysis info submitted
result['_audit'] = {
'started_at': json_serial(start),
'ended_at': json_serial(end),
'version': 'v1'
}
ecosystem_name = node_args.get('ecosystem')
result['_release'] = '{}:{}:{}'.format(ecosystem_name,
node_args.get('name'),
node_args.get('version'))
return result
def default(self, obj):
try:
if isinstance(obj, datetime.datetime):
return json_serial(obj)
iterable = iter(obj)
except TypeError:
pass
else:
return list(iterable)
return JSONEncoder.default(self, obj)
def _add_audit_info(task_result: dict, task_start: datetime,
task_end: datetime, node_args):
"""Add the audit and release information to the result dictionary.
:param task_result: dict, task result
:param task_start: datetime, the start of the task
:param task_end: datetime, the end of the task
:param node_args: arguments passed to flow/node
"""
task_result['_audit'] = {
'started_at': json_serial(task_start),
'ended_at': json_serial(task_end),
'version': 'v1'
}
ecosystem_name = node_args.get('ecosystem')
task_result['_release'] = '{}:{}:{}'.format(ecosystem_name,
node_args.get('name'),
node_args.get('version'))
def run(self, arguments):
self._strict_assert(arguments.get('document_id'))
try:
record = self.storage.session.query(PackageAnalysis).\
filter(PackageAnalysis.id == arguments['document_id']).one()
record.finished_at = json_serial(datetime.datetime.utcnow())
self.storage.session.commit()
except SQLAlchemyError:
self.storage.session.rollback()
raise
def run(self, arguments):
self._strict_assert(arguments.get('document_id'))
try:
record = self.storage.session.query(Analysis).\
filter(Analysis.id == arguments['document_id']).one()
record.finished_at = json_serial(datetime.datetime.utcnow())
record.release = '{}:{}:{}'.format(arguments.get('ecosystem'),
arguments.get('name'),
arguments.get('version'))
self.storage.session.commit()
except SQLAlchemyError:
self.storage.session.rollback()
raise
def _resolve_dependency(ecosystem, dep):
ret = {
'ecosystem': ecosystem.name,
'declaration': dep,
'resolved_at': json_serial(datetime.datetime.utcnow())
}
# first, if this is a Github dependency, return it right away (we don't resolve these yet)
if ' ' in dep:
# we have both package name and version (version can be an URL)
name, spec = dep.split(' ', 1)
if gh_dep.match(spec):
ret['name'] = name
ret['version'] = 'https://github.com/' + spec
elif urllib.parse.urlparse(spec).scheme is not '':
ret['name'] = name
ret['version'] = spec
else:
if gh_dep.match(dep):
ret['name'] = 'https://github.com/' + dep
ret['version'] = None
elif urllib.parse.urlparse(dep).scheme is not '':
ret['name'] = dep
ret['version'] = None
if 'name' in ret:
return ret
# second, figure out what is the latest upstream version matching the spec and return it
solver = get_ecosystem_solver(ecosystem)
pkgspec = solver.solve([dep])
if not pkgspec:
raise TaskError("invalid dependency: {}".format(dep))
package, version = pkgspec.popitem()
if not version:
raise TaskError("could not resolve {}".format(dep))
ret['name'] = package
ret['version'] = version
return ret
