def _base_file_content(old_file_content, result):
# remove entries we don't want to keep
result.pop('access_count', None)
# do not keep track of tasks that were used for Selinon's Dispatcher book-keeping
analyses_list = set(old_file_content.get('analyses', [])) | set(result.get('analyses', {}).keys())
analyses_list = [a for a in analyses_list if not a[0].isupper()]
content = result
content['analyses'] = analyses_list
if 'finished_at' in content:
content['finished_at'] = json_serial(content['finished_at'])
if 'started_at' in content:
content['started_at'] = json_serial(content['started_at'])
return content
def _resolve_dependency(self, ecosystem, dep):
ret = {
'ecosystem': ecosystem.name,
'declaration': dep,
'resolved_at': json_serial(datetime.datetime.now())
}
# first, if this is a Github dependency, return it right away (we don't resolve these yet)
if ' ' in dep:
name, spec = dep.split(' ', 1)
if gh_dep.match(spec):
ret['name'] = name
ret['version'] = 'https://github.com/' + spec
else:
if gh_dep.match(dep):
ret['name'] = 'https://github.com/' + dep
ret['version'] = None
if 'name' in ret:
return ret
# second, figure out what is the latest upstream version matching the spec and return it
solver = get_ecosystem_solver(ecosystem)
pkgspec = solver.solve([dep])
if not pkgspec:
raise TaskError("invalid dependency: {}".format(dep))
package, version = pkgspec.popitem()
if not version:
raise TaskError("bad version resolved for {}".format(dep))
ret['name'] = package
ret['version'] = version
return ret
def default(self, obj):
try:
if isinstance(obj, datetime.datetime):
return json_serial(obj)
iterable = iter(obj)
except TypeError:
pass
else:
return list(iterable)
return JSONEncoder.default(self, obj)
def run(self, arguments):
self._strict_assert(arguments.get('document_id'))
try:
record = self.storage.session.query(PackageAnalysis).\
filter(PackageAnalysis.id == arguments['document_id']).one()
record.finished_at = json_serial(datetime.datetime.now())
self.storage.session.commit()
except:
self.storage.session.rollback()
raise
def run(self, arguments):
self._strict_assert(arguments.get('document_id'))
try:
record = self.storage.session.query(Analysis).filter(Analysis.id == arguments['document_id']).one()
record.finished_at = json_serial(datetime.datetime.now())
record.release = '{}:{}:{}'.format(arguments.get('ecosystem'),
arguments.get('name'),
arguments.get('version'))
self.storage.session.commit()
except:
self.storage.session.rollback()
raise
def run(self, node_args):
if self.storage and isinstance(self.storage, BayesianPostgres):
# SQS guarantees 'deliver at least once', so there could be multiple messages of a type, give up immediately
if self.storage.get_worker_id_count(self.task_id) > 0:
raise FatalTaskError(
"Task with ID '%s' was already processed" % self.task_id)
start = datetime.now()
try:
result = self.execute(node_args)
finally:
# remove all files that were downloaded for this task
ObjectCache.wipe()
end = datetime.now()
if result:
# Ensure result complies with the defined schema (if any) before saving
self.validate_result(result)
if result is None:
# Keep track of None results and add _audit and _release keys
result = {}
if self.add_audit_info:
# `_audit` key is added to every analysis info submitted
result['_audit'] = {
'started_at': json_serial(start),
'ended_at': json_serial(end),
'version': 'v1'
}
ecosystem_name = node_args.get('ecosystem')
result['_release'] = '{}:{}:{}'.format(ecosystem_name,
node_args.get('name'),
node_args.get('version'))
return result