def _base_file_content(old_file_content, result): # remove entries we don't want to keep result.pop('access_count', None) # do not keep track of tasks that were used for Selinon's Dispatcher book-keeping analyses_list = (set(old_file_content.get('analyses', [])) | set(result.get('analyses', {}).keys())) analyses_list = [a for a in analyses_list if not a[0].isupper()] content = result content['analyses'] = analyses_list if 'finished_at' in content: content['finished_at'] = json_serial(content['finished_at']) if 'started_at' in content: content['started_at'] = json_serial(content['started_at']) return content
def run(self, node_args): """To be transparently called by Selinon. Selinon transparently calls run(), which takes care of task audit and some additional checks and calls execute(). """ # SQS guarantees 'deliver at least once', so there could be multiple # messages of a type, give up immediately if self.storage and isinstance(self.storage, (BayesianPostgres, PackagePostgres)): if self.storage.get_worker_id_count(self.task_id) > 0: raise TaskAlreadyExistsError("Task with ID '%s'" " was already processed" % self.task_id) start = datetime.utcnow() try: result = self.execute(node_args) finally: # remove all files that were downloaded for this task ObjectCache.wipe() end = datetime.utcnow() if result: # Ensure result complies with the defined schema (if any) before saving self.validate_result(result) if result is None: # Keep track of None results and add _audit and _release keys result = {} if self.add_audit_info: # `_audit` key is added to every analysis info submitted result['_audit'] = { 'started_at': json_serial(start), 'ended_at': json_serial(end), 'version': 'v1' } ecosystem_name = node_args.get('ecosystem') result['_release'] = '{}:{}:{}'.format(ecosystem_name, node_args.get('name'), node_args.get('version')) return result
def default(self, obj): try: if isinstance(obj, datetime.datetime): return json_serial(obj) iterable = iter(obj) except TypeError: pass else: return list(iterable) return JSONEncoder.default(self, obj)
def _add_audit_info(task_result: dict, task_start: datetime, task_end: datetime, node_args): """Add the audit and release information to the result dictionary. :param task_result: dict, task result :param task_start: datetime, the start of the task :param task_end: datetime, the end of the task :param node_args: arguments passed to flow/node """ task_result['_audit'] = { 'started_at': json_serial(task_start), 'ended_at': json_serial(task_end), 'version': 'v1' } ecosystem_name = node_args.get('ecosystem') task_result['_release'] = '{}:{}:{}'.format(ecosystem_name, node_args.get('name'), node_args.get('version'))
def run(self, arguments): self._strict_assert(arguments.get('document_id')) try: record = self.storage.session.query(PackageAnalysis).\ filter(PackageAnalysis.id == arguments['document_id']).one() record.finished_at = json_serial(datetime.datetime.utcnow()) self.storage.session.commit() except SQLAlchemyError: self.storage.session.rollback() raise
def run(self, arguments): self._strict_assert(arguments.get('document_id')) try: record = self.storage.session.query(Analysis).\ filter(Analysis.id == arguments['document_id']).one() record.finished_at = json_serial(datetime.datetime.utcnow()) record.release = '{}:{}:{}'.format(arguments.get('ecosystem'), arguments.get('name'), arguments.get('version')) self.storage.session.commit() except SQLAlchemyError: self.storage.session.rollback() raise
def _resolve_dependency(ecosystem, dep): ret = { 'ecosystem': ecosystem.name, 'declaration': dep, 'resolved_at': json_serial(datetime.datetime.utcnow()) } # first, if this is a Github dependency, return it right away (we don't resolve these yet) if ' ' in dep: # we have both package name and version (version can be an URL) name, spec = dep.split(' ', 1) if gh_dep.match(spec): ret['name'] = name ret['version'] = 'https://github.com/' + spec elif urllib.parse.urlparse(spec).scheme is not '': ret['name'] = name ret['version'] = spec else: if gh_dep.match(dep): ret['name'] = 'https://github.com/' + dep ret['version'] = None elif urllib.parse.urlparse(dep).scheme is not '': ret['name'] = dep ret['version'] = None if 'name' in ret: return ret # second, figure out what is the latest upstream version matching the spec and return it solver = get_ecosystem_solver(ecosystem) pkgspec = solver.solve([dep]) if not pkgspec: raise TaskError("invalid dependency: {}".format(dep)) package, version = pkgspec.popitem() if not version: raise TaskError("could not resolve {}".format(dep)) ret['name'] = package ret['version'] = version return ret