def prepare(cls, username, **kwargs): """ Take an arbitrary set of keyword arguments and return an instance of a BackgroundJob, or fail with a suitable exception :param kwargs: arbitrary keyword arguments pertaining to this task type :return: a BackgroundJob instance representing this task """ super(ArticleBulkDeleteBackgroundTask, cls).prepare(username, **kwargs) # first prepare a job record job = models.BackgroundJob() job.user = username job.action = cls.__action__ refs = {} cls.set_reference(refs, "selection_query", json.dumps(kwargs['selection_query'])) job.reference = refs params = {} cls.set_param(params, 'ids', kwargs['ids']) if not cls._job_parameter_check(params): raise BackgroundException( u"{}.prepare run without sufficient parameters".format( cls.__name__)) job.params = params return job
def get_task(cls, user=None, run_fail=False, cleanup_fail=False): job = models.BackgroundJob() if user is not None: job.user = user return MockBackgroundTask(job, run_fail=run_fail, cleanup_fail=cleanup_fail)
def prepare(cls, username, **kwargs): """ Take an arbitrary set of keyword arguments and return an instance of a BackgroundJob, or fail with a suitable exception :param kwargs: arbitrary keyword arguments pertaining to this task type :return: a BackgroundJob instance representing this task """ super(JournalBulkDeleteBackgroundTask, cls).prepare(username, **kwargs) # first prepare a job record job = models.BackgroundJob() job.user = username job.action = cls.__action__ job.reference = {'selection_query': json.dumps(kwargs['selection_query'])} params = {} cls.set_param(params, 'ids', kwargs['ids']) if not cls._job_parameter_check(params): raise BackgroundException("{}.prepare run without sufficient parameters".format(cls.__name__)) job.params = params # now ensure that we have the locks for all the records, if they are lockable # will raise an exception if this fails lock.batch_lock('journal', kwargs['ids'], username, timeout=app.config.get("BACKGROUND_TASK_LOCK_TIMEOUT", 3600)) return job
def prepare(cls, username, **kwargs): """ Take an arbitrary set of keyword arguments and return an instance of a BackgroundJob, or fail with a suitable exception :param kwargs: arbitrary keyword arguments pertaining to this task type :return: a BackgroundJob instance representing this task """ write = kwargs.get("write", True) prepall = kwargs.get("prepall", False) if not write and prepall: raise BackgroundException( "'prepall' must be used with the 'write' parameter set to True (why prep but not save?)" ) params = {} cls.set_param(params, "write", write) cls.set_param(params, "prepall", prepall) # first prepare a job record job = models.BackgroundJob() job.user = username job.action = cls.__action__ job.params = params if prepall: job.add_audit_message( "'prepall' arg set. 'unchanged' articles will also have their indexes refreshed." ) return job
def test_1_http_upload_fail(self): requests.head = ResponseMockFactory.head_fail requests.get = ResponseMockFactory.get_fail url= "http://fail" file_upload = models.FileUpload() file_upload.set_id() file_upload.upload("testuser", url, status="exists") upload_dir = app.config.get("UPLOAD_DIR") path = os.path.join(upload_dir, file_upload.local_filename) self.cleanup_paths.append(path) job = models.BackgroundJob() result = ingestarticles.http_upload(job, path, file_upload) assert result is False assert file_upload.status == "failed" assert file_upload.error is not None and file_upload.error != "" assert file_upload.error_details is None assert list(file_upload.failure_reasons.keys()) == [] # now try it with an actual exception url= "http://except" file_upload.upload("testuser", url, status="exists") result = ingestarticles.http_upload(job, path, file_upload) assert result is False assert file_upload.status == "failed" assert file_upload.error is not None and file_upload.error != "" assert file_upload.error_details is None assert list(file_upload.failure_reasons.keys()) == []
def test_3_submit_retry(self): app.config["HUEY_TASKS"]["ingest_articles"]["retries"] = 1 fu = models.FileUpload() fu.validated("doaj") fu.save() job = models.BackgroundJob() params = {} params["ingest_articles__file_upload_id"] = fu.id params["ingest_articles__attempts"] = 0 job.params = params job.save(blocking=True) # this assumes that huey is in always eager mode, and thus this immediately calls the async task, # which in turn calls execute, which ultimately calls run with self.assertRaises(RetryException): ingestarticles.IngestArticlesBackgroundTask.submit(job) job = models.BackgroundJob.pull(job.id) assert job.params.get("ingest_articles__attempts") == 1 assert job.status == "processing" # now do it again, to see the retry cause the job to fail on the second attempt as per the config with self.assertRaises(RetryException): ingestarticles.IngestArticlesBackgroundTask.submit(job) job = models.BackgroundJob.pull(job.id) assert job.params.get("ingest_articles__attempts") == 2 assert job.status == "error"
def prepare(cls, username, **kwargs): """ Take an arbitrary set of keyword arguments and return an instance of a BackgroundJob, or fail with a suitable exception :param kwargs: arbitrary keyword arguments pertaining to this task type :return: a BackgroundJob instance representing this task """ params = {} cls.set_param( params, 'clean', False if "clean" not in kwargs else kwargs["clean"] if kwargs["clean"] is not None else False) cls.set_param( params, "prune", False if "prune" not in kwargs else kwargs["prune"] if kwargs["prune"] is not None else False) cls.set_param( params, "types", "all" if "types" not in kwargs else kwargs["types"] if kwargs["types"] in ["all", "journal", "article"] else "all") container = app.config.get("STORE_PUBLIC_DATA_DUMP_CONTAINER") if container is None: raise BackgroundException( "You must set STORE_PUBLIC_DATA_DUMP_CONTAINER in the config") # first prepare a job record job = models.BackgroundJob() job.user = username job.action = cls.__action__ job.params = params return job
def prepare(cls, username, **kwargs): """ Take an arbitrary set of keyword arguments and return an instance of a BackgroundJob, or fail with a suitable exception :param kwargs: arbitrary keyword arguments pertaining to this task type :return: a BackgroundJob instance representing this task """ # First prepare a job record job = models.BackgroundJob() job.user = username job.action = cls.__action__ params = {} cls.set_param( params, "outdir", kwargs.get("outdir", "article_duplicates_" + dates.today())) cls.set_param(params, "email", kwargs.get("email", False)) cls.set_param( params, "tmpdir", kwargs.get("tmpdir", "tmp_article_duplicates_" + dates.today())) cls.set_param(params, "article_csv", kwargs.get("article_csv", False)) job.params = params return job
def anonymise_background_job(record): try: bgjob = models.BackgroundJob(**record) except DataStructureException: return record if bgjob.params and 'suggestion_bulk_edit__note' in bgjob.params: bgjob.params['suggestion_bulk_edit__note'] = basic_hash(bgjob.params['suggestion_bulk_edit__note']) return bgjob.data
def test_29_background_job(self): source = BackgroundFixtureFactory.example() bj = models.BackgroundJob(**source) bj.save() time.sleep(2) retrieved = models.BackgroundJob.pull(bj.id) assert retrieved is not None source = BackgroundFixtureFactory.example() source["params"]["ids"] = ["1", "2", "3"] source["params"]["type"] = "suggestion" source["reference"]["query"] = json.dumps({"query": {"match_all": {}}}) bj = models.BackgroundJob(**source) bj.save() bj.add_audit_message("message") assert len(bj.audit) == 2
def prepare(cls, username, **kwargs): """ Take an arbitrary set of keyword arguments and return an instance of a BackgroundJob, or fail with a suitable exception :param kwargs: arbitrary keyword arguments pertaining to this task type :return: a BackgroundJob instance representing this task """ super(JournalBulkEditBackgroundTask, cls).prepare(username, **kwargs) # first prepare a job record job = models.BackgroundJob() job.user = username job.action = cls.__action__ refs = {} cls.set_reference(refs, "selection_query", json.dumps(kwargs['selection_query'])) job.reference = refs params = {} # get the named parameters we know may be there cls.set_param(params, 'ids', kwargs['ids']) if "note" in kwargs and kwargs[ "note"] is not None and kwargs["note"] != "": cls.set_param(params, 'note', kwargs.get('note', '')) # get the metadata overwrites if "replacement_metadata" in kwargs: metadata = {} for k, v in kwargs["replacement_metadata"].items(): if v is not None and v != "": metadata[k] = v if len(metadata.keys()) > 0: cls.set_param(params, 'replacement_metadata', json.dumps(metadata)) if not cls._job_parameter_check(params): raise BackgroundException( "{}.prepare run without sufficient parameters".format( cls.__name__)) job.params = params # now ensure that we have the locks for all the journals # will raise an exception if this fails lock.batch_lock("journal", kwargs['ids'], username, timeout=app.config.get("BACKGROUND_TASK_LOCK_TIMEOUT", 3600)) return job
def prepare(cls, username, **kwargs): """ Take an arbitrary set of keyword arguments and return an instance of a BackgroundJob, or fail with a suitable exception :param kwargs: arbitrary keyword arguments pertaining to this task type :return: a BackgroundJob instance representing this task """ # first prepare a job record job = models.BackgroundJob() job.user = username job.action = cls.__action__ return job
def test_2_run_errors(self): job = models.BackgroundJob() task = ingestarticles.IngestArticlesBackgroundTask(job) with self.assertRaises(BackgroundException): task.run() job.params = {} with self.assertRaises(BackgroundException): task.run() job.params = {"ingest_articles__file_upload_id": "whatever"} with self.assertRaises(BackgroundException): task.run()
def prepare(cls, username, **kwargs): """ Take an arbitrary set of keyword arguments and return an instance of a BackgroundJob, or fail with a suitable exception :param kwargs: arbitrary keyword arguments pertaining to this task type :return: a BackgroundJob instance representing this task """ cdir = app.config.get("CACHE_DIR") if cdir is None: raise BackgroundException("You must set CACHE_DIR in the config") # first prepare a job record job = models.BackgroundJob() job.user = username job.action = cls.__action__ return job
def prepare(cls, username, **kwargs): """ Take an arbitrary set of keyword arguments and return an instance of a BackgroundJob, or fail with a suitable exception :param kwargs: arbitrary keyword arguments pertaining to this task type :return: a BackgroundJob instance representing this task """ upload_dir = app.config.get("UPLOAD_DIR") if upload_dir is None: raise BackgroundException("UPLOAD_DIR is not set in configuration") f = kwargs.get("upload_file") schema = kwargs.get("schema") url = kwargs.get("url") previous = kwargs.get("previous", []) if f is None and url is None: raise BackgroundException( "You must specify one of 'upload_file' or 'url' as keyword arguments" ) if schema is None: raise BackgroundException( "You must specify 'schema' in the keyword arguments") file_upload_id = None if f is not None and f.filename != "": file_upload_id = cls._file_upload(username, f, schema, previous) elif url is not None and url != "": file_upload_id = cls._url_upload(username, url, schema, previous) if file_upload_id is None: raise BackgroundException("No file upload record was created") # first prepare a job record job = models.BackgroundJob() job.user = username job.action = cls.__action__ params = {} cls.set_param(params, "file_upload_id", file_upload_id) cls.set_param(params, "attempts", 0) job.params = params return job
def prepare(cls, username, **kwargs): """ Take an arbitrary set of keyword arguments and return an instance of a BackgroundJob, or fail with a suitable exception :param username: user who called this job :param kwargs: arbitrary keyword arguments pertaining to this task type :return: a BackgroundJob instance representing this task """ if not app.config.get("ENABLE_EMAIL", False): raise BackgroundException("Email has been disabled in config. Set ENABLE_EMAIL to True to run this task.") # first prepare a job record job = models.BackgroundJob() job.user = username job.action = cls.__action__ return job
def prepare(cls, username, **kwargs): """ Take an arbitrary set of keyword arguments and return an instance of a BackgroundJob, or fail with a suitable exception :param kwargs: arbitrary keyword arguments pertaining to this task type :return: a BackgroundJob instance representing this task """ # first prepare a job record job = models.BackgroundJob() job.user = username job.action = cls.__action__ journal_ids = kwargs.get("journal_ids") params = {} cls.set_param(params, "journal_ids", journal_ids) cls.set_param(params, "in_doaj", kwargs.get("in_doaj")) if journal_ids is None or len( journal_ids) == 0 or kwargs.get("in_doaj") is None: raise RuntimeError( u"SetInDOAJBackgroundTask.prepare run without sufficient parameters" ) job.params = params if "selection_query" in kwargs: refs = {} cls.set_reference(refs, "selection_query", json.dumps(kwargs.get('selection_query'))) job.reference = refs # now ensure that we have the locks for this journal # will raise an exception if this fails lock.batch_lock("journal", journal_ids, username, timeout=app.config.get("BACKGROUND_TASK_LOCK_TIMEOUT", 3600)) return job
def prepare(cls, username, **kwargs): """ Take an arbitrary set of keyword arguments and return an instance of a BackgroundJob, or fail with a suitable exception :param kwargs: arbitrary keyword arguments pertaining to this task type :return: a BackgroundJob instance representing this task """ job = models.BackgroundJob() job.user = username job.action = cls.__action__ params = {} cls.set_param(params, "outdir", kwargs.get("outdir", "report_" + dates.today())) cls.set_param(params, "from", kwargs.get("from_date", "1970-01-01T00:00:00Z")) cls.set_param(params, "to", kwargs.get("to_date", dates.now())) cls.set_param(params, "email", kwargs.get("email", False)) job.params = params return job