def test_runs_first_task(self): task = VoidTask() uut = TaskRunner([task]) uut.run() task.assert_called_once_with()
def test_passes_initial_parameter_values(self): task = StringConsumingTask() uut = TaskRunner([task]) uut.run("-string-") task.assert_called_once_with("-string-")
def test_runs_second_task_with_result_of_first_task(self): first_task = VoidTask([":some string:"]) second_task = StringConsumingTask() uut = TaskRunner([first_task, second_task]) uut.run() second_task.assert_called_once_with(":some string:")
def test_runs_subsequent_task_with_generic_result_of_previous_task(self): first_task = VoidTask([[1, 2]]) second_task = ListConsumingTask() uut = TaskRunner([first_task, second_task]) uut.run() second_task.assert_called_once_with([1, 2])
def test_continue_runs_next_task(self): first_task = VoidTask(results=Continue) second_task = VoidTask() uut = TaskRunner([first_task, second_task]) uut.run() second_task.assert_called_once_with()
def test_task_can_return_a_single_instance(self): first_task = VoidTask(results="-some string-") second_task = StringConsumingTask() uut = TaskRunner([first_task, second_task]) uut.run() second_task.assert_called_once_with("-some string-")
def test_runs_second_task_with_each_result_of_first_task(self): first_task = VoidTask([":some string:", ":other string:"]) second_task = StringConsumingTask() uut = TaskRunner([first_task, second_task]) uut.run() second_task.assert_has_calls([(":some string:", ), (":other string:", )])
def test_runs_subsequent_task_with_required_results_of_previous_tasks( self): first_task = VoidTask([":some string:"]) second_task = VoidTask() uut = TaskRunner([first_task, second_task]) uut.run() second_task.assert_called_once_with()
def test_none_result_continues_execution(self): first_task = VoidTask(results="-some string-") second_task = NoneReturningTask() third_task = StringConsumingTask() uut = TaskRunner([first_task, second_task, third_task]) uut.run() third_task.assert_called_once_with("-some string-")
def test_continues_with_next_input_if_task_fails(self): first_task = VoidTask(results=["-some string-", "-some other string-"]) second_task = FailingStringConsumingTask("-error-") uut = TaskRunner([first_task, second_task]) uut.logger = MagicMock() uut.run() second_task.assert_called_once_with("-some other string-")
def test_runs_subsequent_task_with_all_results_of_previous_tasks(self): first_task = VoidTask([":some string:"]) second_task = StringConsumingTask([42]) third_task = StringAndIntConsumingTask() uut = TaskRunner([first_task, second_task, third_task]) uut.run() third_task.assert_called_once_with(":some string:", 42)
def test_asterisk_arg_gets_all_available_parameters(self): first_task = VoidTask(results="-some string-") second_task = VoidTask(results=42) asterisk_task = AsteriskTask() uut = TaskRunner([first_task, second_task, asterisk_task]) uut.run() asterisk_task.assert_called_once_with("-some string-", 42)
def test_runs_subsequent_task_with_results_of_previous_tasks_in_any_order( self): first_task = VoidTask([":some string:"]) second_task = StringConsumingTask([42]) third_task = IntAndStringConsumingTask() uut = TaskRunner([first_task, second_task, third_task]) uut.run() third_task.assert_called_once_with(42, ":some string:")
def test_reports_if_a_task_requires_an_unavailable_parameter(self): first_task = VoidTask([42]) second_task = StringConsumingTask() uut = TaskRunner([first_task, second_task]) with assert_raises(TaskParameterUnavailableWarning) as context: uut.run() actual_message = str(context.exception) assert_equals("Missing parameter s for task StringConsumingTask", actual_message)
def test_reports_if_a_task_requests_the_same_type_multiple_times(self): first_task = VoidTask([42]) second_task = DuplicateIntRequestingTask() uut = TaskRunner([first_task, second_task]) with assert_raises(TaskRequestsDuplicateTypeWarning) as context: uut.run() actual_message = str(context.exception) assert_equals( "Task DuplicateIntRequestingTask requests multiple parameters of type int", actual_message)
def test_reports_if_a_task_returns_the_same_type_as_a_previous_task(self): first_task = VoidTask(['-some-string-']) second_task = StringConsumingTask(['-some-other-string-']) uut = TaskRunner([first_task, second_task]) with assert_raises(TaskParameterDuplicateTypeWarning) as context: uut.run() actual_message = str(context.exception) assert_equals( "Parameter type str provided by task StringConsumingTask already exists", actual_message)
def test_can_run_another_task_runner(self): first_outer_task = VoidTask(results="-some string-") first_inner_task = StringConsumingTask(results=42) second_inner_task = StringAndIntConsumingTask() second_outer_task = StringConsumingTask(results=42) inner_runner = TaskRunner([first_inner_task, second_inner_task]) uut = TaskRunner([first_outer_task, inner_runner, second_outer_task]) uut.run() first_inner_task.assert_called_once_with("-some string-") second_inner_task.assert_called_once_with("-some string-", 42) second_outer_task.assert_called_once_with("-some string-")
def start_task(request, task_id): if request.method == "POST": definition = SERVICE_TASK_DEFINITIONS[task_id] started_by = request.POST.get("started_by") if not started_by: return HttpResponseNotFound() task_arguments = {"started_by": started_by} for parameter in definition['parameters']: name = parameter['name'] if parameter['is_primitive']: if parameter['required'] and name not in request.POST: return HttpResponseNotFound() if parameter['type'] == 'bool': if name in request.POST: task_arguments[name] = True else: task_arguments[name] = False else: default = None if not parameter["required"]: default = parameter['default'] task_arguments[name] = PrimitivesHelper.convert( parameter['type'], request.POST.get(name, default)) task_cls = SERVICE_TASKS[task_id] task = TaskRunner.run_task_async(task_cls, **task_arguments) return JsonResponse({"task": task.pk}) return HttpResponseNotFound()
def run(self): self.log("Updating medRxiv/bioRxiv articles...") TaskRunner.run_task(MedBiorxivUpdateTask, force_update=self.force_update, count=self.medrxiv_count, started_by=self._task.started_by) self.log("Finished updating medRxiv/bioRxiv articles...") self.progress(15) self.log("Updating arXiv articles...") TaskRunner.run_task(ArxivUpdateTask, force_update=self.force_update, count=self.arxiv_count, started_by=self._task.started_by) self.progress(30) self.log("Finished updating arXiv articles...") self.log("Updating Elsevier articles...") TaskRunner.run_task(ElsevierUpdateTask, force_update=self.force_update, count=self.elsevier_count, started_by=self._task.started_by) self.log("Finished updating Elsevier articles...") self.progress(45) self.log("Updating Pubmed articles...") TaskRunner.run_task(PubmedUpdateTask, force_update=self.force_update, count=self.pubmed_count, started_by=self._task.started_by) self.log("Finished updating Pubmed articles...") self.progress(60) if settings.UPDATE_VECTORIZER: self.log("Updating Topic assigment...") task_launcher = get_task_launcher('search') task_config = { 'service': 'search', 'parameters': [], 'started_by': self._task.started_by } task_launcher.launch_task(name="setup-vectorizer", config=task_config, block=True) self.progress(70) self.log("Finished setup-vectorizer") task_launcher.launch_task(name="update-category-assignment", config=task_config, block=True) self.progress(80) self.log("Finished updating category assigment") task_launcher.launch_task(name="nearest-neighbor-topic-assignment", config=task_config, block=True) self.progress(90) self.log("Finished nearest-neighbor-topic-assignment") task_launcher.launch_task(name="reduce-embedding-dimensionality", config=task_config, block=True) self.log("Finished reduce-embedding-dimensionality") self.progress(95) else: self.log("Paper matrix update and topic assignment skipped.") self.log("Extract locations from papers...") TaskRunner.run_task(GeoParserTask, started_by=self._task.started_by) self.log("Finished extracting locations from papers")
def run(self): self.log("Get new medRxiv/bioRxiv articles...") TaskRunner.run_task(MedBiorxivNewArticlesTask, started_by=self._task.started_by) self.log("Finished getting new medRxiv/bioRxiv articles...") self.progress(10) self.log("Get new arXiv articles...") TaskRunner.run_task(ArxivNewArticlesTask, started_by=self._task.started_by) self.progress(20) self.log("Finished getting new arXiv articles...") self.log("Get new Elsevier articles...") TaskRunner.run_task(ElsevierNewArticlesTask, started_by=self._task.started_by) self.progress(40) self.log("Get new Pubmed articles...") TaskRunner.run_task(PubmedNewArticlesTask, started_by=self._task.started_by) self.log("Finished getting new Pubmed articles...") self.progress(60) if settings.UPDATE_VECTORIZER: self.log("Updating Topic assigment...") task_launcher = get_task_launcher('search') task_config = { 'service': 'search', 'parameters': [], 'started_by': self._task.started_by } task_launcher.launch_task(name="setup-vectorizer", config=task_config, block=True) self.progress(70) self.log("Finished setup-vectorizer") task_launcher.launch_task(name="update-category-assignment", config=task_config, block=True) self.progress(80) self.log("Finished updating category assigment") task_launcher.launch_task(name="nearest-neighbor-topic-assignment", config=task_config, block=True) self.progress(90) self.log("Finished nearest-neighbor-topic-assignment") task_launcher.launch_task(name="reduce-embedding-dimensionality", config=task_config, block=True) self.log("Finished reduce-embedding-dimensionality") self.progress(95) else: self.log("Paper matrix update and topic assignment skipped.") self.log("Update Altmetric data of new papers") TaskRunner.run_task(AltmetricUpdateTask, started_by=self._task.started_by, update_all=True, only_new=True) self.log("Finished updating Altmetric data of new papers") self.log("Extract locations from papers...") TaskRunner.run_task(GeoParserTask, started_by=self._task.started_by) self.log("Finished extracting locations from papers")
def test_continue_is_not_added_to_available_parameters(self): first_task = VoidTask(results=Continue) second_task = ContinueConsumingTask() uut = TaskRunner([first_task, second_task]) assert_raises(TaskParameterUnavailableWarning, uut.run)
_INDEX_PATH = os.path.join(__MUBENCH_ROOT_PATH, "checkouts-xp", "index.csv") class PrintIndexTask: def run(self, project: Project, version: ProjectVersion, misuse: Misuse): print("{}\t{}\t{}\t{}\t{}\t{}\t{}".format( project.id, version.version_id, misuse.misuse_id, ':'.join(version.source_dirs), misuse.location.file, misuse.location.method, "\t".join(misuse.apis)), file=open(_INDEX_PATH, "a")) datasets = sys.argv[1:] white_list = [] for dataset in datasets: white_list.extend(get_white_list(__MUBENCH_DATASETS_FILE, dataset.lower())) initial_parameters = [DataEntityLists(white_list, [])] runner = TaskRunner([ CollectProjectsTask(__MUBENCH_DATA_PATH), CollectVersionsTask(False), CollectMisusesTask(), PrintIndexTask() ]) if os.path.exists(_INDEX_PATH): os.remove(_INDEX_PATH) runner.run(*initial_parameters)
def test_handles_empty_tasks(self): uut = TaskRunner([]) uut.run()
if __name__ == '__main__': import django django.setup() from scrape.task_check_covid_related import CheckCovidRelatedTask from tasks.task_runner import TaskRunner TaskRunner.run_task(CheckCovidRelatedTask, started_by="Setup Script")
def run(self) -> None: RequirementsCheck() task_configuration = get_task_configuration(self.config) initial_parameters = [self.data_entity_lists] runner = TaskRunner(task_configuration) runner.run(*initial_parameters)