def test_runs_first_task(self):
task = VoidTask()
uut = TaskRunner([task])
uut.run()
task.assert_called_once_with()
def test_passes_initial_parameter_values(self):
task = StringConsumingTask()
uut = TaskRunner([task])
uut.run("-string-")
task.assert_called_once_with("-string-")
def test_runs_second_task_with_result_of_first_task(self):
first_task = VoidTask([":some string:"])
second_task = StringConsumingTask()
uut = TaskRunner([first_task, second_task])
uut.run()
second_task.assert_called_once_with(":some string:")
def test_runs_subsequent_task_with_generic_result_of_previous_task(self):
first_task = VoidTask([[1, 2]])
second_task = ListConsumingTask()
uut = TaskRunner([first_task, second_task])
uut.run()
second_task.assert_called_once_with([1, 2])
def test_continue_runs_next_task(self):
first_task = VoidTask(results=Continue)
second_task = VoidTask()
uut = TaskRunner([first_task, second_task])
uut.run()
second_task.assert_called_once_with()
def test_task_can_return_a_single_instance(self):
first_task = VoidTask(results="-some string-")
second_task = StringConsumingTask()
uut = TaskRunner([first_task, second_task])
uut.run()
second_task.assert_called_once_with("-some string-")
def test_runs_second_task_with_each_result_of_first_task(self):
first_task = VoidTask([":some string:", ":other string:"])
second_task = StringConsumingTask()
uut = TaskRunner([first_task, second_task])
uut.run()
second_task.assert_has_calls([(":some string:", ),
(":other string:", )])
def test_runs_subsequent_task_with_required_results_of_previous_tasks(
self):
first_task = VoidTask([":some string:"])
second_task = VoidTask()
uut = TaskRunner([first_task, second_task])
uut.run()
second_task.assert_called_once_with()
def test_none_result_continues_execution(self):
first_task = VoidTask(results="-some string-")
second_task = NoneReturningTask()
third_task = StringConsumingTask()
uut = TaskRunner([first_task, second_task, third_task])
uut.run()
third_task.assert_called_once_with("-some string-")
def test_continues_with_next_input_if_task_fails(self):
first_task = VoidTask(results=["-some string-", "-some other string-"])
second_task = FailingStringConsumingTask("-error-")
uut = TaskRunner([first_task, second_task])
uut.logger = MagicMock()
uut.run()
second_task.assert_called_once_with("-some other string-")
def test_runs_subsequent_task_with_all_results_of_previous_tasks(self):
first_task = VoidTask([":some string:"])
second_task = StringConsumingTask([42])
third_task = StringAndIntConsumingTask()
uut = TaskRunner([first_task, second_task, third_task])
uut.run()
third_task.assert_called_once_with(":some string:", 42)
def test_asterisk_arg_gets_all_available_parameters(self):
first_task = VoidTask(results="-some string-")
second_task = VoidTask(results=42)
asterisk_task = AsteriskTask()
uut = TaskRunner([first_task, second_task, asterisk_task])
uut.run()
asterisk_task.assert_called_once_with("-some string-", 42)
def test_runs_subsequent_task_with_results_of_previous_tasks_in_any_order(
self):
first_task = VoidTask([":some string:"])
second_task = StringConsumingTask([42])
third_task = IntAndStringConsumingTask()
uut = TaskRunner([first_task, second_task, third_task])
uut.run()
third_task.assert_called_once_with(42, ":some string:")
def test_reports_if_a_task_requires_an_unavailable_parameter(self):
first_task = VoidTask([42])
second_task = StringConsumingTask()
uut = TaskRunner([first_task, second_task])
with assert_raises(TaskParameterUnavailableWarning) as context:
uut.run()
actual_message = str(context.exception)
assert_equals("Missing parameter s for task StringConsumingTask",
actual_message)
def test_reports_if_a_task_requests_the_same_type_multiple_times(self):
first_task = VoidTask([42])
second_task = DuplicateIntRequestingTask()
uut = TaskRunner([first_task, second_task])
with assert_raises(TaskRequestsDuplicateTypeWarning) as context:
uut.run()
actual_message = str(context.exception)
assert_equals(
"Task DuplicateIntRequestingTask requests multiple parameters of type int",
actual_message)
def test_reports_if_a_task_returns_the_same_type_as_a_previous_task(self):
first_task = VoidTask(['-some-string-'])
second_task = StringConsumingTask(['-some-other-string-'])
uut = TaskRunner([first_task, second_task])
with assert_raises(TaskParameterDuplicateTypeWarning) as context:
uut.run()
actual_message = str(context.exception)
assert_equals(
"Parameter type str provided by task StringConsumingTask already exists",
actual_message)
def test_can_run_another_task_runner(self):
first_outer_task = VoidTask(results="-some string-")
first_inner_task = StringConsumingTask(results=42)
second_inner_task = StringAndIntConsumingTask()
second_outer_task = StringConsumingTask(results=42)
inner_runner = TaskRunner([first_inner_task, second_inner_task])
uut = TaskRunner([first_outer_task, inner_runner, second_outer_task])
uut.run()
first_inner_task.assert_called_once_with("-some string-")
second_inner_task.assert_called_once_with("-some string-", 42)
second_outer_task.assert_called_once_with("-some string-")
def start_task(request, task_id):
if request.method == "POST":
definition = SERVICE_TASK_DEFINITIONS[task_id]
started_by = request.POST.get("started_by")
if not started_by:
return HttpResponseNotFound()
task_arguments = {"started_by": started_by}
for parameter in definition['parameters']:
name = parameter['name']
if parameter['is_primitive']:
if parameter['required'] and name not in request.POST:
return HttpResponseNotFound()
if parameter['type'] == 'bool':
if name in request.POST:
task_arguments[name] = True
else:
task_arguments[name] = False
else:
default = None
if not parameter["required"]:
default = parameter['default']
task_arguments[name] = PrimitivesHelper.convert(
parameter['type'], request.POST.get(name, default))
task_cls = SERVICE_TASKS[task_id]
task = TaskRunner.run_task_async(task_cls, **task_arguments)
return JsonResponse({"task": task.pk})
return HttpResponseNotFound()
def run(self):
self.log("Updating medRxiv/bioRxiv articles...")
TaskRunner.run_task(MedBiorxivUpdateTask,
force_update=self.force_update,
count=self.medrxiv_count,
started_by=self._task.started_by)
self.log("Finished updating medRxiv/bioRxiv articles...")
self.progress(15)
self.log("Updating arXiv articles...")
TaskRunner.run_task(ArxivUpdateTask,
force_update=self.force_update,
count=self.arxiv_count,
started_by=self._task.started_by)
self.progress(30)
self.log("Finished updating arXiv articles...")
self.log("Updating Elsevier articles...")
TaskRunner.run_task(ElsevierUpdateTask,
force_update=self.force_update,
count=self.elsevier_count,
started_by=self._task.started_by)
self.log("Finished updating Elsevier articles...")
self.progress(45)
self.log("Updating Pubmed articles...")
TaskRunner.run_task(PubmedUpdateTask,
force_update=self.force_update,
count=self.pubmed_count,
started_by=self._task.started_by)
self.log("Finished updating Pubmed articles...")
self.progress(60)
if settings.UPDATE_VECTORIZER:
self.log("Updating Topic assigment...")
task_launcher = get_task_launcher('search')
task_config = {
'service': 'search',
'parameters': [],
'started_by': self._task.started_by
}
task_launcher.launch_task(name="setup-vectorizer",
config=task_config,
block=True)
self.progress(70)
self.log("Finished setup-vectorizer")
task_launcher.launch_task(name="update-category-assignment",
config=task_config,
block=True)
self.progress(80)
self.log("Finished updating category assigment")
task_launcher.launch_task(name="nearest-neighbor-topic-assignment",
config=task_config,
block=True)
self.progress(90)
self.log("Finished nearest-neighbor-topic-assignment")
task_launcher.launch_task(name="reduce-embedding-dimensionality",
config=task_config,
block=True)
self.log("Finished reduce-embedding-dimensionality")
self.progress(95)
else:
self.log("Paper matrix update and topic assignment skipped.")
self.log("Extract locations from papers...")
TaskRunner.run_task(GeoParserTask, started_by=self._task.started_by)
self.log("Finished extracting locations from papers")
def run(self):
self.log("Get new medRxiv/bioRxiv articles...")
TaskRunner.run_task(MedBiorxivNewArticlesTask,
started_by=self._task.started_by)
self.log("Finished getting new medRxiv/bioRxiv articles...")
self.progress(10)
self.log("Get new arXiv articles...")
TaskRunner.run_task(ArxivNewArticlesTask,
started_by=self._task.started_by)
self.progress(20)
self.log("Finished getting new arXiv articles...")
self.log("Get new Elsevier articles...")
TaskRunner.run_task(ElsevierNewArticlesTask,
started_by=self._task.started_by)
self.progress(40)
self.log("Get new Pubmed articles...")
TaskRunner.run_task(PubmedNewArticlesTask,
started_by=self._task.started_by)
self.log("Finished getting new Pubmed articles...")
self.progress(60)
if settings.UPDATE_VECTORIZER:
self.log("Updating Topic assigment...")
task_launcher = get_task_launcher('search')
task_config = {
'service': 'search',
'parameters': [],
'started_by': self._task.started_by
}
task_launcher.launch_task(name="setup-vectorizer",
config=task_config,
block=True)
self.progress(70)
self.log("Finished setup-vectorizer")
task_launcher.launch_task(name="update-category-assignment",
config=task_config,
block=True)
self.progress(80)
self.log("Finished updating category assigment")
task_launcher.launch_task(name="nearest-neighbor-topic-assignment",
config=task_config,
block=True)
self.progress(90)
self.log("Finished nearest-neighbor-topic-assignment")
task_launcher.launch_task(name="reduce-embedding-dimensionality",
config=task_config,
block=True)
self.log("Finished reduce-embedding-dimensionality")
self.progress(95)
else:
self.log("Paper matrix update and topic assignment skipped.")
self.log("Update Altmetric data of new papers")
TaskRunner.run_task(AltmetricUpdateTask,
started_by=self._task.started_by,
update_all=True,
only_new=True)
self.log("Finished updating Altmetric data of new papers")
self.log("Extract locations from papers...")
TaskRunner.run_task(GeoParserTask, started_by=self._task.started_by)
self.log("Finished extracting locations from papers")
def test_continue_is_not_added_to_available_parameters(self):
first_task = VoidTask(results=Continue)
second_task = ContinueConsumingTask()
uut = TaskRunner([first_task, second_task])
assert_raises(TaskParameterUnavailableWarning, uut.run)
_INDEX_PATH = os.path.join(__MUBENCH_ROOT_PATH, "checkouts-xp", "index.csv")
class PrintIndexTask:
def run(self, project: Project, version: ProjectVersion, misuse: Misuse):
print("{}\t{}\t{}\t{}\t{}\t{}\t{}".format(
project.id, version.version_id, misuse.misuse_id,
':'.join(version.source_dirs), misuse.location.file,
misuse.location.method, "\t".join(misuse.apis)),
file=open(_INDEX_PATH, "a"))
datasets = sys.argv[1:]
white_list = []
for dataset in datasets:
white_list.extend(get_white_list(__MUBENCH_DATASETS_FILE, dataset.lower()))
initial_parameters = [DataEntityLists(white_list, [])]
runner = TaskRunner([
CollectProjectsTask(__MUBENCH_DATA_PATH),
CollectVersionsTask(False),
CollectMisusesTask(),
PrintIndexTask()
])
if os.path.exists(_INDEX_PATH):
os.remove(_INDEX_PATH)
runner.run(*initial_parameters)
def test_handles_empty_tasks(self):
uut = TaskRunner([])
uut.run()
if __name__ == '__main__':
import django
django.setup()
from scrape.task_check_covid_related import CheckCovidRelatedTask
from tasks.task_runner import TaskRunner
TaskRunner.run_task(CheckCovidRelatedTask, started_by="Setup Script")
def run(self) -> None:
RequirementsCheck()
task_configuration = get_task_configuration(self.config)
initial_parameters = [self.data_entity_lists]
runner = TaskRunner(task_configuration)
runner.run(*initial_parameters)