def test_execute(self):
"""Test executing a sequence of supported commands."""
context = dict()
fh = self.backend.projects.get_project(
self.PROJECT_ID).filestore.upload_file(CSV_FILE)
cmd = vizual.load_dataset(dataset_name=DATASET_NAME,
file={pckg.FILE_ID: fh.identifier},
validate=True)
result = self.backend.execute(task=TaskHandle(
task_id='000', project_id=self.PROJECT_ID),
command=cmd,
context=context)
self.assertTrue(result.is_success)
state = result.provenance.get_database_state(prev_state=dict())
context = task_context(state)
cmd = vizual.update_cell(dataset_name=DATASET_NAME,
column=1,
row=0,
value=9,
validate=True)
result = self.backend.execute(task=TaskHandle(
task_id='000', project_id=self.PROJECT_ID),
command=cmd,
context=context)
self.assertTrue(result.is_success)
state = result.provenance.get_database_state(prev_state=state)
self.assertNotEqual(context[DATASET_NAME],
task_context(state)[DATASET_NAME])
context = task_context(state)
cmd = pycell.python_cell(source=CREATE_DATASET_PY, validate=True)
result = self.backend.execute(task=TaskHandle(
task_id='000', project_id=self.PROJECT_ID),
command=cmd,
context=context)
self.assertTrue(result.is_success)
state = result.provenance.get_database_state(prev_state=state)
self.assertTrue(SECOND_DATASET_NAME in state)
self.assertEqual(context[DATASET_NAME],
task_context(state)[DATASET_NAME])
context = task_context(state)
cmd = vizual.update_cell(dataset_name=SECOND_DATASET_NAME,
column=1,
row=0,
value=9,
validate=True)
result = self.backend.execute(task=TaskHandle(
task_id='000', project_id=self.PROJECT_ID),
command=cmd,
context=context)
self.assertTrue(result.is_success)
state = result.provenance.get_database_state(prev_state=state)
self.assertEqual(context[DATASET_NAME],
task_context(state)[DATASET_NAME])
self.assertNotEqual(context[SECOND_DATASET_NAME],
task_context(state)[SECOND_DATASET_NAME])
def test_execute_unsupported_command(self):
"""Test executing commands that are not supported for synchronous
execution.
"""
with self.assertRaises(ValueError):
self.backend.execute(task=TaskHandle(task_id='000',
project_id=self.PROJECT_ID),
command=vizual.insert_row(
dataset_name=DATASET_NAME,
position=1,
validate=True),
context=dict())
with self.assertRaises(ValueError):
self.backend.execute(task=TaskHandle(task_id='000',
project_id=self.PROJECT_ID),
command=vizual.drop_dataset(
dataset_name=DATASET_NAME, validate=True),
context=dict())
def test_execute(self):
"""Test executing a sequence of supported commands."""
context = dict()
cmd = pycell.python_cell(source='print(2+2)', validate=True)
controller = FakeWorkflowController()
self.backend.execute_async(task=TaskHandle(task_id='000',
project_id=self.PROJECT_ID,
controller=controller),
command=cmd,
artifacts=context)
time.sleep(3)
self.assertEqual(controller.task_id, '000')
self.assertEqual(controller.state, 'SUCCESS')
self.assertEqual(controller.outputs.stdout[0].value, '4')
def test_cancel(self) -> None:
"""Test executing a sequence of supported commands."""
context: Dict[str, ArtifactDescriptor] = dict()
cmd = pycell.python_cell(source='import time\ntime.sleep(5)',
validate=True)
controller = FakeWorkflowController()
self.backend.execute_async(task=TaskHandle(task_id='000',
project_id=self.PROJECT_ID,
controller=controller),
command=cmd,
artifacts=context)
time.sleep(1)
self.backend.cancel_task('000')
time.sleep(5)
self.assertIsNone(controller.task_id)
self.assertIsNone(controller.state)
def test_error(self) -> None:
"""Test executing a command with processor that raises an exception
instead of returning an execution result.
"""
context: Dict[str, ArtifactDescriptor] = dict()
cmd = ModuleCommand(package_id='error',
command_id='error',
arguments=[],
packages=None)
controller = FakeWorkflowController()
self.backend.execute_async(task=TaskHandle(task_id='000',
project_id=self.PROJECT_ID,
controller=controller),
command=cmd,
artifacts=context)
time.sleep(2)
self.assertEqual(controller.task_id, '000')
self.assertEqual(controller.state, 'ERROR')
self.assertEqual(len(controller.outputs.stdout), 0)
self.assertNotEqual(len(controller.outputs.stderr), 0)
def execute_task(self,
project_id,
task_id,
command,
context,
resources=None):
"""Execute a given command asynchronously using the engies backend.
Make use to use the remote web service as controller that orchestrates
workflow execution.
Parameters
----------
project_id: string
Unique project identifier
task_id: string
Unique task identifier
command : vizier.viztrail.command.ModuleCommand
Specification of the command that is to be executed
context: dict
Dictionary of available resource in the database state. The key is
the resource name. Values are resource identifiers.
resources: dict, optional
Optional information about resources that were generated during a
previous execution of the command
Returns
------
dict
"""
self.engine.backend.execute_async(task=TaskHandle(
project_id=project_id,
task_id=task_id,
controller=RemoteWorkflowController(urls=TaskUrlFactory(
base_url=self.controller_url))),
command=command,
context=context,
resources=resources)
return {labels.RESULT: True}
def append_workflow_module(self, project_id, branch_id, command):
"""Append module to the workflow at the head of the given viztrail
branch. The modified workflow will be executed. The result is the new
head of the branch.
Returns the handle for the new module in the modified workflow. The
result is None if the specified project or branch do not exist.
Parameters
----------
project_id: string
Unique project identifier
branch_id : string
Unique branch identifier
command : vizier.viztrail.command.ModuleCommand
Specification of the command that is to be executed by the appended
workflow module
Returns
-------
vizier.viztrail.module.base.ModuleHandle
"""
with self.backend.lock:
# Get the handle for the specified branch
branch = self.projects.get_branch(project_id=project_id,
branch_id=branch_id)
if branch is None:
return None
# Get the current database state from the last module in the current
# branch head. At the same time we retrieve the list of modules for
# the current head of the branch.
head = branch.get_head()
if not head is None and len(head.modules) > 0:
datasets = head.modules[-1].datasets
modules = head.modules
is_active = head.is_active
is_error = head.modules[-1].is_error or head.modules[
-1].is_canceled
else:
datasets = dict()
modules = list()
is_active = False
is_error = False
# Get the external representation for the command
external_form = command.to_external_form(
command=self.packages[command.package_id].get(
command.command_id),
datasets=datasets)
# If the workflow is not active and the command can be executed
# synchronously we run the command immediately and return the
# completed workflow. Otherwise, a pending workflow is created.
if not is_active and self.backend.can_execute(command):
ts_start = get_current_time()
result = self.backend.execute(task=TaskHandle(
task_id=get_unique_identifier(),
project_id=project_id,
controller=self),
command=command,
context=task_context(datasets))
ts = ModuleTimestamp(created_at=ts_start,
started_at=ts_start,
finished_at=get_current_time())
# Depending on the execution outcome create a handle for the
# executed module
if result.is_success:
module = ModuleHandle(
state=mstate.MODULE_SUCCESS,
command=command,
external_form=external_form,
timestamp=ts,
datasets=result.provenance.get_database_state(
modules[-1].datasets if len(modules) > 0 else dict(
)),
outputs=result.outputs,
provenance=result.provenance)
else:
module = ModuleHandle(state=mstate.MODULE_ERROR,
command=command,
external_form=external_form,
timestamp=ts,
outputs=result.outputs)
workflow = branch.append_workflow(modules=modules,
action=wf.ACTION_APPEND,
command=command,
pending_modules=[module])
else:
# Create new workflow by appending one module to the current
# head of the branch. The module state is pending if the
# workflow is active otherwise it depends on the associated
# backend.
if is_active:
state = mstate.MODULE_PENDING
elif is_error:
state = mstate.MODULE_CANCELED
else:
state = self.backend.next_task_state()
workflow = branch.append_workflow(
modules=modules,
action=wf.ACTION_APPEND,
command=command,
pending_modules=[
ModuleHandle(state=state,
command=command,
external_form=external_form)
])
if not is_active and not state == mstate.MODULE_CANCELED:
self.execute_module(project_id=project_id,
branch_id=branch_id,
module=workflow.modules[-1],
datasets=datasets)
return workflow.modules[-1]