def test_running(self):
"""Update module state from pending to running."""
# Create original module
module = OSModuleHandle.create_module(
command=python_cell(source='print 2+2'),
external_form='TEST MODULE',
state=MODULE_PENDING,
module_folder=MODULE_DIR,
timestamp=ModuleTimestamp(),
datasets={'DS1': DS1},
outputs=ModuleOutputs(stdout=[TextOutput('ABC')]),
provenance=ModuleProvenance(
read={'DS1': 'ID1'},
write={'DS1': DatasetDescriptor(identifier='ID2')},
resources={'fileid': '0123456789'}))
self.assertTrue(module.is_pending)
module.set_running(external_form='TEST MODULE')
self.assertTrue(module.is_running)
self.assertIsNotNone(module.timestamp.started_at)
self.assertEqual(len(module.datasets), 0)
self.assertEqual(len(module.outputs.stderr), 0)
self.assertEqual(len(module.outputs.stdout), 0)
self.assertIsNotNone(module.provenance.read)
self.assertIsNotNone(module.provenance.write)
self.assertIsNotNone(module.provenance.resources)
# Read module from object store and ensure that tall changes have been
# materialized properly
module = OSModuleHandle.load_module(identifier=module.identifier,
module_path=module.module_path)
self.assertTrue(module.is_running)
self.assertIsNotNone(module.timestamp.started_at)
self.assertEqual(len(module.datasets), 0)
self.assertEqual(len(module.outputs.stderr), 0)
self.assertEqual(len(module.outputs.stdout), 0)
self.assertIsNotNone(module.provenance.read)
self.assertIsNotNone(module.provenance.write)
self.assertIsNotNone(module.provenance.resources)
# Set running with all optional parameters
module.set_running(started_at=module.timestamp.created_at,
external_form='Some form')
self.assertEqual(module.timestamp.started_at,
module.timestamp.created_at)
self.assertEqual(module.external_form, 'Some form')
module = OSModuleHandle.load_module(identifier=module.identifier,
module_path=module.module_path)
self.assertEqual(module.timestamp.started_at,
module.timestamp.created_at)
self.assertEqual(module.external_form, 'Some form')
def read_workflow_modules(
modules_list: List[str],
modules_folder: str,
object_store: ObjectStore
) -> List[ModuleHandle]:
"""Read workflow modules from object store.
Parameters
----------
modules_list: list(string)
List of module identifier
modules_folder: string
Path to the folder containing moudle objects
object_store: vizier.core.io.base.ObjectStore
Object store implementation to access and maintain resources
Returns
-------
list(vizier.viztrail.objectstore.module.OSModuleHandle)
"""
modules: List[ModuleHandle] = list()
for module_id in modules_list:
module_path=get_module_path(
modules_folder=modules_folder,
module_id=module_id,
object_store=object_store
)
m = OSModuleHandle.load_module(
identifier=module_id,
module_path=module_path,
object_store=object_store
)
modules.append(m)
return modules
def test_timestamps(self):
"""Test reading and writing modules with different timestamp values."""
mod0 = OSModuleHandle.create_module(
command=python_cell(source='print 2+2'),
external_form='TEST MODULE',
state=MODULE_PENDING,
outputs=ModuleOutputs(),
provenance=ModuleProvenance(),
timestamp=ModuleTimestamp(),
module_folder=MODULE_DIR)
m = OSModuleHandle.load_module(identifier=mod0.identifier,
module_path=mod0.module_path)
# Test timestamps
created_at = m.timestamp.created_at
started_at = to_datetime('2018-11-26T13:00:00.000000')
m.timestamp.started_at = started_at
m.write_module()
m = OSModuleHandle.load_module(identifier=mod0.identifier,
module_path=mod0.module_path)
self.assertEqual(m.timestamp.created_at, created_at)
self.assertEqual(m.timestamp.started_at, started_at)
finished_at = to_datetime('2018-11-26T13:00:00.000010')
m.timestamp.created_at = finished_at
m.timestamp.finished_at = finished_at
m.write_module()
m = OSModuleHandle.load_module(identifier=mod0.identifier,
module_path=mod0.module_path)
self.assertEqual(m.timestamp.created_at, finished_at)
self.assertEqual(m.timestamp.started_at, started_at)
self.assertEqual(m.timestamp.finished_at, finished_at)
mod0 = OSModuleHandle.create_module(
command=python_cell(source='print 2+2'),
external_form='TEST MODULE',
state=MODULE_PENDING,
outputs=ModuleOutputs(),
provenance=ModuleProvenance(),
timestamp=ModuleTimestamp(created_at=created_at,
started_at=started_at),
module_folder=MODULE_DIR)
m = OSModuleHandle.load_module(identifier=mod0.identifier,
module_path=mod0.module_path)
self.assertEqual(m.timestamp.created_at, created_at)
self.assertEqual(m.timestamp.started_at, started_at)
self.assertIsNone(m.timestamp.finished_at)
def test_datasets(self):
"""Test reading and writing modules with dataset information."""
mod0 = OSModuleHandle.create_module(
command=python_cell(source='print 2+2'),
external_form='TEST MODULE',
state=MODULE_PENDING,
outputs=ModuleOutputs(),
provenance=ModuleProvenance(write=DATASETS),
timestamp=ModuleTimestamp(),
module_folder=MODULE_DIR,
datasets=DATASETS)
m = OSModuleHandle.load_module(identifier=mod0.identifier,
module_path=mod0.module_path,
prev_state=dict())
self.assertEqual(len(m.datasets), 0)
mod0 = OSModuleHandle.create_module(
command=python_cell(source='print 2+2'),
external_form='TEST MODULE',
state=MODULE_SUCCESS,
outputs=ModuleOutputs(),
provenance=ModuleProvenance(write=DATASETS),
timestamp=ModuleTimestamp(),
module_folder=MODULE_DIR,
datasets=DATASETS)
m = OSModuleHandle.load_module(identifier=mod0.identifier,
module_path=mod0.module_path,
prev_state=dict())
self.assertEqual(len(m.datasets), 2)
self.assertEqual(m.datasets['DS1'].identifier, 'ID1')
self.assertEqual(len(m.datasets['DS1'].columns), 0)
self.assertEqual(m.datasets['DS1'].row_count, 0)
ds2 = m.datasets['DS2']
self.assertEqual(ds2.identifier, 'ID2')
self.assertEqual(len(ds2.columns), 2)
col0 = ds2.columns[0]
self.assertEqual(col0.identifier, 0)
self.assertEqual(col0.name, 'ABC')
self.assertEqual(col0.data_type, 'int')
col1 = ds2.columns[1]
self.assertEqual(col1.identifier, 1)
self.assertEqual(col1.name, 'xyz')
self.assertEqual(col1.data_type, 'real')
self.assertEqual(ds2.row_count, 100)
def test_safe_write(self):
"""Update module state with write error."""
# Create original module
module = OSModuleHandle.create_module(
command=python_cell(source='print 2+2'),
external_form='TEST MODULE',
state=MODULE_PENDING,
module_folder=MODULE_DIR,
timestamp=ModuleTimestamp(),
outputs=ModuleOutputs(stdout=[TextOutput('ABC')]),
provenance=ModuleProvenance(
read={'DS1': 'ID1'},
write={'DS1': DatasetDescriptor(identifier='ID2',
name='ID2')}))
self.assertTrue(module.is_pending)
module.set_running(external_form='TEST MODULE')
self.assertTrue(module.is_running)
module.set_success(outputs=ModuleOutputs(stderr=[None]))
self.assertTrue(module.is_error)
module = OSModuleHandle.load_module(identifier=module.identifier,
module_path=module.module_path)
self.assertTrue(module.is_running)
def test_read_write_module(self):
"""Test reading and writing modules."""
mod0 = OSModuleHandle.create_module(
command=create_plot(
dataset_name='dataset',
chart_name='My Chart',
series=[{
'column': 1,
'range': '0:50',
'label': 'A'
}, {
'column': 2,
'range': '51:100',
'label': 'B'
}, {
'column': 3,
'label': 'C'
}, {
'column': 4
}],
chart_type='bar',
chart_grouped=False,
xaxis_range='0:100',
xaxis_column=None,
),
external_form='TEST MODULE',
state=MODULE_PENDING,
outputs=ModuleOutputs(),
provenance=ModuleProvenance(),
timestamp=ModuleTimestamp(),
module_folder=MODULE_DIR,
)
m = OSModuleHandle.load_module(identifier=mod0.identifier,
module_path=mod0.module_path)
self.assertTrue(os.path.isfile(m.module_path))
self.assertEqual(m.external_form, 'TEST MODULE')
self.assertTrue(m.is_pending)
def test_outputs(self):
"""Test reading and writing modules with output information."""
mod0 = OSModuleHandle.create_module(
command=python_cell(source='print 2+2'),
external_form='TEST MODULE',
state=MODULE_PENDING,
outputs=ModuleOutputs(),
provenance=ModuleProvenance(),
timestamp=ModuleTimestamp(),
module_folder=MODULE_DIR)
m = OSModuleHandle.load_module(identifier=mod0.identifier,
module_path=mod0.module_path)
self.assertEqual(len(m.outputs.stderr), 0)
self.assertEqual(len(m.outputs.stdout), 0)
# Module with error output
mod0 = OSModuleHandle.create_module(
command=python_cell(source='print 2+2'),
external_form='TEST MODULE',
state=MODULE_PENDING,
outputs=ModuleOutputs(stderr=[TextOutput('Some text')]),
provenance=ModuleProvenance(),
timestamp=ModuleTimestamp(),
module_folder=MODULE_DIR)
m = OSModuleHandle.load_module(identifier=mod0.identifier,
module_path=mod0.module_path)
self.assertEqual(len(m.outputs.stderr), 1)
self.assertTrue(m.outputs.stderr[0].is_text)
self.assertEqual(m.outputs.stderr[0].value, 'Some text')
self.assertEqual(len(m.outputs.stdout), 0)
# Module with standard output
mod0 = OSModuleHandle.create_module(
command=python_cell(source='print 2+2'),
external_form='TEST MODULE',
state=MODULE_PENDING,
outputs=ModuleOutputs(stdout=[
TextOutput('Some text'),
OutputObject(type='chart', value='123')
]),
provenance=ModuleProvenance(),
timestamp=ModuleTimestamp(),
module_folder=MODULE_DIR)
m = OSModuleHandle.load_module(identifier=mod0.identifier,
module_path=mod0.module_path)
self.assertEqual(len(m.outputs.stdout), 2)
self.assertTrue(m.outputs.stdout[0].is_text)
self.assertEqual(m.outputs.stdout[0].value, 'Some text')
self.assertFalse(m.outputs.stdout[1].is_text)
self.assertEqual(m.outputs.stdout[1].value, '123')
self.assertEqual(len(m.outputs.stderr), 0)
# Module with standard error and standard output
mod0 = OSModuleHandle.create_module(
command=python_cell(source='print 2+2'),
external_form='TEST MODULE',
state=MODULE_PENDING,
outputs=ModuleOutputs(stderr=[TextOutput('Some text')],
stdout=[
TextOutput('Some text'),
OutputObject(type='chart', value='123')
]),
provenance=ModuleProvenance(),
timestamp=ModuleTimestamp(),
module_folder=MODULE_DIR)
m = OSModuleHandle.load_module(identifier=mod0.identifier,
module_path=mod0.module_path)
self.assertEqual(len(m.outputs.stdout), 2)
self.assertEqual(len(m.outputs.stderr), 1)
def test_provenance(self):
"""Test reading and writing modules with provenance information."""
mod0 = OSModuleHandle.create_module(
command=python_cell(source='print 2+2'),
external_form='TEST MODULE',
state=MODULE_PENDING,
outputs=ModuleOutputs(),
provenance=ModuleProvenance(),
timestamp=ModuleTimestamp(),
module_folder=MODULE_DIR,
datasets=DATASETS)
m = OSModuleHandle.load_module(identifier=mod0.identifier,
module_path=mod0.module_path)
self.assertIsNone(m.provenance.read)
self.assertIsNone(m.provenance.write)
self.assertIsNone(m.provenance.delete)
self.assertIsNone(m.provenance.resources)
# Modules that only has read provenance
mod0 = OSModuleHandle.create_module(
command=python_cell(source='print 2+2'),
external_form='TEST MODULE',
state=MODULE_PENDING,
outputs=ModuleOutputs(),
provenance=ModuleProvenance(read={'DS1': 'ID1'},
resources={'fileId': '0123456789'}),
timestamp=ModuleTimestamp(),
module_folder=MODULE_DIR)
m = OSModuleHandle.load_module(identifier=mod0.identifier,
module_path=mod0.module_path)
self.assertIsNotNone(m.provenance.read)
self.assertEqual(len(m.provenance.read), 1)
self.assertEqual(m.provenance.read['DS1'], 'ID1')
self.assertEqual(m.provenance.resources['fileId'], '0123456789')
self.assertIsNone(m.provenance.write)
# Modules that only has write provenance
mod0 = OSModuleHandle.create_module(
command=python_cell(source='print 2+2'),
external_form='TEST MODULE',
state=MODULE_PENDING,
outputs=ModuleOutputs(),
provenance=ModuleProvenance(write=DATASETS),
timestamp=ModuleTimestamp(),
module_folder=MODULE_DIR)
m = OSModuleHandle.load_module(identifier=mod0.identifier,
module_path=mod0.module_path)
self.assertIsNotNone(m.provenance.write)
self.assertEqual(len(m.provenance.write), 2)
self.assertEqual(m.provenance.write['DS1'].identifier, 'ID1')
self.assertEqual(m.provenance.write['DS2'].identifier, 'ID2')
self.assertIsNone(m.provenance.read)
# Module with read and write provenance
mod0 = OSModuleHandle.create_module(
command=python_cell(source='print 2+2'),
external_form='TEST MODULE',
state=MODULE_PENDING,
outputs=ModuleOutputs(),
provenance=ModuleProvenance(read={'DS1': 'ID1'},
write=DATASETS,
delete=['A', 'B']),
timestamp=ModuleTimestamp(),
module_folder=MODULE_DIR)
m = OSModuleHandle.load_module(identifier=mod0.identifier,
module_path=mod0.module_path)
self.assertIsNotNone(m.provenance.read)
self.assertEqual(len(m.provenance.read), 1)
self.assertEqual(m.provenance.read['DS1'], 'ID1')
self.assertIsNotNone(m.provenance.write)
self.assertEqual(len(m.provenance.write), 2)
self.assertEqual(m.provenance.write['DS1'].identifier, 'ID1')
self.assertEqual(m.provenance.write['DS2'].identifier, 'ID2')
self.assertEqual(m.provenance.delete, ['A', 'B'])
# Module with chart
chart = ChartViewHandle(identifier='A',
dataset_name='DS1',
chart_name='My Chart',
data=[
DataSeriesHandle(column='COL1',
label='SERIES1',
range_start=0,
range_end=100),
DataSeriesHandle(column='COL2',
range_start=101,
range_end=200),
DataSeriesHandle(column='COL3',
label='SERIES2')
],
x_axis=1,
chart_type='bar',
grouped_chart=True)
mod0 = OSModuleHandle.create_module(
command=python_cell(source='print 2+2'),
external_form='TEST MODULE',
state=MODULE_PENDING,
outputs=ModuleOutputs(),
provenance=ModuleProvenance(charts=[chart]),
timestamp=ModuleTimestamp(),
module_folder=MODULE_DIR)
m = OSModuleHandle.load_module(identifier=mod0.identifier,
module_path=mod0.module_path)
self.assertEqual(len(m.provenance.charts), 1)
c = m.provenance.charts[0]
self.assertEqual(chart.identifier, c.identifier)
self.assertEqual(chart.dataset_name, c.dataset_name)
self.assertEqual(chart.chart_name, c.chart_name)
self.assertEqual(chart.x_axis, c.x_axis)
self.assertEqual(chart.chart_type, c.chart_type)
self.assertEqual(chart.grouped_chart, c.grouped_chart)
self.assertEqual(len(c.data), 3)
for i in range(3):
self.assertEqual(c.data[i].column, chart.data[i].column)
self.assertEqual(c.data[i].label, chart.data[i].label)
self.assertEqual(c.data[i].range_start, chart.data[i].range_start)
self.assertEqual(c.data[i].range_end, chart.data[i].range_end)
def test_error(self):
"""Update module state from pending to error."""
# Create original module
module = OSModuleHandle.create_module(
command=python_cell(source='print 2+2'),
external_form='TEST MODULE',
state=MODULE_PENDING,
module_folder=MODULE_DIR,
outputs=ModuleOutputs(stdout=[TextOutput('ABC')]),
provenance=ModuleProvenance(
read={'DS1': 'ID1'},
write={'DS1': DatasetDescriptor(identifier='ID2', name='ID2')},
resources={'fileid': '0123456789'}),
timestamp=ModuleTimestamp())
module.set_error()
self.assertTrue(module.is_error)
self.assertIsNotNone(module.timestamp.finished_at)
self.assertEqual(len(module.outputs.stderr), 0)
self.assertEqual(len(module.outputs.stdout), 0)
self.assertIsNotNone(module.provenance.read)
self.assertIsNotNone(module.provenance.write)
self.assertIsNotNone(module.provenance.resources)
self.assertEqual(module.provenance.resources['fileid'], '0123456789')
# Read module from object store and ensure that tall changes have been
# materialized properly
module = OSModuleHandle.load_module(identifier=module.identifier,
module_path=module.module_path)
self.assertTrue(module.is_error)
self.assertIsNotNone(module.timestamp.finished_at)
self.assertEqual(len(module.outputs.stderr), 0)
self.assertEqual(len(module.outputs.stdout), 0)
self.assertIsNotNone(module.provenance.read)
self.assertIsNotNone(module.provenance.write)
self.assertIsNotNone(module.provenance.resources)
self.assertEqual(module.provenance.resources['fileid'], '0123456789')
# Set canceled with timestamp and output information
ts = get_current_time()
module.set_error(
finished_at=ts,
outputs=ModuleOutputs(stderr=[TextOutput('Some Error')]))
self.assertTrue(module.is_error)
self.assertIsNotNone(module.timestamp.finished_at)
self.assertEqual(module.timestamp.finished_at, ts)
self.assertEqual(len(module.outputs.stderr), 1)
self.assertEqual(module.outputs.stderr[0].value, 'Some Error')
self.assertEqual(len(module.outputs.stdout), 0)
self.assertIsNotNone(module.provenance.read)
self.assertIsNotNone(module.provenance.write)
self.assertIsNotNone(module.provenance.resources)
self.assertEqual(module.provenance.resources['fileid'], '0123456789')
module = OSModuleHandle.load_module(identifier=module.identifier,
module_path=module.module_path)
self.assertTrue(module.is_error)
self.assertIsNotNone(module.timestamp.finished_at)
self.assertEqual(module.timestamp.finished_at, ts)
self.assertEqual(len(module.outputs.stderr), 1)
self.assertEqual(module.outputs.stderr[0].value, 'Some Error')
self.assertEqual(len(module.outputs.stdout), 0)
self.assertIsNotNone(module.provenance.read)
self.assertIsNotNone(module.provenance.write)
self.assertIsNotNone(module.provenance.resources)
self.assertEqual(module.provenance.resources['fileid'], '0123456789')
def test_success(self) -> None:
"""Update module state from pending to success."""
# Create original module
module = OSModuleHandle.create_module(
command=python_cell(source='print 2+2'),
external_form='TEST MODULE',
state=MODULE_PENDING,
module_folder=MODULE_DIR,
timestamp=ModuleTimestamp(),
outputs=ModuleOutputs(stdout=[TextOutput('ABC')]),
provenance=ModuleProvenance(
read={'DS1': 'ID1'},
write={'DS1': DatasetDescriptor(identifier='ID2',
name='ID2')}))
self.assertTrue(module.is_pending)
module.set_running(external_form='TEST MODULE')
module.set_success()
self.assertTrue(module.is_success)
self.assertIsNotNone(module.timestamp.started_at)
self.assertIsNotNone(module.timestamp.finished_at)
self.assertEqual(len(module.outputs.stderr), 0)
self.assertEqual(len(module.outputs.stdout), 0)
self.assertTrue(module.provenance.read == {})
self.assertTrue(module.provenance.write == {})
# Read module from object store and ensure that tall changes have been
# materialized properly
module = OSModuleHandle.load_module(identifier=module.identifier,
module_path=module.module_path)
self.assertTrue(module.is_success)
self.assertIsNotNone(module.timestamp.started_at)
self.assertIsNotNone(module.timestamp.finished_at)
self.assertEqual(len(module.outputs.stderr), 0)
self.assertEqual(len(module.outputs.stdout), 0)
self.assertTrue(module.provenance.read == {})
self.assertTrue(module.provenance.write == {})
# Set success with all optional parameters
ts = get_current_time()
module.set_success(
finished_at=ts,
outputs=ModuleOutputs(stdout=[TextOutput('XYZ')]),
provenance=ModuleProvenance(
read={'DS1': 'ID1'},
write={'DS1': DatasetDescriptor(identifier='ID2',
name='ID2')}))
self.assertTrue(module.is_success)
self.assertIsNotNone(module.timestamp.started_at)
self.assertIsNotNone(module.timestamp.finished_at)
self.assertEqual(module.timestamp.finished_at, ts)
self.assertEqual(len(module.outputs.stderr), 0)
self.assertEqual(len(module.outputs.stdout), 1)
self.assertEqual(module.outputs.stdout[0].value, 'XYZ')
self.assertIsNotNone(module.provenance.read)
self.assertEqual(module.provenance.read['DS1'], 'ID1')
self.assertIsNotNone(module.provenance.write)
self.assertEqual(module.provenance.write['DS1'].identifier, 'ID2')
module = OSModuleHandle.load_module(identifier=module.identifier,
module_path=module.module_path)
module = OSModuleHandle.load_module(identifier=module.identifier,
module_path=module.module_path,
prev_state=dict())
self.assertTrue(module.is_success)
self.assertIsNotNone(module.timestamp.started_at)
self.assertIsNotNone(module.timestamp.finished_at)
self.assertEqual(module.timestamp.finished_at, ts)
self.assertEqual(len(module.outputs.stderr), 0)
self.assertEqual(len(module.outputs.stdout), 1)
self.assertEqual(module.outputs.stdout[0].value, 'XYZ')
self.assertIsNotNone(module.provenance.read)
self.assertEqual(module.provenance.read['DS1'], 'ID1')
self.assertIsNotNone(module.provenance.write)
self.assertEqual(module.provenance.write['DS1'].identifier, 'ID2')