def test_running(self): """Update module state from pending to running.""" # Create original module module = OSModuleHandle.create_module( command=python_cell(source='print 2+2'), external_form='TEST MODULE', state=MODULE_PENDING, module_folder=MODULE_DIR, timestamp=ModuleTimestamp(), datasets={'DS1': DS1}, outputs=ModuleOutputs(stdout=[TextOutput('ABC')]), provenance=ModuleProvenance( read={'DS1': 'ID1'}, write={'DS1': DatasetDescriptor(identifier='ID2')}, resources={'fileid': '0123456789'})) self.assertTrue(module.is_pending) module.set_running(external_form='TEST MODULE') self.assertTrue(module.is_running) self.assertIsNotNone(module.timestamp.started_at) self.assertEqual(len(module.datasets), 0) self.assertEqual(len(module.outputs.stderr), 0) self.assertEqual(len(module.outputs.stdout), 0) self.assertIsNotNone(module.provenance.read) self.assertIsNotNone(module.provenance.write) self.assertIsNotNone(module.provenance.resources) # Read module from object store and ensure that tall changes have been # materialized properly module = OSModuleHandle.load_module(identifier=module.identifier, module_path=module.module_path) self.assertTrue(module.is_running) self.assertIsNotNone(module.timestamp.started_at) self.assertEqual(len(module.datasets), 0) self.assertEqual(len(module.outputs.stderr), 0) self.assertEqual(len(module.outputs.stdout), 0) self.assertIsNotNone(module.provenance.read) self.assertIsNotNone(module.provenance.write) self.assertIsNotNone(module.provenance.resources) # Set running with all optional parameters module.set_running(started_at=module.timestamp.created_at, external_form='Some form') self.assertEqual(module.timestamp.started_at, module.timestamp.created_at) self.assertEqual(module.external_form, 'Some form') module = OSModuleHandle.load_module(identifier=module.identifier, module_path=module.module_path) self.assertEqual(module.timestamp.started_at, module.timestamp.created_at) self.assertEqual(module.external_form, 'Some form')
def read_workflow_modules( modules_list: List[str], modules_folder: str, object_store: ObjectStore ) -> List[ModuleHandle]: """Read workflow modules from object store. Parameters ---------- modules_list: list(string) List of module identifier modules_folder: string Path to the folder containing moudle objects object_store: vizier.core.io.base.ObjectStore Object store implementation to access and maintain resources Returns ------- list(vizier.viztrail.objectstore.module.OSModuleHandle) """ modules: List[ModuleHandle] = list() for module_id in modules_list: module_path=get_module_path( modules_folder=modules_folder, module_id=module_id, object_store=object_store ) m = OSModuleHandle.load_module( identifier=module_id, module_path=module_path, object_store=object_store ) modules.append(m) return modules
def test_timestamps(self): """Test reading and writing modules with different timestamp values.""" mod0 = OSModuleHandle.create_module( command=python_cell(source='print 2+2'), external_form='TEST MODULE', state=MODULE_PENDING, outputs=ModuleOutputs(), provenance=ModuleProvenance(), timestamp=ModuleTimestamp(), module_folder=MODULE_DIR) m = OSModuleHandle.load_module(identifier=mod0.identifier, module_path=mod0.module_path) # Test timestamps created_at = m.timestamp.created_at started_at = to_datetime('2018-11-26T13:00:00.000000') m.timestamp.started_at = started_at m.write_module() m = OSModuleHandle.load_module(identifier=mod0.identifier, module_path=mod0.module_path) self.assertEqual(m.timestamp.created_at, created_at) self.assertEqual(m.timestamp.started_at, started_at) finished_at = to_datetime('2018-11-26T13:00:00.000010') m.timestamp.created_at = finished_at m.timestamp.finished_at = finished_at m.write_module() m = OSModuleHandle.load_module(identifier=mod0.identifier, module_path=mod0.module_path) self.assertEqual(m.timestamp.created_at, finished_at) self.assertEqual(m.timestamp.started_at, started_at) self.assertEqual(m.timestamp.finished_at, finished_at) mod0 = OSModuleHandle.create_module( command=python_cell(source='print 2+2'), external_form='TEST MODULE', state=MODULE_PENDING, outputs=ModuleOutputs(), provenance=ModuleProvenance(), timestamp=ModuleTimestamp(created_at=created_at, started_at=started_at), module_folder=MODULE_DIR) m = OSModuleHandle.load_module(identifier=mod0.identifier, module_path=mod0.module_path) self.assertEqual(m.timestamp.created_at, created_at) self.assertEqual(m.timestamp.started_at, started_at) self.assertIsNone(m.timestamp.finished_at)
def test_datasets(self): """Test reading and writing modules with dataset information.""" mod0 = OSModuleHandle.create_module( command=python_cell(source='print 2+2'), external_form='TEST MODULE', state=MODULE_PENDING, outputs=ModuleOutputs(), provenance=ModuleProvenance(write=DATASETS), timestamp=ModuleTimestamp(), module_folder=MODULE_DIR, datasets=DATASETS) m = OSModuleHandle.load_module(identifier=mod0.identifier, module_path=mod0.module_path, prev_state=dict()) self.assertEqual(len(m.datasets), 0) mod0 = OSModuleHandle.create_module( command=python_cell(source='print 2+2'), external_form='TEST MODULE', state=MODULE_SUCCESS, outputs=ModuleOutputs(), provenance=ModuleProvenance(write=DATASETS), timestamp=ModuleTimestamp(), module_folder=MODULE_DIR, datasets=DATASETS) m = OSModuleHandle.load_module(identifier=mod0.identifier, module_path=mod0.module_path, prev_state=dict()) self.assertEqual(len(m.datasets), 2) self.assertEqual(m.datasets['DS1'].identifier, 'ID1') self.assertEqual(len(m.datasets['DS1'].columns), 0) self.assertEqual(m.datasets['DS1'].row_count, 0) ds2 = m.datasets['DS2'] self.assertEqual(ds2.identifier, 'ID2') self.assertEqual(len(ds2.columns), 2) col0 = ds2.columns[0] self.assertEqual(col0.identifier, 0) self.assertEqual(col0.name, 'ABC') self.assertEqual(col0.data_type, 'int') col1 = ds2.columns[1] self.assertEqual(col1.identifier, 1) self.assertEqual(col1.name, 'xyz') self.assertEqual(col1.data_type, 'real') self.assertEqual(ds2.row_count, 100)
def test_safe_write(self): """Update module state with write error.""" # Create original module module = OSModuleHandle.create_module( command=python_cell(source='print 2+2'), external_form='TEST MODULE', state=MODULE_PENDING, module_folder=MODULE_DIR, timestamp=ModuleTimestamp(), outputs=ModuleOutputs(stdout=[TextOutput('ABC')]), provenance=ModuleProvenance( read={'DS1': 'ID1'}, write={'DS1': DatasetDescriptor(identifier='ID2', name='ID2')})) self.assertTrue(module.is_pending) module.set_running(external_form='TEST MODULE') self.assertTrue(module.is_running) module.set_success(outputs=ModuleOutputs(stderr=[None])) self.assertTrue(module.is_error) module = OSModuleHandle.load_module(identifier=module.identifier, module_path=module.module_path) self.assertTrue(module.is_running)
def test_read_write_module(self): """Test reading and writing modules.""" mod0 = OSModuleHandle.create_module( command=create_plot( dataset_name='dataset', chart_name='My Chart', series=[{ 'column': 1, 'range': '0:50', 'label': 'A' }, { 'column': 2, 'range': '51:100', 'label': 'B' }, { 'column': 3, 'label': 'C' }, { 'column': 4 }], chart_type='bar', chart_grouped=False, xaxis_range='0:100', xaxis_column=None, ), external_form='TEST MODULE', state=MODULE_PENDING, outputs=ModuleOutputs(), provenance=ModuleProvenance(), timestamp=ModuleTimestamp(), module_folder=MODULE_DIR, ) m = OSModuleHandle.load_module(identifier=mod0.identifier, module_path=mod0.module_path) self.assertTrue(os.path.isfile(m.module_path)) self.assertEqual(m.external_form, 'TEST MODULE') self.assertTrue(m.is_pending)
def test_outputs(self): """Test reading and writing modules with output information.""" mod0 = OSModuleHandle.create_module( command=python_cell(source='print 2+2'), external_form='TEST MODULE', state=MODULE_PENDING, outputs=ModuleOutputs(), provenance=ModuleProvenance(), timestamp=ModuleTimestamp(), module_folder=MODULE_DIR) m = OSModuleHandle.load_module(identifier=mod0.identifier, module_path=mod0.module_path) self.assertEqual(len(m.outputs.stderr), 0) self.assertEqual(len(m.outputs.stdout), 0) # Module with error output mod0 = OSModuleHandle.create_module( command=python_cell(source='print 2+2'), external_form='TEST MODULE', state=MODULE_PENDING, outputs=ModuleOutputs(stderr=[TextOutput('Some text')]), provenance=ModuleProvenance(), timestamp=ModuleTimestamp(), module_folder=MODULE_DIR) m = OSModuleHandle.load_module(identifier=mod0.identifier, module_path=mod0.module_path) self.assertEqual(len(m.outputs.stderr), 1) self.assertTrue(m.outputs.stderr[0].is_text) self.assertEqual(m.outputs.stderr[0].value, 'Some text') self.assertEqual(len(m.outputs.stdout), 0) # Module with standard output mod0 = OSModuleHandle.create_module( command=python_cell(source='print 2+2'), external_form='TEST MODULE', state=MODULE_PENDING, outputs=ModuleOutputs(stdout=[ TextOutput('Some text'), OutputObject(type='chart', value='123') ]), provenance=ModuleProvenance(), timestamp=ModuleTimestamp(), module_folder=MODULE_DIR) m = OSModuleHandle.load_module(identifier=mod0.identifier, module_path=mod0.module_path) self.assertEqual(len(m.outputs.stdout), 2) self.assertTrue(m.outputs.stdout[0].is_text) self.assertEqual(m.outputs.stdout[0].value, 'Some text') self.assertFalse(m.outputs.stdout[1].is_text) self.assertEqual(m.outputs.stdout[1].value, '123') self.assertEqual(len(m.outputs.stderr), 0) # Module with standard error and standard output mod0 = OSModuleHandle.create_module( command=python_cell(source='print 2+2'), external_form='TEST MODULE', state=MODULE_PENDING, outputs=ModuleOutputs(stderr=[TextOutput('Some text')], stdout=[ TextOutput('Some text'), OutputObject(type='chart', value='123') ]), provenance=ModuleProvenance(), timestamp=ModuleTimestamp(), module_folder=MODULE_DIR) m = OSModuleHandle.load_module(identifier=mod0.identifier, module_path=mod0.module_path) self.assertEqual(len(m.outputs.stdout), 2) self.assertEqual(len(m.outputs.stderr), 1)
def test_provenance(self): """Test reading and writing modules with provenance information.""" mod0 = OSModuleHandle.create_module( command=python_cell(source='print 2+2'), external_form='TEST MODULE', state=MODULE_PENDING, outputs=ModuleOutputs(), provenance=ModuleProvenance(), timestamp=ModuleTimestamp(), module_folder=MODULE_DIR, datasets=DATASETS) m = OSModuleHandle.load_module(identifier=mod0.identifier, module_path=mod0.module_path) self.assertIsNone(m.provenance.read) self.assertIsNone(m.provenance.write) self.assertIsNone(m.provenance.delete) self.assertIsNone(m.provenance.resources) # Modules that only has read provenance mod0 = OSModuleHandle.create_module( command=python_cell(source='print 2+2'), external_form='TEST MODULE', state=MODULE_PENDING, outputs=ModuleOutputs(), provenance=ModuleProvenance(read={'DS1': 'ID1'}, resources={'fileId': '0123456789'}), timestamp=ModuleTimestamp(), module_folder=MODULE_DIR) m = OSModuleHandle.load_module(identifier=mod0.identifier, module_path=mod0.module_path) self.assertIsNotNone(m.provenance.read) self.assertEqual(len(m.provenance.read), 1) self.assertEqual(m.provenance.read['DS1'], 'ID1') self.assertEqual(m.provenance.resources['fileId'], '0123456789') self.assertIsNone(m.provenance.write) # Modules that only has write provenance mod0 = OSModuleHandle.create_module( command=python_cell(source='print 2+2'), external_form='TEST MODULE', state=MODULE_PENDING, outputs=ModuleOutputs(), provenance=ModuleProvenance(write=DATASETS), timestamp=ModuleTimestamp(), module_folder=MODULE_DIR) m = OSModuleHandle.load_module(identifier=mod0.identifier, module_path=mod0.module_path) self.assertIsNotNone(m.provenance.write) self.assertEqual(len(m.provenance.write), 2) self.assertEqual(m.provenance.write['DS1'].identifier, 'ID1') self.assertEqual(m.provenance.write['DS2'].identifier, 'ID2') self.assertIsNone(m.provenance.read) # Module with read and write provenance mod0 = OSModuleHandle.create_module( command=python_cell(source='print 2+2'), external_form='TEST MODULE', state=MODULE_PENDING, outputs=ModuleOutputs(), provenance=ModuleProvenance(read={'DS1': 'ID1'}, write=DATASETS, delete=['A', 'B']), timestamp=ModuleTimestamp(), module_folder=MODULE_DIR) m = OSModuleHandle.load_module(identifier=mod0.identifier, module_path=mod0.module_path) self.assertIsNotNone(m.provenance.read) self.assertEqual(len(m.provenance.read), 1) self.assertEqual(m.provenance.read['DS1'], 'ID1') self.assertIsNotNone(m.provenance.write) self.assertEqual(len(m.provenance.write), 2) self.assertEqual(m.provenance.write['DS1'].identifier, 'ID1') self.assertEqual(m.provenance.write['DS2'].identifier, 'ID2') self.assertEqual(m.provenance.delete, ['A', 'B']) # Module with chart chart = ChartViewHandle(identifier='A', dataset_name='DS1', chart_name='My Chart', data=[ DataSeriesHandle(column='COL1', label='SERIES1', range_start=0, range_end=100), DataSeriesHandle(column='COL2', range_start=101, range_end=200), DataSeriesHandle(column='COL3', label='SERIES2') ], x_axis=1, chart_type='bar', grouped_chart=True) mod0 = OSModuleHandle.create_module( command=python_cell(source='print 2+2'), external_form='TEST MODULE', state=MODULE_PENDING, outputs=ModuleOutputs(), provenance=ModuleProvenance(charts=[chart]), timestamp=ModuleTimestamp(), module_folder=MODULE_DIR) m = OSModuleHandle.load_module(identifier=mod0.identifier, module_path=mod0.module_path) self.assertEqual(len(m.provenance.charts), 1) c = m.provenance.charts[0] self.assertEqual(chart.identifier, c.identifier) self.assertEqual(chart.dataset_name, c.dataset_name) self.assertEqual(chart.chart_name, c.chart_name) self.assertEqual(chart.x_axis, c.x_axis) self.assertEqual(chart.chart_type, c.chart_type) self.assertEqual(chart.grouped_chart, c.grouped_chart) self.assertEqual(len(c.data), 3) for i in range(3): self.assertEqual(c.data[i].column, chart.data[i].column) self.assertEqual(c.data[i].label, chart.data[i].label) self.assertEqual(c.data[i].range_start, chart.data[i].range_start) self.assertEqual(c.data[i].range_end, chart.data[i].range_end)
def test_error(self): """Update module state from pending to error.""" # Create original module module = OSModuleHandle.create_module( command=python_cell(source='print 2+2'), external_form='TEST MODULE', state=MODULE_PENDING, module_folder=MODULE_DIR, outputs=ModuleOutputs(stdout=[TextOutput('ABC')]), provenance=ModuleProvenance( read={'DS1': 'ID1'}, write={'DS1': DatasetDescriptor(identifier='ID2', name='ID2')}, resources={'fileid': '0123456789'}), timestamp=ModuleTimestamp()) module.set_error() self.assertTrue(module.is_error) self.assertIsNotNone(module.timestamp.finished_at) self.assertEqual(len(module.outputs.stderr), 0) self.assertEqual(len(module.outputs.stdout), 0) self.assertIsNotNone(module.provenance.read) self.assertIsNotNone(module.provenance.write) self.assertIsNotNone(module.provenance.resources) self.assertEqual(module.provenance.resources['fileid'], '0123456789') # Read module from object store and ensure that tall changes have been # materialized properly module = OSModuleHandle.load_module(identifier=module.identifier, module_path=module.module_path) self.assertTrue(module.is_error) self.assertIsNotNone(module.timestamp.finished_at) self.assertEqual(len(module.outputs.stderr), 0) self.assertEqual(len(module.outputs.stdout), 0) self.assertIsNotNone(module.provenance.read) self.assertIsNotNone(module.provenance.write) self.assertIsNotNone(module.provenance.resources) self.assertEqual(module.provenance.resources['fileid'], '0123456789') # Set canceled with timestamp and output information ts = get_current_time() module.set_error( finished_at=ts, outputs=ModuleOutputs(stderr=[TextOutput('Some Error')])) self.assertTrue(module.is_error) self.assertIsNotNone(module.timestamp.finished_at) self.assertEqual(module.timestamp.finished_at, ts) self.assertEqual(len(module.outputs.stderr), 1) self.assertEqual(module.outputs.stderr[0].value, 'Some Error') self.assertEqual(len(module.outputs.stdout), 0) self.assertIsNotNone(module.provenance.read) self.assertIsNotNone(module.provenance.write) self.assertIsNotNone(module.provenance.resources) self.assertEqual(module.provenance.resources['fileid'], '0123456789') module = OSModuleHandle.load_module(identifier=module.identifier, module_path=module.module_path) self.assertTrue(module.is_error) self.assertIsNotNone(module.timestamp.finished_at) self.assertEqual(module.timestamp.finished_at, ts) self.assertEqual(len(module.outputs.stderr), 1) self.assertEqual(module.outputs.stderr[0].value, 'Some Error') self.assertEqual(len(module.outputs.stdout), 0) self.assertIsNotNone(module.provenance.read) self.assertIsNotNone(module.provenance.write) self.assertIsNotNone(module.provenance.resources) self.assertEqual(module.provenance.resources['fileid'], '0123456789')
def test_success(self) -> None: """Update module state from pending to success.""" # Create original module module = OSModuleHandle.create_module( command=python_cell(source='print 2+2'), external_form='TEST MODULE', state=MODULE_PENDING, module_folder=MODULE_DIR, timestamp=ModuleTimestamp(), outputs=ModuleOutputs(stdout=[TextOutput('ABC')]), provenance=ModuleProvenance( read={'DS1': 'ID1'}, write={'DS1': DatasetDescriptor(identifier='ID2', name='ID2')})) self.assertTrue(module.is_pending) module.set_running(external_form='TEST MODULE') module.set_success() self.assertTrue(module.is_success) self.assertIsNotNone(module.timestamp.started_at) self.assertIsNotNone(module.timestamp.finished_at) self.assertEqual(len(module.outputs.stderr), 0) self.assertEqual(len(module.outputs.stdout), 0) self.assertTrue(module.provenance.read == {}) self.assertTrue(module.provenance.write == {}) # Read module from object store and ensure that tall changes have been # materialized properly module = OSModuleHandle.load_module(identifier=module.identifier, module_path=module.module_path) self.assertTrue(module.is_success) self.assertIsNotNone(module.timestamp.started_at) self.assertIsNotNone(module.timestamp.finished_at) self.assertEqual(len(module.outputs.stderr), 0) self.assertEqual(len(module.outputs.stdout), 0) self.assertTrue(module.provenance.read == {}) self.assertTrue(module.provenance.write == {}) # Set success with all optional parameters ts = get_current_time() module.set_success( finished_at=ts, outputs=ModuleOutputs(stdout=[TextOutput('XYZ')]), provenance=ModuleProvenance( read={'DS1': 'ID1'}, write={'DS1': DatasetDescriptor(identifier='ID2', name='ID2')})) self.assertTrue(module.is_success) self.assertIsNotNone(module.timestamp.started_at) self.assertIsNotNone(module.timestamp.finished_at) self.assertEqual(module.timestamp.finished_at, ts) self.assertEqual(len(module.outputs.stderr), 0) self.assertEqual(len(module.outputs.stdout), 1) self.assertEqual(module.outputs.stdout[0].value, 'XYZ') self.assertIsNotNone(module.provenance.read) self.assertEqual(module.provenance.read['DS1'], 'ID1') self.assertIsNotNone(module.provenance.write) self.assertEqual(module.provenance.write['DS1'].identifier, 'ID2') module = OSModuleHandle.load_module(identifier=module.identifier, module_path=module.module_path) module = OSModuleHandle.load_module(identifier=module.identifier, module_path=module.module_path, prev_state=dict()) self.assertTrue(module.is_success) self.assertIsNotNone(module.timestamp.started_at) self.assertIsNotNone(module.timestamp.finished_at) self.assertEqual(module.timestamp.finished_at, ts) self.assertEqual(len(module.outputs.stderr), 0) self.assertEqual(len(module.outputs.stdout), 1) self.assertEqual(module.outputs.stdout[0].value, 'XYZ') self.assertIsNotNone(module.provenance.read) self.assertEqual(module.provenance.read['DS1'], 'ID1') self.assertIsNotNone(module.provenance.write) self.assertEqual(module.provenance.write['DS1'].identifier, 'ID2')