def test_capture_failure(test_config): start_s = datetime.utcnow().timestamp() test_obs_id = 'test_obs_id' test_obs_id_2 = 'test_obs_id_2' log_file_directory = os.path.join(tc.THIS_DIR, 'logs') test_config.log_to_file = True test_config.log_file_directory = log_file_directory success_log_file_name = 'success_log.txt' test_config.success_log_file_name = success_log_file_name failure_log_file_name = 'failure_log.txt' test_config.failure_log_file_name = failure_log_file_name retry_file_name = 'retries.txt' test_config.retry_file_name = retry_file_name rejected_file_name = 'rejected.yml' test_config.rejected_file_name = rejected_file_name # clean up from last execution if not os.path.exists(log_file_directory): os.mkdir(log_file_directory) if os.path.exists(test_config.success_fqn): os.remove(test_config.success_fqn) if os.path.exists(test_config.failure_fqn): os.remove(test_config.failure_fqn) if os.path.exists(test_config.retry_fqn): os.remove(test_config.retry_fqn) if os.path.exists(test_config.rejected_fqn): os.remove(test_config.rejected_fqn) test_oe = ec.OrganizeExecutesWithDoOne(test_config, 'command', [], []) test_sname = tc.TestStorageName(obs_id=test_obs_id_2) test_oe.capture_failure(test_sname, 'Cannot build an observation') test_sname = tc.TestStorageName(obs_id=test_obs_id) test_oe.capture_failure(test_sname, 'exception text') test_oe.capture_success(test_obs_id, 'C121212_01234_CAL.fits.gz', start_s) test_oe.finish_run(test_config) assert os.path.exists(test_config.success_fqn) assert os.path.exists(test_config.failure_fqn) assert os.path.exists(test_config.retry_fqn) assert os.path.exists(test_config.rejected_fqn) success_content = open(test_config.success_fqn).read() assert ('test_obs_id C121212_01234_CAL.fits.gz' in success_content), 'wrong content' retry_content = open(test_config.retry_fqn).read() assert retry_content == 'test_obs_id\n' failure_content = open(test_config.failure_fqn).read() assert failure_content.endswith( 'Unknown error. Check specific log.\n'), failure_content assert os.path.exists(test_config.rejected_fqn), test_config.rejected_fqn rejected_content = mc.read_as_yaml(test_config.rejected_fqn) assert rejected_content is not None, 'expect a result' test_result = rejected_content.get('bad_metadata') assert test_result is not None, 'wrong result' assert len(test_result) == 1, 'wrong number of entries' assert test_result[0] == test_obs_id, 'wrong entry'
def test_do_one(test_config): test_config.task_types = [] test_organizer = ec.OrganizeExecutes(test_config, 'test2caom2', [], []) # no client test_result = test_organizer.do_one(tc.TestStorageName()) assert test_result is not None assert test_result == -1 # client test_config.features.use_clients = True test_result = test_organizer.do_one(tc.TestStorageName()) assert test_result is not None assert test_result == -1
def test_data_execute_v(test_config): test_config.features.supports_latest_client = True test_obs_id = 'test_obs_id' test_dir = os.path.join(tc.THIS_DIR, test_obs_id) test_fits_fqn = os.path.join(test_dir, tc.TestStorageName().file_name) try: if not os.path.exists(test_dir): os.mkdir(test_dir, mode=0o755) # precondition = open(test_fits_fqn, 'w') # precondition.close() test_data_visitors = [TestVisit] repo_client_mock = Mock(autospec=True) cadc_client_mock = Mock(autospec=True) cadc_client_mock.copy.side_effect = tc.mock_copy_md5 test_observer = Mock(autospec=True) test_cred = '' test_transferrer = transfer_composable.VoTransfer() test_transferrer.cadc_client = cadc_client_mock ec.CaomExecute._data_cmd_info = Mock(side_effect=_get_fname) repo_client_mock.read.side_effect = tc.mock_read # run the test test_executor = ec.DataVisit( test_config, tc.TestStorageName(), test_cred, cadc_client_mock, repo_client_mock, test_data_visitors, mc.TaskType.MODIFY, test_observer, test_transferrer, ) test_executor.execute(None) # check that things worked as expected assert repo_client_mock.read.called, 'read call missed' assert repo_client_mock.update.called, 'update call missed' assert test_observer.metrics.observe.called, 'observe not called' assert cadc_client_mock.copy.called, 'copy not called' cadc_client_mock.copy.assert_called_with( 'ad:TEST/test_obs_id.fits.gz', test_fits_fqn, send_md5=True ), 'wrong call args' finally: if os.path.exists(test_fits_fqn): os.unlink(test_fits_fqn) if os.path.exists(test_dir): os.rmdir(test_dir)
def test_store(test_config): test_config.working_directory = tc.TEST_DATA_DIR test_sn = tc.TestStorageName() test_sn.source_names = ['vos:goliaths/nonexistent.fits.gz'] test_command = 'collection2caom2' test_data_client = Mock(autospec=True) test_observable = Mock(autospec=True) test_transferrer = Mock(autospec=True) test_transferrer.get.side_effect = _transfer_get_mock test_subject = ec.Store( test_config, test_sn, test_command, test_data_client, test_observable, test_transferrer, ) assert test_subject is not None, 'expect construction' assert test_subject.working_dir == os.path.join( tc.TEST_DATA_DIR, 'test_obs_id'), 'wrong working directory' assert (len( test_subject._storage_name.destination_uris) == 1), 'wrong file count' assert (test_subject._storage_name.destination_uris[0] == 'cadc:TEST/test_file.fits.gz'), 'wrong destination' test_subject.execute(None) assert test_data_client.put.called, 'data put not called' test_data_client.put.assert_called_with( '/usr/src/app/caom2pipe/caom2pipe/tests/data/test_obs_id', 'cadc:TEST/test_file.fits.gz', 'TEST', ), 'wrong put call args'
def test_choose_exceptions(test_config): test_config.init_local_files = False test_config.task_types = [mc.TaskType.SCRAPE] with pytest.raises(mc.CadcException): test_organizer = ec.OrganizeExecutes(test_config, 'command name', [], []) test_organizer.choose(tc.TestStorageName())
def test_data_store(test_config): test_dir = f'{tc.TEST_DATA_DIR}/test_obs_id' if os.path.exists(test_dir): os.rmdir(test_dir) data_client_mock = Mock() test_observer = Mock() # stat mock is for CadcDataClient stat_orig = os.stat os.stat = Mock() os.stat.st_size.return_value = 1243 # path.exists mock is because the stat mock causes os.path.exists to # return True path_orig = os.path.exists os.path.exists = Mock(return_value=False) test_config.features.supports_multiple_files = False try: test_config.working_directory = tc.TEST_DATA_DIR test_executor = ec.Store( test_config, tc.TestStorageName(entry=f'{tc.TEST_DATA_DIR}/test_file.fits.gz'), 'command_name', data_client_mock, observable=test_observer, transferrer=transfer_composable.Transfer(), ) test_executor.execute(None) # check that things worked as expected - no cleanup assert data_client_mock.put.called, 'put_file call missed' finally: os.stat = stat_orig os.path.exists = path_orig
def test_data_local_execute(test_config): # ensure the model uses the log directory for writing the model file test_config.log_to_file = True test_config.working_directory = tc.TEST_DATA_DIR test_data_visitors = [LocalTestVisit] data_client_mock = Mock() repo_client_mock = Mock() repo_client_mock.read.return_value = _read_obs(None) test_observer = Mock() test_model_fqn = os.path.join(tc.TEST_DATA_DIR, 'test_obs_id.xml') # check that a file is written to disk if os.path.exists(test_model_fqn): os.unlink(test_model_fqn) # run the test test_executor = ec.LocalDataVisit( test_config, tc.TestStorageName(entry=f'{tc.TEST_DATA_DIR}/test_file.fits.gz'), data_client_mock, repo_client_mock, test_data_visitors, observable=test_observer, ) test_executor.execute(None) # check that things worked as expected - no cleanup assert repo_client_mock.read.called, 'read call missed' repo_client_mock.read.assert_called_with( 'OMM', 'test_obs_id'), 'wrong repo client read args' assert repo_client_mock.update.called, 'update call missed' assert test_observer.metrics.observe.called, 'observe not called' assert os.path.exists(test_model_fqn), 'observation not written to disk'
def test_scrape(test_config): # clean up from previous tests if os.path.exists(tc.TestStorageName().model_file_name): os.remove(tc.TestStorageName().model_file_name) netrc = os.path.join(tc.TEST_DATA_DIR, 'test_netrc') assert os.path.exists(netrc) test_config.working_directory = tc.TEST_DATA_DIR test_config.logging_level = 'INFO' test_executor = ec.Scrape( test_config, tc.TestStorageName(), __name__, observable=None, meta_visitors=[], ) test_executor.execute(None)
def test_local_store(test_config): test_config.working_directory = tc.TEST_DATA_DIR test_config.data_source = ['/test_files/caom2pipe'] test_config.use_local_files = True test_config.store_newer_files_only = False test_config.features.supports_latest_client = False test_sn = tc.TestStorageName(entry=f'{tc.TEST_DATA_DIR}/test_file.fits.gz') if not os.path.exists(test_sn.source_names[0]): with open(test_sn.source_names[0], 'w') as f: f.write('test content') test_command = 'collection2caom2' test_data_client = Mock(autospec=True) test_observable = Mock(autospec=True) test_subject = ec.LocalStore(test_config, test_sn, test_command, test_data_client, test_observable) assert test_subject is not None, 'expect construction' test_subject.execute(None) # does the working directory get used if it's just a local store? assert test_subject.working_dir == os.path.join( tc.TEST_DATA_DIR, 'test_obs_id'), 'wrong working directory' # do one file at a time, so it doesn't matter how many files are # in the working directory assert (len( test_subject._storage_name.destination_uris) == 1), 'wrong file count' assert (test_subject._storage_name.destination_uris[0] == 'cadc:TEST/test_file.fits.gz'), 'wrong destination' assert test_data_client.put.called, 'data put not called' assert (test_data_client.put.call_args.args[0] == tc.TEST_DATA_DIR ), 'working directory' assert (test_data_client.put.call_args.args[1] == test_sn.destination_uris[0]), 'expect a file name'
def test_data_local_execute(test_config): test_data_visitors = [TestVisit] data_client_mock = Mock() data_client_mock.get_file_info.return_value = {'name': 'test_file.fits'} repo_client_mock = Mock() repo_client_mock.read.return_value = _read_obs(None) test_cred = None test_observer = Mock() # run the test test_executor = ec.LocalDataVisit( test_config, tc.TestStorageName(), test_cred, data_client_mock, repo_client_mock, test_data_visitors, observable=test_observer, ) test_executor.execute(None) # check that things worked as expected - no cleanup assert repo_client_mock.read.called, 'read call missed' assert repo_client_mock.update.called, 'update call missed' assert test_observer.metrics.observe.called, 'observe not called'
def test_meta_create_client_execute_failed_update( headers_mock, f2c2_data_client_mock, test_config ): test_cred = '' data_client_mock = Mock() data_client_mock.get_file_info.return_value = {'name': 'test_file.fits'} repo_client_mock = Mock() test_observer = Mock() read_obs_orig = mc.read_obs_from_file mc.read_obs_from_file = Mock() mc.read_obs_from_file.return_value = _read_obs(None) f2c2_data_client_mock.return_value.get_file_info.side_effect = \ _mock_get_file_info headers_mock.side_effect = _get_headers test_executor = ec.MetaCreate( test_config, tc.TestStorageName(), 'failedUpdateCollection2caom2', test_cred, data_client_mock, repo_client_mock, meta_visitors=None, observable=test_observer, ) try: with pytest.raises(mc.CadcException): test_executor.execute(None) assert not repo_client_mock.create.called, 'should have no create call' assert test_executor.url == 'https://test_url/', 'url' finally: mc.read_obs_from_file = read_obs_orig
def test_local_meta_create_client_execute(test_config): test_dir = os.path.join(tc.THIS_DIR, 'test_obs_id') test_f_fqn = os.path.join(test_dir, 'test_obs_id.fits.xml') if os.path.exists(test_f_fqn): os.unlink(test_f_fqn) os.rmdir(test_dir) test_cred = '' data_client_mock = Mock() data_client_mock.get_file_info.return_value = {'name': 'test_file.fits'} repo_client_mock = Mock() test_observer = Mock() test_config.logging_level = 'INFO' test_executor = ec.LocalMetaCreate( test_config, tc.TestStorageName(entry=f'{tc.TEST_DATA_DIR}/test_file.fits.gz'), __name__, test_cred, data_client_mock, repo_client_mock, meta_visitors=None, observable=test_observer, ) test_executor.execute(None) assert repo_client_mock.create.called, 'create call missed' assert test_observer.metrics.observe.called, 'observe not called'
def test_meta_create_client_execute(test_config): test_cred = '' data_client_mock = Mock() data_client_mock.get_file_info.return_value = {'name': 'test_file.fits'} repo_client_mock = Mock() test_observer = Mock() read_obs_orig = mc.read_obs_from_file mc.read_obs_from_file = Mock() mc.read_obs_from_file.return_value = _read_obs(None) test_executor = ec.MetaCreate( test_config, tc.TestStorageName(), TEST_APP, test_cred, data_client_mock, repo_client_mock, meta_visitors=None, observable=test_observer, ) try: test_executor.execute(None) assert repo_client_mock.create.called, 'create call missed' assert test_executor.url == 'https://test_url/', 'url' assert test_observer.metrics.observe.called, 'observe not called' finally: mc.read_obs_from_file = read_obs_orig
def test_meta_update_observation_direct(test_config): test_cred_param = '--cert /usr/src/app/cadcproxy.pem' data_client_mock = Mock() repo_client_mock = Mock() test_sn = tc.TestStorageName() test_observation = mc.read_obs_from_file( f'{tc.TEST_DATA_DIR}/fpf_start_obs.xml' ) try: test_observable = mc.Observable( mc.Rejected(test_config.rejected_fqn), mc.Metrics(test_config) ) test_executor = ec.MetaUpdateObservation( test_config, test_sn, TEST_APP, test_cred_param, data_client_mock, repo_client_mock, test_observation, [], observable=test_observable, ) test_executor.execute(None) assert repo_client_mock.update.called, 'repo update called' finally: pass
def test_data_execute(test_config): test_obs_id = 'test_obs_id' test_dir = os.path.join(tc.THIS_DIR, test_obs_id) test_fits_fqn = os.path.join(test_dir, tc.TestStorageName().file_name) try: if not os.path.exists(test_dir): os.mkdir(test_dir, mode=0o755) precondition = open(test_fits_fqn, 'w') precondition.close() logging.error(test_fits_fqn) test_data_visitors = [TestVisit] repo_client_mock = Mock() data_client_mock = Mock() test_observer = Mock() test_cred = '' test_transferrer = transfer_composable.VoTransfer() test_transferrer.cadc_client = data_client_mock ec.CaomExecute._data_cmd_info = Mock(side_effect=_get_fname) repo_client_mock.read.side_effect = tc.mock_read # run the test test_executor = ec.DataVisit( test_config, tc.TestStorageName(), test_cred, data_client_mock, repo_client_mock, test_data_visitors, mc.TaskType.MODIFY, test_observer, test_transferrer, ) test_executor.execute(None) # check that things worked as expected assert repo_client_mock.read.called, 'read call missed' assert repo_client_mock.update.called, 'update call missed' assert test_observer.metrics.observe.called, 'observe not called' finally: if os.path.exists(test_fits_fqn): os.unlink(test_fits_fqn) if os.path.exists(test_dir): os.rmdir(test_dir)
def test_store(test_config): test_config.working_directory = '/test_files/caom2pipe' test_sn = tc.TestStorageName() test_command = 'collection2caom2' test_cred_param = '' test_data_client = Mock(autospec=True) test_repo_client = Mock(autospec=True) test_observable = Mock(autospec=True) test_transferrer = Mock(autospec=True) test_transferrer.get.side_effect = _transfer_get_mock test_subject = ec.Store( test_config, test_sn, test_command, test_cred_param, test_data_client, test_repo_client, test_observable, test_transferrer, ) assert test_subject is not None, 'expect construction' assert ( test_subject.working_dir == '/test_files/caom2pipe/test_obs_id' ), 'wrong working directory' assert len(test_subject.multiple_files) == 1, 'wrong file count' assert ( len(test_subject._destination_f_names) == 1 ), 'wrong destination file count' assert ( test_subject._destination_f_names[0] == 'test_obs_id.fits' ), 'wrong destination' test_subject.execute(None) assert test_data_client.put_file.called, 'data put not called' assert ( test_data_client.put_file.call_args.args[0] is None ), 'archive not set for test_config' assert ( test_data_client.put_file.call_args.args[1] == 'test_obs_id.fits' ), 'expect a file name' assert ( test_data_client.put_file.call_args.kwargs['archive_stream'] == 'TEST' ), 'wrong archive stream' assert ( test_data_client.put_file.call_args.kwargs['mime_type'] == 'application/fits' ), 'wrong archive' assert ( test_data_client.put_file.call_args.kwargs['mime_encoding'] is None ), 'wrong archive' assert ( test_data_client.put_file.call_args.kwargs['md5_check'] is True ), 'wrong archive'
def test_data_scrape_execute(read_obs_mock, test_config): test_config.log_to_file = True test_data_visitors = [TestVisit] read_obs_mock.side_effect = _read_obs # run the test test_executor = ec.DataScrape( test_config, tc.TestStorageName(), test_data_visitors, observable=None, ) test_executor.execute(None) assert read_obs_mock.called, 'read obs call missed'
def test_organize_executes_chooser(test_config): test_obs_id = tc.TestStorageName() test_config.use_local_files = True log_file_directory = os.path.join(tc.THIS_DIR, 'logs') test_config.log_file_directory = log_file_directory test_config.features.supports_composite = True caom_client = Mock(autospec=True) caom_client.read.side_effect = _read_obs2 test_config.task_types = [mc.TaskType.INGEST] test_chooser = tc.TestChooser() test_oe = ec.OrganizeExecutes( test_config, 'command_name', [], [], test_chooser, cadc_client=Mock(autospec=True, return_value=None), caom_client=caom_client, ) executors = test_oe.choose(test_obs_id) assert executors is not None assert len(executors) == 1 assert isinstance(executors[0], ec.LocalMetaDeleteCreate) assert executors[0].stream == 'TEST', 'stream' assert executors[0].working_dir == os.path.join( tc.THIS_DIR, 'test_obs_id'), 'working_dir' assert caom_client.read.called, 'read should be called' caom_client.read.reset() test_config.use_local_files = False test_config.task_types = [mc.TaskType.INGEST] test_oe = ec.OrganizeExecutes( test_config, 'command_name', [], [], test_chooser, cadc_client=Mock(autospec=True, return_value=None), caom_client=caom_client, ) executors = test_oe.choose(test_obs_id) assert executors is not None assert len(executors) == 1 assert isinstance(executors[0], ec.MetaDeleteCreate) assert caom_client.read.called, 'read should be called'
def test_data_scrape_execute(test_config): test_data_visitors = [TestVisit] read_orig = mc.read_obs_from_file mc.read_obs_from_file = Mock(side_effect=_read_obs) try: # run the test test_executor = ec.DataScrape( test_config, tc.TestStorageName(), test_data_visitors, observable=None, ) test_executor.execute(None) assert mc.read_obs_from_file.called, 'read obs call missed' finally: mc.read_obs_from_file = read_orig
def test_local_meta_update_client_execute(test_config): test_cred = '' data_client_mock = Mock() data_client_mock.get_file_info.return_value = {'name': 'test_file.fits'} repo_client_mock = Mock() test_observer = Mock() test_executor = ec.LocalMetaUpdate( test_config, tc.TestStorageName(), TEST_APP, test_cred, data_client_mock, repo_client_mock, _read_obs(None), meta_visitors=None, observable=test_observer, ) test_executor.execute(None) assert repo_client_mock.update.called, 'update call missed' assert test_observer.metrics.observe.called, 'observe not called'
def test_organize_executes_client_visit(test_config): test_obs_id = tc.TestStorageName() test_config.features.use_clients = True test_config.task_types = [mc.TaskType.VISIT] test_config.use_local_files = False repo_client_mock = Mock(autospec=True) repo_client_mock.read.side_effect = _read_obs2 test_oe = ec.OrganizeExecutes( test_config, 'command_name', [], [], cadc_client=Mock(autospec=True), caom_client=repo_client_mock, ) executors = test_oe.choose(test_obs_id) assert executors is not None assert len(executors) == 1 assert isinstance(executors[0], ec.MetaVisit) assert not repo_client_mock.read.called, 'mock should not be called?'
def test_local_meta_delete_create_client_execute(test_config): test_cred = '' data_client_mock = Mock() data_client_mock.get_file_info.return_value = {'name': 'test_file.fits'} repo_client_mock = Mock() test_observer = Mock() test_executor = ec.LocalMetaDeleteCreate( test_config, tc.TestStorageName(entry=f'{tc.TEST_DATA_DIR}/test_file.fits.gz'), TEST_APP, test_cred, data_client_mock, repo_client_mock, meta_visitors=None, observation=_read_obs(None), observable=test_observer, ) test_executor.execute(None) assert repo_client_mock.delete.called, 'delete call missed' assert repo_client_mock.create.called, 'create call missed' assert test_observer.metrics.observe.called, 'observe not called'
def test_client_visit(test_config): test_cred = None data_client_mock = Mock() repo_client_mock = Mock() test_observer = Mock() with patch('caom2pipe.manage_composable.write_obs_to_file') as write_mock: test_executor = ec.MetaVisit( test_config, tc.TestStorageName(), test_cred, data_client_mock, repo_client_mock, meta_visitors=None, observable=test_observer, ) test_executor.execute(None) repo_client_mock.read.assert_called_with( 'OMM', 'test_obs_id' ), 'read call missed' assert repo_client_mock.update.called, 'update call missed' assert test_observer.metrics.observe.called, 'observe not called' assert write_mock.called, 'write mock not called'
def test_storage_name_builder(): test_subject = nbc.StorageNameBuilder() test_storage_name = tc.TestStorageName() assert (test_subject.build(test_storage_name) == test_storage_name ), 'build wrong result'
def test_organize_executes_client_do_one(test_config): test_obs_id = tc.TestStorageName() test_config.use_local_files = True log_file_directory = os.path.join(tc.THIS_DIR, 'logs') test_config.log_file_directory = log_file_directory success_log_file_name = 'success_log.txt' test_config.success_log_file_name = success_log_file_name failure_log_file_name = 'failure_log.txt' test_config.failure_log_file_name = failure_log_file_name test_config.features.use_clients = True retry_file_name = 'retries.txt' test_config.retry_file_name = retry_file_name repo_client_mock = Mock(autospec=True) repo_client_mock.read.return_value = None test_config.task_types = [mc.TaskType.SCRAPE] test_oe = ec.OrganizeExecutes( test_config, TEST_APP, [], [], chooser=None, cadc_client=Mock(autospec=True), caom_client=repo_client_mock, ) executors = test_oe.choose(test_obs_id) assert executors is not None assert len(executors) == 1 assert isinstance(executors[0], ec.ScrapeUpdate), f'{type(executors[0])}' test_config.task_types = [ mc.TaskType.STORE, mc.TaskType.INGEST, mc.TaskType.MODIFY, ] test_oe = ec.OrganizeExecutes( test_config, TEST_APP, [], [], chooser=None, cadc_client=Mock(autospec=True), caom_client=repo_client_mock, ) executors = test_oe.choose(test_obs_id) assert executors is not None assert len(executors) == 3 assert (isinstance(executors[0], ec.Store), type(executors[0])) assert isinstance(executors[1], ec.LocalMetaCreate) assert isinstance(executors[2], ec.LocalDataVisit) assert repo_client_mock.read.called, 'mock should be called' assert repo_client_mock.read.reset() test_config.use_local_files = False test_config.task_types = [mc.TaskType.INGEST, mc.TaskType.MODIFY] test_oe = ec.OrganizeExecutes( test_config, TEST_APP, [], [], chooser=None, cadc_client=Mock(autospec=True), caom_client=repo_client_mock, ) executors = test_oe.choose(test_obs_id) assert executors is not None assert len(executors) == 2 assert isinstance(executors[0], ec.MetaCreate) assert isinstance(executors[1], ec.DataVisit) assert repo_client_mock.read.called, 'mock should be called' test_config.use_local_files = True test_config.task_types = [mc.TaskType.INGEST, mc.TaskType.MODIFY] test_oe = ec.OrganizeExecutes( test_config, TEST_APP, [], [], chooser=None, cadc_client=Mock(autospec=True), caom_client=repo_client_mock, ) executors = test_oe.choose(test_obs_id) assert executors is not None assert len(executors) == 2 assert isinstance(executors[0], ec.LocalMetaCreate) assert isinstance(executors[1], ec.LocalDataVisit) assert repo_client_mock.read.called, 'mock should be called' assert repo_client_mock.read.reset() repo_client_mock.read.side_effect = _read_obs2 test_config.task_types = [mc.TaskType.SCRAPE, mc.TaskType.MODIFY] test_config.use_local_files = True test_oe = ec.OrganizeExecutes( test_config, TEST_APP, [], [], chooser=None, cadc_client=Mock(autospec=True), caom_client=repo_client_mock, ) executors = test_oe.choose(test_obs_id) assert executors is not None assert len(executors) == 2 assert isinstance(executors[0], ec.ScrapeUpdate) assert isinstance(executors[1], ec.DataScrape) assert repo_client_mock.read.called, 'mock should be called' assert repo_client_mock.read.reset() test_config.task_types = [mc.TaskType.INGEST] test_config.use_local_files = False test_chooser = tc.TestChooser() ec.CaomExecute.repo_cmd_get_client = Mock(return_value=_read_obs(None)) test_oe = ec.OrganizeExecutes( test_config, TEST_APP, [], [], chooser=test_chooser, cadc_client=Mock(autospec=True), caom_client=repo_client_mock, ) executors = test_oe.choose(test_obs_id) assert executors is not None assert len(executors) == 1 assert isinstance(executors[0], ec.MetaDeleteCreate) assert repo_client_mock.read.called, 'mock should be called' assert repo_client_mock.read.reset() test_config.task_types = [mc.TaskType.INGEST_OBS] test_config.use_local_files = False ec.CaomExecute.repo_cmd_get_client = Mock( return_value=_read_obs(test_obs_id)) test_oe = ec.OrganizeExecutes( test_config, TEST_APP, [], [], cadc_client=Mock(autospec=True), caom_client=repo_client_mock, ) executors = test_oe.choose(test_obs_id) assert executors is not None assert len(executors) == 1 assert isinstance(executors[0], ec.MetaUpdateObservation) assert repo_client_mock.read.called, 'mock should be called' assert executors[0].stream == 'TEST', 'stream' assert executors[0].external_urls_param == '', 'external_url_params' assert (executors[0].working_dir == f'{tc.THIS_DIR}/test_obs_id' ), 'working_dir' assert test_oe.todo_fqn == f'{tc.THIS_DIR}/todo.txt', 'wrong todo'
def test_organize_executes_client_do_one(test_config): test_obs_id = tc.TestStorageName() test_config.use_local_files = True log_file_directory = os.path.join(tc.THIS_DIR, 'logs') test_config.log_file_directory = log_file_directory success_log_file_name = 'success_log.txt' test_config.success_log_file_name = success_log_file_name failure_log_file_name = 'failure_log.txt' test_config.failure_log_file_name = failure_log_file_name test_config.features.use_clients = True retry_file_name = 'retries.txt' test_config.retry_file_name = retry_file_name exec_cmd_orig = mc.exec_cmd_info repo_client_mock = Mock(autospec=True) repo_client_mock.read.return_value = None try: mc.exec_cmd_info = Mock( return_value='INFO:cadc-data:info\n' 'File C170324_0054_SCI_prev.jpg:\n' ' archive: OMM\n' ' encoding: None\n' ' lastmod: Mon, 25 Jun 2018 16:52:07 GMT\n' ' md5sum: f37d21c53055498d1b5cb7753e1c6d6f\n' ' name: C120902_sh2-132_J_old_' 'SCIRED.fits.gz\n' ' size: 754408\n' ' type: image/jpeg\n' ' umd5sum: 704b494a972eed30b18b817e243ced7d\n' ' usize: 754408\n'.encode('utf-8') ) test_config.task_types = [mc.TaskType.SCRAPE] test_oe = ec.OrganizeExecutes( test_config, TEST_APP, [], [], chooser=None, cadc_client=Mock(autospec=True), caom_client=repo_client_mock, ) executors = test_oe.choose(test_obs_id) assert executors is not None assert len(executors) == 1 assert isinstance(executors[0], ec.ScrapeUpdate) test_config.task_types = [ mc.TaskType.STORE, mc.TaskType.INGEST, mc.TaskType.MODIFY ] test_oe = ec.OrganizeExecutes( test_config, TEST_APP, [], [], chooser=None, cadc_client=Mock(autospec=True), caom_client=repo_client_mock, ) executors = test_oe.choose(test_obs_id) assert executors is not None assert len(executors) == 3 assert isinstance(executors[0], ec.Store), type(executors[0]) assert isinstance(executors[1], ec.LocalMetaCreate) assert isinstance(executors[2], ec.LocalDataVisit) assert repo_client_mock.read.called, 'mock should be called' assert repo_client_mock.read.reset() test_config.use_local_files = False test_config.task_types = [mc.TaskType.INGEST, mc.TaskType.MODIFY] test_oe = ec.OrganizeExecutes( test_config, TEST_APP, [], [], chooser=None, cadc_client=Mock(autospec=True), caom_client=repo_client_mock, ) executors = test_oe.choose(test_obs_id) assert executors is not None assert len(executors) == 2 assert isinstance(executors[0], ec.MetaCreate) assert isinstance(executors[1], ec.DataVisit) assert repo_client_mock.read.called, 'mock should be called' test_config.use_local_files = True test_config.task_types = [mc.TaskType.INGEST, mc.TaskType.MODIFY] test_oe = ec.OrganizeExecutes( test_config, TEST_APP, [], [], chooser=None, cadc_client=Mock(autospec=True), caom_client=repo_client_mock, ) executors = test_oe.choose(test_obs_id) assert executors is not None assert len(executors) == 2 assert isinstance(executors[0], ec.LocalMetaCreate) assert isinstance(executors[1], ec.LocalDataVisit) assert repo_client_mock.read.called, 'mock should be called' assert repo_client_mock.read.reset() repo_client_mock.read.side_effect = _read_obs2 test_config.task_types = [mc.TaskType.SCRAPE, mc.TaskType.MODIFY] test_config.use_local_files = True test_oe = ec.OrganizeExecutes( test_config, TEST_APP, [], [], chooser=None, cadc_client=Mock(autospec=True), caom_client=repo_client_mock, ) executors = test_oe.choose(test_obs_id) assert executors is not None assert len(executors) == 2 assert isinstance(executors[0], ec.ScrapeUpdate) assert isinstance(executors[1], ec.DataScrape) assert repo_client_mock.read.called, 'mock should be called' assert repo_client_mock.read.reset() test_config.task_types = [mc.TaskType.INGEST] test_config.use_local_files = False test_chooser = tc.TestChooser() ec.CaomExecute.repo_cmd_get_client = Mock(return_value=_read_obs(None)) test_oe = ec.OrganizeExecutes( test_config, TEST_APP, [], [], chooser=test_chooser, cadc_client=Mock(autospec=True), caom_client=repo_client_mock, ) executors = test_oe.choose(test_obs_id) assert executors is not None assert len(executors) == 1 assert isinstance(executors[0], ec.MetaDeleteCreate) assert repo_client_mock.read.called, 'mock should be called' assert repo_client_mock.read.reset() test_config.task_types = [mc.TaskType.INGEST_OBS] test_config.use_local_files = False ec.CaomExecute.repo_cmd_get_client = Mock( return_value=_read_obs(test_obs_id) ) test_oe = ec.OrganizeExecutes( test_config, TEST_APP, [], [], cadc_client=Mock(autospec=True), caom_client=repo_client_mock, ) executors = test_oe.choose(test_obs_id) assert executors is not None assert len(executors) == 1 assert isinstance(executors[0], ec.MetaUpdateObservation) assert repo_client_mock.read.called, 'mock should be called' assert executors[0].url == 'https://test_url/', 'url' assert executors[0].fname is None, 'file name' assert executors[0].stream == 'TEST', 'stream' assert executors[0].lineage is None, 'lineage' assert executors[0].external_urls_param == '', 'external_url_params' assert ( executors[0].working_dir == f'{tc.THIS_DIR}/test_obs_id' ), 'working_dir' assert test_oe.todo_fqn == f'{tc.THIS_DIR}/todo.txt', 'wrong todo' finally: mc.exec_cmd_orig = exec_cmd_orig
def test_local_store(test_config): test_config.working_directory = '/test_files/caom2pipe' test_config.use_local_files = True test_config.archive = 'LOCAL_TEST' test_config.store_newer_files_only = False test_config.features.supports_latest_client = False test_sn = tc.TestStorageName() test_fqn = os.path.join(test_config.working_directory, test_sn.file_name) if not os.path.exists(test_fqn): with open(test_fqn, 'w') as f: f.write('test content') test_command = 'collection2caom2' test_cred_param = '' test_data_client = Mock(autospec=True) test_repo_client = Mock(autospec=True) test_observable = Mock(autospec=True) test_transferrer = transfer_composable.Transfer() test_subject = ec.LocalStore( test_config, test_sn, test_command, test_cred_param, test_data_client, test_repo_client, test_observable, test_transferrer, ) assert test_subject is not None, 'expect construction' test_subject.execute(None) # does the working directory get used if it's just a local store? assert ( test_subject.working_dir == '/test_files/caom2pipe' ), 'wrong working directory' # do one file at a time, so it doesn't matter how many files are # in the working directory assert len(test_subject.multiple_files) == 1, 'wrong file count' assert ( len(test_subject._destination_f_names) == 1 ), 'wrong destination file count' assert ( test_subject._destination_f_names[0] == 'test_obs_id.fits' ), 'wrong destination' assert test_data_client.put_file.called, 'data put not called' assert ( test_data_client.put_file.call_args.args[0] == 'LOCAL_TEST' ), 'expect an archive' assert ( test_data_client.put_file.call_args.args[1] == 'test_obs_id.fits' ), 'expect a file name' assert ( test_data_client.put_file.call_args.kwargs['archive_stream'] == 'TEST' ), 'wrong archive' assert ( test_data_client.put_file.call_args.kwargs['mime_type'] == 'application/fits' ), 'wrong archive' assert ( test_data_client.put_file.call_args.kwargs['mime_encoding'] is None ), 'wrong archive' assert ( test_data_client.put_file.call_args.kwargs['md5_check'] is True ), 'wrong archive'
def test_organize_executes_chooser(test_config): test_obs_id = tc.TestStorageName() test_config.use_local_files = True log_file_directory = os.path.join(tc.THIS_DIR, 'logs') test_config.log_file_directory = log_file_directory test_config.features.supports_composite = True exec_cmd_orig = mc.exec_cmd_info caom_client = Mock(autospec=True) caom_client.read.side_effect = _read_obs2 try: mc.exec_cmd_info = \ Mock( return_value='INFO:cadc-data:info\n' 'File C170324_0054_SCI_prev.jpg:\n' ' archive: OMM\n' ' encoding: None\n' ' lastmod: Mon, 25 Jun 2018 16:52:07 GMT\n' ' md5sum: f37d21c53055498d1b5cb7753e1c6d6f\n' ' name: C120902_sh2-132_J_old_' 'SCIRED.fits.gz\n' ' size: 754408\n' ' type: image/jpeg\n' ' umd5sum: 704b494a972eed30b18b817e243ced7d\n' ' usize: 754408\n'.encode('utf-8') ) test_config.task_types = [mc.TaskType.INGEST] test_chooser = tc.TestChooser() test_oe = ec.OrganizeExecutes( test_config, 'command_name', [], [], test_chooser, cadc_client=Mock(autospec=True, return_value=None), caom_client=caom_client, ) executors = test_oe.choose(test_obs_id) assert executors is not None assert len(executors) == 1 assert isinstance(executors[0], ec.LocalMetaDeleteCreate) assert executors[0].fname == 'test_obs_id.fits', 'file name' assert executors[0].stream == 'TEST', 'stream' assert executors[0].working_dir == tc.THIS_DIR, 'working_dir' assert caom_client.read.called, 'read should be called' caom_client.read.reset() test_config.use_local_files = False test_config.task_types = [mc.TaskType.INGEST] test_oe = ec.OrganizeExecutes( test_config, 'command_name', [], [], test_chooser, cadc_client=Mock(autospec=True, return_value=None), caom_client=caom_client, ) executors = test_oe.choose(test_obs_id) assert executors is not None assert len(executors) == 1 assert isinstance(executors[0], ec.MetaDeleteCreate) assert caom_client.read.called, 'read should be called' finally: mc.exec_cmd_orig = exec_cmd_orig