def test_capture_failure(test_config):
start_s = datetime.utcnow().timestamp()
test_obs_id = 'test_obs_id'
test_obs_id_2 = 'test_obs_id_2'
log_file_directory = os.path.join(tc.THIS_DIR, 'logs')
test_config.log_to_file = True
test_config.log_file_directory = log_file_directory
success_log_file_name = 'success_log.txt'
test_config.success_log_file_name = success_log_file_name
failure_log_file_name = 'failure_log.txt'
test_config.failure_log_file_name = failure_log_file_name
retry_file_name = 'retries.txt'
test_config.retry_file_name = retry_file_name
rejected_file_name = 'rejected.yml'
test_config.rejected_file_name = rejected_file_name
# clean up from last execution
if not os.path.exists(log_file_directory):
os.mkdir(log_file_directory)
if os.path.exists(test_config.success_fqn):
os.remove(test_config.success_fqn)
if os.path.exists(test_config.failure_fqn):
os.remove(test_config.failure_fqn)
if os.path.exists(test_config.retry_fqn):
os.remove(test_config.retry_fqn)
if os.path.exists(test_config.rejected_fqn):
os.remove(test_config.rejected_fqn)
test_oe = ec.OrganizeExecutesWithDoOne(test_config, 'command', [], [])
test_sname = tc.TestStorageName(obs_id=test_obs_id_2)
test_oe.capture_failure(test_sname, 'Cannot build an observation')
test_sname = tc.TestStorageName(obs_id=test_obs_id)
test_oe.capture_failure(test_sname, 'exception text')
test_oe.capture_success(test_obs_id, 'C121212_01234_CAL.fits.gz', start_s)
test_oe.finish_run(test_config)
assert os.path.exists(test_config.success_fqn)
assert os.path.exists(test_config.failure_fqn)
assert os.path.exists(test_config.retry_fqn)
assert os.path.exists(test_config.rejected_fqn)
success_content = open(test_config.success_fqn).read()
assert ('test_obs_id C121212_01234_CAL.fits.gz'
in success_content), 'wrong content'
retry_content = open(test_config.retry_fqn).read()
assert retry_content == 'test_obs_id\n'
failure_content = open(test_config.failure_fqn).read()
assert failure_content.endswith(
'Unknown error. Check specific log.\n'), failure_content
assert os.path.exists(test_config.rejected_fqn), test_config.rejected_fqn
rejected_content = mc.read_as_yaml(test_config.rejected_fqn)
assert rejected_content is not None, 'expect a result'
test_result = rejected_content.get('bad_metadata')
assert test_result is not None, 'wrong result'
assert len(test_result) == 1, 'wrong number of entries'
assert test_result[0] == test_obs_id, 'wrong entry'
def test_do_one(test_config):
test_config.task_types = []
test_organizer = ec.OrganizeExecutes(test_config, 'test2caom2', [], [])
# no client
test_result = test_organizer.do_one(tc.TestStorageName())
assert test_result is not None
assert test_result == -1
# client
test_config.features.use_clients = True
test_result = test_organizer.do_one(tc.TestStorageName())
assert test_result is not None
assert test_result == -1
def test_data_execute_v(test_config):
test_config.features.supports_latest_client = True
test_obs_id = 'test_obs_id'
test_dir = os.path.join(tc.THIS_DIR, test_obs_id)
test_fits_fqn = os.path.join(test_dir, tc.TestStorageName().file_name)
try:
if not os.path.exists(test_dir):
os.mkdir(test_dir, mode=0o755)
# precondition = open(test_fits_fqn, 'w')
# precondition.close()
test_data_visitors = [TestVisit]
repo_client_mock = Mock(autospec=True)
cadc_client_mock = Mock(autospec=True)
cadc_client_mock.copy.side_effect = tc.mock_copy_md5
test_observer = Mock(autospec=True)
test_cred = ''
test_transferrer = transfer_composable.VoTransfer()
test_transferrer.cadc_client = cadc_client_mock
ec.CaomExecute._data_cmd_info = Mock(side_effect=_get_fname)
repo_client_mock.read.side_effect = tc.mock_read
# run the test
test_executor = ec.DataVisit(
test_config,
tc.TestStorageName(),
test_cred,
cadc_client_mock,
repo_client_mock,
test_data_visitors,
mc.TaskType.MODIFY,
test_observer,
test_transferrer,
)
test_executor.execute(None)
# check that things worked as expected
assert repo_client_mock.read.called, 'read call missed'
assert repo_client_mock.update.called, 'update call missed'
assert test_observer.metrics.observe.called, 'observe not called'
assert cadc_client_mock.copy.called, 'copy not called'
cadc_client_mock.copy.assert_called_with(
'ad:TEST/test_obs_id.fits.gz', test_fits_fqn, send_md5=True
), 'wrong call args'
finally:
if os.path.exists(test_fits_fqn):
os.unlink(test_fits_fqn)
if os.path.exists(test_dir):
os.rmdir(test_dir)
def test_store(test_config):
test_config.working_directory = tc.TEST_DATA_DIR
test_sn = tc.TestStorageName()
test_sn.source_names = ['vos:goliaths/nonexistent.fits.gz']
test_command = 'collection2caom2'
test_data_client = Mock(autospec=True)
test_observable = Mock(autospec=True)
test_transferrer = Mock(autospec=True)
test_transferrer.get.side_effect = _transfer_get_mock
test_subject = ec.Store(
test_config,
test_sn,
test_command,
test_data_client,
test_observable,
test_transferrer,
)
assert test_subject is not None, 'expect construction'
assert test_subject.working_dir == os.path.join(
tc.TEST_DATA_DIR, 'test_obs_id'), 'wrong working directory'
assert (len(
test_subject._storage_name.destination_uris) == 1), 'wrong file count'
assert (test_subject._storage_name.destination_uris[0] ==
'cadc:TEST/test_file.fits.gz'), 'wrong destination'
test_subject.execute(None)
assert test_data_client.put.called, 'data put not called'
test_data_client.put.assert_called_with(
'/usr/src/app/caom2pipe/caom2pipe/tests/data/test_obs_id',
'cadc:TEST/test_file.fits.gz',
'TEST',
), 'wrong put call args'
def test_choose_exceptions(test_config):
test_config.init_local_files = False
test_config.task_types = [mc.TaskType.SCRAPE]
with pytest.raises(mc.CadcException):
test_organizer = ec.OrganizeExecutes(test_config, 'command name', [],
[])
test_organizer.choose(tc.TestStorageName())
def test_data_store(test_config):
test_dir = f'{tc.TEST_DATA_DIR}/test_obs_id'
if os.path.exists(test_dir):
os.rmdir(test_dir)
data_client_mock = Mock()
test_observer = Mock()
# stat mock is for CadcDataClient
stat_orig = os.stat
os.stat = Mock()
os.stat.st_size.return_value = 1243
# path.exists mock is because the stat mock causes os.path.exists to
# return True
path_orig = os.path.exists
os.path.exists = Mock(return_value=False)
test_config.features.supports_multiple_files = False
try:
test_config.working_directory = tc.TEST_DATA_DIR
test_executor = ec.Store(
test_config,
tc.TestStorageName(entry=f'{tc.TEST_DATA_DIR}/test_file.fits.gz'),
'command_name',
data_client_mock,
observable=test_observer,
transferrer=transfer_composable.Transfer(),
)
test_executor.execute(None)
# check that things worked as expected - no cleanup
assert data_client_mock.put.called, 'put_file call missed'
finally:
os.stat = stat_orig
os.path.exists = path_orig
def test_data_local_execute(test_config):
# ensure the model uses the log directory for writing the model file
test_config.log_to_file = True
test_config.working_directory = tc.TEST_DATA_DIR
test_data_visitors = [LocalTestVisit]
data_client_mock = Mock()
repo_client_mock = Mock()
repo_client_mock.read.return_value = _read_obs(None)
test_observer = Mock()
test_model_fqn = os.path.join(tc.TEST_DATA_DIR, 'test_obs_id.xml')
# check that a file is written to disk
if os.path.exists(test_model_fqn):
os.unlink(test_model_fqn)
# run the test
test_executor = ec.LocalDataVisit(
test_config,
tc.TestStorageName(entry=f'{tc.TEST_DATA_DIR}/test_file.fits.gz'),
data_client_mock,
repo_client_mock,
test_data_visitors,
observable=test_observer,
)
test_executor.execute(None)
# check that things worked as expected - no cleanup
assert repo_client_mock.read.called, 'read call missed'
repo_client_mock.read.assert_called_with(
'OMM', 'test_obs_id'), 'wrong repo client read args'
assert repo_client_mock.update.called, 'update call missed'
assert test_observer.metrics.observe.called, 'observe not called'
assert os.path.exists(test_model_fqn), 'observation not written to disk'
def test_scrape(test_config):
# clean up from previous tests
if os.path.exists(tc.TestStorageName().model_file_name):
os.remove(tc.TestStorageName().model_file_name)
netrc = os.path.join(tc.TEST_DATA_DIR, 'test_netrc')
assert os.path.exists(netrc)
test_config.working_directory = tc.TEST_DATA_DIR
test_config.logging_level = 'INFO'
test_executor = ec.Scrape(
test_config,
tc.TestStorageName(),
__name__,
observable=None,
meta_visitors=[],
)
test_executor.execute(None)
def test_local_store(test_config):
test_config.working_directory = tc.TEST_DATA_DIR
test_config.data_source = ['/test_files/caom2pipe']
test_config.use_local_files = True
test_config.store_newer_files_only = False
test_config.features.supports_latest_client = False
test_sn = tc.TestStorageName(entry=f'{tc.TEST_DATA_DIR}/test_file.fits.gz')
if not os.path.exists(test_sn.source_names[0]):
with open(test_sn.source_names[0], 'w') as f:
f.write('test content')
test_command = 'collection2caom2'
test_data_client = Mock(autospec=True)
test_observable = Mock(autospec=True)
test_subject = ec.LocalStore(test_config, test_sn, test_command,
test_data_client, test_observable)
assert test_subject is not None, 'expect construction'
test_subject.execute(None)
# does the working directory get used if it's just a local store?
assert test_subject.working_dir == os.path.join(
tc.TEST_DATA_DIR, 'test_obs_id'), 'wrong working directory'
# do one file at a time, so it doesn't matter how many files are
# in the working directory
assert (len(
test_subject._storage_name.destination_uris) == 1), 'wrong file count'
assert (test_subject._storage_name.destination_uris[0] ==
'cadc:TEST/test_file.fits.gz'), 'wrong destination'
assert test_data_client.put.called, 'data put not called'
assert (test_data_client.put.call_args.args[0] == tc.TEST_DATA_DIR
), 'working directory'
assert (test_data_client.put.call_args.args[1] ==
test_sn.destination_uris[0]), 'expect a file name'
def test_data_local_execute(test_config):
test_data_visitors = [TestVisit]
data_client_mock = Mock()
data_client_mock.get_file_info.return_value = {'name': 'test_file.fits'}
repo_client_mock = Mock()
repo_client_mock.read.return_value = _read_obs(None)
test_cred = None
test_observer = Mock()
# run the test
test_executor = ec.LocalDataVisit(
test_config,
tc.TestStorageName(),
test_cred,
data_client_mock,
repo_client_mock,
test_data_visitors,
observable=test_observer,
)
test_executor.execute(None)
# check that things worked as expected - no cleanup
assert repo_client_mock.read.called, 'read call missed'
assert repo_client_mock.update.called, 'update call missed'
assert test_observer.metrics.observe.called, 'observe not called'
def test_meta_create_client_execute_failed_update(
headers_mock, f2c2_data_client_mock, test_config
):
test_cred = ''
data_client_mock = Mock()
data_client_mock.get_file_info.return_value = {'name': 'test_file.fits'}
repo_client_mock = Mock()
test_observer = Mock()
read_obs_orig = mc.read_obs_from_file
mc.read_obs_from_file = Mock()
mc.read_obs_from_file.return_value = _read_obs(None)
f2c2_data_client_mock.return_value.get_file_info.side_effect = \
_mock_get_file_info
headers_mock.side_effect = _get_headers
test_executor = ec.MetaCreate(
test_config,
tc.TestStorageName(),
'failedUpdateCollection2caom2',
test_cred,
data_client_mock,
repo_client_mock,
meta_visitors=None,
observable=test_observer,
)
try:
with pytest.raises(mc.CadcException):
test_executor.execute(None)
assert not repo_client_mock.create.called, 'should have no create call'
assert test_executor.url == 'https://test_url/', 'url'
finally:
mc.read_obs_from_file = read_obs_orig
def test_local_meta_create_client_execute(test_config):
test_dir = os.path.join(tc.THIS_DIR, 'test_obs_id')
test_f_fqn = os.path.join(test_dir, 'test_obs_id.fits.xml')
if os.path.exists(test_f_fqn):
os.unlink(test_f_fqn)
os.rmdir(test_dir)
test_cred = ''
data_client_mock = Mock()
data_client_mock.get_file_info.return_value = {'name': 'test_file.fits'}
repo_client_mock = Mock()
test_observer = Mock()
test_config.logging_level = 'INFO'
test_executor = ec.LocalMetaCreate(
test_config,
tc.TestStorageName(entry=f'{tc.TEST_DATA_DIR}/test_file.fits.gz'),
__name__,
test_cred,
data_client_mock,
repo_client_mock,
meta_visitors=None,
observable=test_observer,
)
test_executor.execute(None)
assert repo_client_mock.create.called, 'create call missed'
assert test_observer.metrics.observe.called, 'observe not called'
def test_meta_create_client_execute(test_config):
test_cred = ''
data_client_mock = Mock()
data_client_mock.get_file_info.return_value = {'name': 'test_file.fits'}
repo_client_mock = Mock()
test_observer = Mock()
read_obs_orig = mc.read_obs_from_file
mc.read_obs_from_file = Mock()
mc.read_obs_from_file.return_value = _read_obs(None)
test_executor = ec.MetaCreate(
test_config,
tc.TestStorageName(),
TEST_APP,
test_cred,
data_client_mock,
repo_client_mock,
meta_visitors=None,
observable=test_observer,
)
try:
test_executor.execute(None)
assert repo_client_mock.create.called, 'create call missed'
assert test_executor.url == 'https://test_url/', 'url'
assert test_observer.metrics.observe.called, 'observe not called'
finally:
mc.read_obs_from_file = read_obs_orig
def test_meta_update_observation_direct(test_config):
test_cred_param = '--cert /usr/src/app/cadcproxy.pem'
data_client_mock = Mock()
repo_client_mock = Mock()
test_sn = tc.TestStorageName()
test_observation = mc.read_obs_from_file(
f'{tc.TEST_DATA_DIR}/fpf_start_obs.xml'
)
try:
test_observable = mc.Observable(
mc.Rejected(test_config.rejected_fqn), mc.Metrics(test_config)
)
test_executor = ec.MetaUpdateObservation(
test_config,
test_sn,
TEST_APP,
test_cred_param,
data_client_mock,
repo_client_mock,
test_observation,
[],
observable=test_observable,
)
test_executor.execute(None)
assert repo_client_mock.update.called, 'repo update called'
finally:
pass
def test_data_execute(test_config):
test_obs_id = 'test_obs_id'
test_dir = os.path.join(tc.THIS_DIR, test_obs_id)
test_fits_fqn = os.path.join(test_dir, tc.TestStorageName().file_name)
try:
if not os.path.exists(test_dir):
os.mkdir(test_dir, mode=0o755)
precondition = open(test_fits_fqn, 'w')
precondition.close()
logging.error(test_fits_fqn)
test_data_visitors = [TestVisit]
repo_client_mock = Mock()
data_client_mock = Mock()
test_observer = Mock()
test_cred = ''
test_transferrer = transfer_composable.VoTransfer()
test_transferrer.cadc_client = data_client_mock
ec.CaomExecute._data_cmd_info = Mock(side_effect=_get_fname)
repo_client_mock.read.side_effect = tc.mock_read
# run the test
test_executor = ec.DataVisit(
test_config,
tc.TestStorageName(),
test_cred,
data_client_mock,
repo_client_mock,
test_data_visitors,
mc.TaskType.MODIFY,
test_observer,
test_transferrer,
)
test_executor.execute(None)
# check that things worked as expected
assert repo_client_mock.read.called, 'read call missed'
assert repo_client_mock.update.called, 'update call missed'
assert test_observer.metrics.observe.called, 'observe not called'
finally:
if os.path.exists(test_fits_fqn):
os.unlink(test_fits_fqn)
if os.path.exists(test_dir):
os.rmdir(test_dir)
def test_store(test_config):
test_config.working_directory = '/test_files/caom2pipe'
test_sn = tc.TestStorageName()
test_command = 'collection2caom2'
test_cred_param = ''
test_data_client = Mock(autospec=True)
test_repo_client = Mock(autospec=True)
test_observable = Mock(autospec=True)
test_transferrer = Mock(autospec=True)
test_transferrer.get.side_effect = _transfer_get_mock
test_subject = ec.Store(
test_config,
test_sn,
test_command,
test_cred_param,
test_data_client,
test_repo_client,
test_observable,
test_transferrer,
)
assert test_subject is not None, 'expect construction'
assert (
test_subject.working_dir == '/test_files/caom2pipe/test_obs_id'
), 'wrong working directory'
assert len(test_subject.multiple_files) == 1, 'wrong file count'
assert (
len(test_subject._destination_f_names) == 1
), 'wrong destination file count'
assert (
test_subject._destination_f_names[0] == 'test_obs_id.fits'
), 'wrong destination'
test_subject.execute(None)
assert test_data_client.put_file.called, 'data put not called'
assert (
test_data_client.put_file.call_args.args[0] is None
), 'archive not set for test_config'
assert (
test_data_client.put_file.call_args.args[1] == 'test_obs_id.fits'
), 'expect a file name'
assert (
test_data_client.put_file.call_args.kwargs['archive_stream'] == 'TEST'
), 'wrong archive stream'
assert (
test_data_client.put_file.call_args.kwargs['mime_type'] ==
'application/fits'
), 'wrong archive'
assert (
test_data_client.put_file.call_args.kwargs['mime_encoding'] is None
), 'wrong archive'
assert (
test_data_client.put_file.call_args.kwargs['md5_check'] is True
), 'wrong archive'
def test_data_scrape_execute(read_obs_mock, test_config):
test_config.log_to_file = True
test_data_visitors = [TestVisit]
read_obs_mock.side_effect = _read_obs
# run the test
test_executor = ec.DataScrape(
test_config,
tc.TestStorageName(),
test_data_visitors,
observable=None,
)
test_executor.execute(None)
assert read_obs_mock.called, 'read obs call missed'
def test_organize_executes_chooser(test_config):
test_obs_id = tc.TestStorageName()
test_config.use_local_files = True
log_file_directory = os.path.join(tc.THIS_DIR, 'logs')
test_config.log_file_directory = log_file_directory
test_config.features.supports_composite = True
caom_client = Mock(autospec=True)
caom_client.read.side_effect = _read_obs2
test_config.task_types = [mc.TaskType.INGEST]
test_chooser = tc.TestChooser()
test_oe = ec.OrganizeExecutes(
test_config,
'command_name',
[],
[],
test_chooser,
cadc_client=Mock(autospec=True, return_value=None),
caom_client=caom_client,
)
executors = test_oe.choose(test_obs_id)
assert executors is not None
assert len(executors) == 1
assert isinstance(executors[0], ec.LocalMetaDeleteCreate)
assert executors[0].stream == 'TEST', 'stream'
assert executors[0].working_dir == os.path.join(
tc.THIS_DIR, 'test_obs_id'), 'working_dir'
assert caom_client.read.called, 'read should be called'
caom_client.read.reset()
test_config.use_local_files = False
test_config.task_types = [mc.TaskType.INGEST]
test_oe = ec.OrganizeExecutes(
test_config,
'command_name',
[],
[],
test_chooser,
cadc_client=Mock(autospec=True, return_value=None),
caom_client=caom_client,
)
executors = test_oe.choose(test_obs_id)
assert executors is not None
assert len(executors) == 1
assert isinstance(executors[0], ec.MetaDeleteCreate)
assert caom_client.read.called, 'read should be called'
def test_data_scrape_execute(test_config):
test_data_visitors = [TestVisit]
read_orig = mc.read_obs_from_file
mc.read_obs_from_file = Mock(side_effect=_read_obs)
try:
# run the test
test_executor = ec.DataScrape(
test_config,
tc.TestStorageName(),
test_data_visitors,
observable=None,
)
test_executor.execute(None)
assert mc.read_obs_from_file.called, 'read obs call missed'
finally:
mc.read_obs_from_file = read_orig
def test_local_meta_update_client_execute(test_config):
test_cred = ''
data_client_mock = Mock()
data_client_mock.get_file_info.return_value = {'name': 'test_file.fits'}
repo_client_mock = Mock()
test_observer = Mock()
test_executor = ec.LocalMetaUpdate(
test_config,
tc.TestStorageName(),
TEST_APP,
test_cred,
data_client_mock,
repo_client_mock,
_read_obs(None),
meta_visitors=None,
observable=test_observer,
)
test_executor.execute(None)
assert repo_client_mock.update.called, 'update call missed'
assert test_observer.metrics.observe.called, 'observe not called'
def test_organize_executes_client_visit(test_config):
test_obs_id = tc.TestStorageName()
test_config.features.use_clients = True
test_config.task_types = [mc.TaskType.VISIT]
test_config.use_local_files = False
repo_client_mock = Mock(autospec=True)
repo_client_mock.read.side_effect = _read_obs2
test_oe = ec.OrganizeExecutes(
test_config,
'command_name',
[],
[],
cadc_client=Mock(autospec=True),
caom_client=repo_client_mock,
)
executors = test_oe.choose(test_obs_id)
assert executors is not None
assert len(executors) == 1
assert isinstance(executors[0], ec.MetaVisit)
assert not repo_client_mock.read.called, 'mock should not be called?'
def test_local_meta_delete_create_client_execute(test_config):
test_cred = ''
data_client_mock = Mock()
data_client_mock.get_file_info.return_value = {'name': 'test_file.fits'}
repo_client_mock = Mock()
test_observer = Mock()
test_executor = ec.LocalMetaDeleteCreate(
test_config,
tc.TestStorageName(entry=f'{tc.TEST_DATA_DIR}/test_file.fits.gz'),
TEST_APP,
test_cred,
data_client_mock,
repo_client_mock,
meta_visitors=None,
observation=_read_obs(None),
observable=test_observer,
)
test_executor.execute(None)
assert repo_client_mock.delete.called, 'delete call missed'
assert repo_client_mock.create.called, 'create call missed'
assert test_observer.metrics.observe.called, 'observe not called'
def test_client_visit(test_config):
test_cred = None
data_client_mock = Mock()
repo_client_mock = Mock()
test_observer = Mock()
with patch('caom2pipe.manage_composable.write_obs_to_file') as write_mock:
test_executor = ec.MetaVisit(
test_config,
tc.TestStorageName(), test_cred,
data_client_mock,
repo_client_mock,
meta_visitors=None,
observable=test_observer,
)
test_executor.execute(None)
repo_client_mock.read.assert_called_with(
'OMM', 'test_obs_id'
), 'read call missed'
assert repo_client_mock.update.called, 'update call missed'
assert test_observer.metrics.observe.called, 'observe not called'
assert write_mock.called, 'write mock not called'
def test_storage_name_builder():
test_subject = nbc.StorageNameBuilder()
test_storage_name = tc.TestStorageName()
assert (test_subject.build(test_storage_name) == test_storage_name
), 'build wrong result'
def test_organize_executes_client_do_one(test_config):
test_obs_id = tc.TestStorageName()
test_config.use_local_files = True
log_file_directory = os.path.join(tc.THIS_DIR, 'logs')
test_config.log_file_directory = log_file_directory
success_log_file_name = 'success_log.txt'
test_config.success_log_file_name = success_log_file_name
failure_log_file_name = 'failure_log.txt'
test_config.failure_log_file_name = failure_log_file_name
test_config.features.use_clients = True
retry_file_name = 'retries.txt'
test_config.retry_file_name = retry_file_name
repo_client_mock = Mock(autospec=True)
repo_client_mock.read.return_value = None
test_config.task_types = [mc.TaskType.SCRAPE]
test_oe = ec.OrganizeExecutes(
test_config,
TEST_APP,
[],
[],
chooser=None,
cadc_client=Mock(autospec=True),
caom_client=repo_client_mock,
)
executors = test_oe.choose(test_obs_id)
assert executors is not None
assert len(executors) == 1
assert isinstance(executors[0], ec.ScrapeUpdate), f'{type(executors[0])}'
test_config.task_types = [
mc.TaskType.STORE,
mc.TaskType.INGEST,
mc.TaskType.MODIFY,
]
test_oe = ec.OrganizeExecutes(
test_config,
TEST_APP,
[],
[],
chooser=None,
cadc_client=Mock(autospec=True),
caom_client=repo_client_mock,
)
executors = test_oe.choose(test_obs_id)
assert executors is not None
assert len(executors) == 3
assert (isinstance(executors[0], ec.Store), type(executors[0]))
assert isinstance(executors[1], ec.LocalMetaCreate)
assert isinstance(executors[2], ec.LocalDataVisit)
assert repo_client_mock.read.called, 'mock should be called'
assert repo_client_mock.read.reset()
test_config.use_local_files = False
test_config.task_types = [mc.TaskType.INGEST, mc.TaskType.MODIFY]
test_oe = ec.OrganizeExecutes(
test_config,
TEST_APP,
[],
[],
chooser=None,
cadc_client=Mock(autospec=True),
caom_client=repo_client_mock,
)
executors = test_oe.choose(test_obs_id)
assert executors is not None
assert len(executors) == 2
assert isinstance(executors[0], ec.MetaCreate)
assert isinstance(executors[1], ec.DataVisit)
assert repo_client_mock.read.called, 'mock should be called'
test_config.use_local_files = True
test_config.task_types = [mc.TaskType.INGEST, mc.TaskType.MODIFY]
test_oe = ec.OrganizeExecutes(
test_config,
TEST_APP,
[],
[],
chooser=None,
cadc_client=Mock(autospec=True),
caom_client=repo_client_mock,
)
executors = test_oe.choose(test_obs_id)
assert executors is not None
assert len(executors) == 2
assert isinstance(executors[0], ec.LocalMetaCreate)
assert isinstance(executors[1], ec.LocalDataVisit)
assert repo_client_mock.read.called, 'mock should be called'
assert repo_client_mock.read.reset()
repo_client_mock.read.side_effect = _read_obs2
test_config.task_types = [mc.TaskType.SCRAPE, mc.TaskType.MODIFY]
test_config.use_local_files = True
test_oe = ec.OrganizeExecutes(
test_config,
TEST_APP,
[],
[],
chooser=None,
cadc_client=Mock(autospec=True),
caom_client=repo_client_mock,
)
executors = test_oe.choose(test_obs_id)
assert executors is not None
assert len(executors) == 2
assert isinstance(executors[0], ec.ScrapeUpdate)
assert isinstance(executors[1], ec.DataScrape)
assert repo_client_mock.read.called, 'mock should be called'
assert repo_client_mock.read.reset()
test_config.task_types = [mc.TaskType.INGEST]
test_config.use_local_files = False
test_chooser = tc.TestChooser()
ec.CaomExecute.repo_cmd_get_client = Mock(return_value=_read_obs(None))
test_oe = ec.OrganizeExecutes(
test_config,
TEST_APP,
[],
[],
chooser=test_chooser,
cadc_client=Mock(autospec=True),
caom_client=repo_client_mock,
)
executors = test_oe.choose(test_obs_id)
assert executors is not None
assert len(executors) == 1
assert isinstance(executors[0], ec.MetaDeleteCreate)
assert repo_client_mock.read.called, 'mock should be called'
assert repo_client_mock.read.reset()
test_config.task_types = [mc.TaskType.INGEST_OBS]
test_config.use_local_files = False
ec.CaomExecute.repo_cmd_get_client = Mock(
return_value=_read_obs(test_obs_id))
test_oe = ec.OrganizeExecutes(
test_config,
TEST_APP,
[],
[],
cadc_client=Mock(autospec=True),
caom_client=repo_client_mock,
)
executors = test_oe.choose(test_obs_id)
assert executors is not None
assert len(executors) == 1
assert isinstance(executors[0], ec.MetaUpdateObservation)
assert repo_client_mock.read.called, 'mock should be called'
assert executors[0].stream == 'TEST', 'stream'
assert executors[0].external_urls_param == '', 'external_url_params'
assert (executors[0].working_dir == f'{tc.THIS_DIR}/test_obs_id'
), 'working_dir'
assert test_oe.todo_fqn == f'{tc.THIS_DIR}/todo.txt', 'wrong todo'
def test_organize_executes_client_do_one(test_config):
test_obs_id = tc.TestStorageName()
test_config.use_local_files = True
log_file_directory = os.path.join(tc.THIS_DIR, 'logs')
test_config.log_file_directory = log_file_directory
success_log_file_name = 'success_log.txt'
test_config.success_log_file_name = success_log_file_name
failure_log_file_name = 'failure_log.txt'
test_config.failure_log_file_name = failure_log_file_name
test_config.features.use_clients = True
retry_file_name = 'retries.txt'
test_config.retry_file_name = retry_file_name
exec_cmd_orig = mc.exec_cmd_info
repo_client_mock = Mock(autospec=True)
repo_client_mock.read.return_value = None
try:
mc.exec_cmd_info = Mock(
return_value='INFO:cadc-data:info\n'
'File C170324_0054_SCI_prev.jpg:\n'
' archive: OMM\n'
' encoding: None\n'
' lastmod: Mon, 25 Jun 2018 16:52:07 GMT\n'
' md5sum: f37d21c53055498d1b5cb7753e1c6d6f\n'
' name: C120902_sh2-132_J_old_'
'SCIRED.fits.gz\n'
' size: 754408\n'
' type: image/jpeg\n'
' umd5sum: 704b494a972eed30b18b817e243ced7d\n'
' usize: 754408\n'.encode('utf-8')
)
test_config.task_types = [mc.TaskType.SCRAPE]
test_oe = ec.OrganizeExecutes(
test_config,
TEST_APP,
[],
[],
chooser=None,
cadc_client=Mock(autospec=True),
caom_client=repo_client_mock,
)
executors = test_oe.choose(test_obs_id)
assert executors is not None
assert len(executors) == 1
assert isinstance(executors[0], ec.ScrapeUpdate)
test_config.task_types = [
mc.TaskType.STORE, mc.TaskType.INGEST, mc.TaskType.MODIFY
]
test_oe = ec.OrganizeExecutes(
test_config,
TEST_APP,
[],
[],
chooser=None,
cadc_client=Mock(autospec=True),
caom_client=repo_client_mock,
)
executors = test_oe.choose(test_obs_id)
assert executors is not None
assert len(executors) == 3
assert isinstance(executors[0], ec.Store), type(executors[0])
assert isinstance(executors[1], ec.LocalMetaCreate)
assert isinstance(executors[2], ec.LocalDataVisit)
assert repo_client_mock.read.called, 'mock should be called'
assert repo_client_mock.read.reset()
test_config.use_local_files = False
test_config.task_types = [mc.TaskType.INGEST, mc.TaskType.MODIFY]
test_oe = ec.OrganizeExecutes(
test_config,
TEST_APP,
[],
[],
chooser=None,
cadc_client=Mock(autospec=True),
caom_client=repo_client_mock,
)
executors = test_oe.choose(test_obs_id)
assert executors is not None
assert len(executors) == 2
assert isinstance(executors[0], ec.MetaCreate)
assert isinstance(executors[1], ec.DataVisit)
assert repo_client_mock.read.called, 'mock should be called'
test_config.use_local_files = True
test_config.task_types = [mc.TaskType.INGEST, mc.TaskType.MODIFY]
test_oe = ec.OrganizeExecutes(
test_config,
TEST_APP,
[],
[],
chooser=None,
cadc_client=Mock(autospec=True),
caom_client=repo_client_mock,
)
executors = test_oe.choose(test_obs_id)
assert executors is not None
assert len(executors) == 2
assert isinstance(executors[0], ec.LocalMetaCreate)
assert isinstance(executors[1], ec.LocalDataVisit)
assert repo_client_mock.read.called, 'mock should be called'
assert repo_client_mock.read.reset()
repo_client_mock.read.side_effect = _read_obs2
test_config.task_types = [mc.TaskType.SCRAPE, mc.TaskType.MODIFY]
test_config.use_local_files = True
test_oe = ec.OrganizeExecutes(
test_config,
TEST_APP,
[],
[],
chooser=None,
cadc_client=Mock(autospec=True),
caom_client=repo_client_mock,
)
executors = test_oe.choose(test_obs_id)
assert executors is not None
assert len(executors) == 2
assert isinstance(executors[0], ec.ScrapeUpdate)
assert isinstance(executors[1], ec.DataScrape)
assert repo_client_mock.read.called, 'mock should be called'
assert repo_client_mock.read.reset()
test_config.task_types = [mc.TaskType.INGEST]
test_config.use_local_files = False
test_chooser = tc.TestChooser()
ec.CaomExecute.repo_cmd_get_client = Mock(return_value=_read_obs(None))
test_oe = ec.OrganizeExecutes(
test_config,
TEST_APP,
[],
[],
chooser=test_chooser,
cadc_client=Mock(autospec=True),
caom_client=repo_client_mock,
)
executors = test_oe.choose(test_obs_id)
assert executors is not None
assert len(executors) == 1
assert isinstance(executors[0], ec.MetaDeleteCreate)
assert repo_client_mock.read.called, 'mock should be called'
assert repo_client_mock.read.reset()
test_config.task_types = [mc.TaskType.INGEST_OBS]
test_config.use_local_files = False
ec.CaomExecute.repo_cmd_get_client = Mock(
return_value=_read_obs(test_obs_id)
)
test_oe = ec.OrganizeExecutes(
test_config,
TEST_APP,
[],
[],
cadc_client=Mock(autospec=True),
caom_client=repo_client_mock,
)
executors = test_oe.choose(test_obs_id)
assert executors is not None
assert len(executors) == 1
assert isinstance(executors[0], ec.MetaUpdateObservation)
assert repo_client_mock.read.called, 'mock should be called'
assert executors[0].url == 'https://test_url/', 'url'
assert executors[0].fname is None, 'file name'
assert executors[0].stream == 'TEST', 'stream'
assert executors[0].lineage is None, 'lineage'
assert executors[0].external_urls_param == '', 'external_url_params'
assert (
executors[0].working_dir == f'{tc.THIS_DIR}/test_obs_id'
), 'working_dir'
assert test_oe.todo_fqn == f'{tc.THIS_DIR}/todo.txt', 'wrong todo'
finally:
mc.exec_cmd_orig = exec_cmd_orig
def test_local_store(test_config):
test_config.working_directory = '/test_files/caom2pipe'
test_config.use_local_files = True
test_config.archive = 'LOCAL_TEST'
test_config.store_newer_files_only = False
test_config.features.supports_latest_client = False
test_sn = tc.TestStorageName()
test_fqn = os.path.join(test_config.working_directory, test_sn.file_name)
if not os.path.exists(test_fqn):
with open(test_fqn, 'w') as f:
f.write('test content')
test_command = 'collection2caom2'
test_cred_param = ''
test_data_client = Mock(autospec=True)
test_repo_client = Mock(autospec=True)
test_observable = Mock(autospec=True)
test_transferrer = transfer_composable.Transfer()
test_subject = ec.LocalStore(
test_config,
test_sn,
test_command,
test_cred_param,
test_data_client,
test_repo_client,
test_observable,
test_transferrer,
)
assert test_subject is not None, 'expect construction'
test_subject.execute(None)
# does the working directory get used if it's just a local store?
assert (
test_subject.working_dir == '/test_files/caom2pipe'
), 'wrong working directory'
# do one file at a time, so it doesn't matter how many files are
# in the working directory
assert len(test_subject.multiple_files) == 1, 'wrong file count'
assert (
len(test_subject._destination_f_names) == 1
), 'wrong destination file count'
assert (
test_subject._destination_f_names[0] == 'test_obs_id.fits'
), 'wrong destination'
assert test_data_client.put_file.called, 'data put not called'
assert (
test_data_client.put_file.call_args.args[0] == 'LOCAL_TEST'
), 'expect an archive'
assert (
test_data_client.put_file.call_args.args[1] == 'test_obs_id.fits'
), 'expect a file name'
assert (
test_data_client.put_file.call_args.kwargs['archive_stream'] ==
'TEST'
), 'wrong archive'
assert (
test_data_client.put_file.call_args.kwargs['mime_type'] ==
'application/fits'
), 'wrong archive'
assert (
test_data_client.put_file.call_args.kwargs['mime_encoding'] is None
), 'wrong archive'
assert (
test_data_client.put_file.call_args.kwargs['md5_check'] is True
), 'wrong archive'
def test_organize_executes_chooser(test_config):
test_obs_id = tc.TestStorageName()
test_config.use_local_files = True
log_file_directory = os.path.join(tc.THIS_DIR, 'logs')
test_config.log_file_directory = log_file_directory
test_config.features.supports_composite = True
exec_cmd_orig = mc.exec_cmd_info
caom_client = Mock(autospec=True)
caom_client.read.side_effect = _read_obs2
try:
mc.exec_cmd_info = \
Mock(
return_value='INFO:cadc-data:info\n'
'File C170324_0054_SCI_prev.jpg:\n'
' archive: OMM\n'
' encoding: None\n'
' lastmod: Mon, 25 Jun 2018 16:52:07 GMT\n'
' md5sum: f37d21c53055498d1b5cb7753e1c6d6f\n'
' name: C120902_sh2-132_J_old_'
'SCIRED.fits.gz\n'
' size: 754408\n'
' type: image/jpeg\n'
' umd5sum: 704b494a972eed30b18b817e243ced7d\n'
' usize: 754408\n'.encode('utf-8')
)
test_config.task_types = [mc.TaskType.INGEST]
test_chooser = tc.TestChooser()
test_oe = ec.OrganizeExecutes(
test_config,
'command_name',
[],
[],
test_chooser,
cadc_client=Mock(autospec=True, return_value=None),
caom_client=caom_client,
)
executors = test_oe.choose(test_obs_id)
assert executors is not None
assert len(executors) == 1
assert isinstance(executors[0], ec.LocalMetaDeleteCreate)
assert executors[0].fname == 'test_obs_id.fits', 'file name'
assert executors[0].stream == 'TEST', 'stream'
assert executors[0].working_dir == tc.THIS_DIR, 'working_dir'
assert caom_client.read.called, 'read should be called'
caom_client.read.reset()
test_config.use_local_files = False
test_config.task_types = [mc.TaskType.INGEST]
test_oe = ec.OrganizeExecutes(
test_config,
'command_name',
[],
[],
test_chooser,
cadc_client=Mock(autospec=True, return_value=None),
caom_client=caom_client,
)
executors = test_oe.choose(test_obs_id)
assert executors is not None
assert len(executors) == 1
assert isinstance(executors[0], ec.MetaDeleteCreate)
assert caom_client.read.called, 'read should be called'
finally:
mc.exec_cmd_orig = exec_cmd_orig
