def test_list_dir_data_source():
get_cwd_orig = os.getcwd
os.getcwd = Mock(return_value=tc.TEST_DATA_DIR)
test_config = mc.Config()
test_config.get_executors()
test_config.working_directory = '/test_files/1'
if not os.path.exists(test_config.working_directory):
os.mkdir(test_config.working_directory)
os.chmod(
test_config.working_directory,
stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR
)
for entry in ['TEST.fits.gz', 'TEST1.fits', 'TEST2.fits.fz', 'TEST3.hdf5']:
if not os.path.exists(f'{test_config.working_directory}/{entry}'):
with open(f'{test_config.working_directory}/{entry}', 'w') as f:
f.write('test content')
test_chooser = tc.TestChooser()
try:
test_subject = dsc.ListDirDataSource(test_config, test_chooser)
test_result = test_subject.get_work()
assert test_result is not None, 'expected a result'
assert len(test_result) == 4, 'wrong result'
assert 'TEST.fits.gz' in test_result, 'wrong gz extensions'
assert 'TEST1.fits.gz' in test_result, 'wrong no extension'
assert 'TEST2.fits.fz' in test_result, 'wrong fz extensions'
assert 'TEST3.hdf5' in test_result, 'wrong hdf5'
test_subject = dsc.ListDirDataSource(test_config, chooser=None)
test_result = test_subject.get_work()
assert test_result is not None, 'expected a result'
assert len(test_result) == 4, 'wrong result'
assert 'TEST.fits.gz' in test_result, 'wrong gz extensions'
assert 'TEST1.fits' in test_result, 'wrong no extension'
assert 'TEST2.fits.fz' in test_result, 'wrong fz extensions'
assert 'TEST3.hdf5' in test_result, 'wrong hdf5'
finally:
os.getcwd = get_cwd_orig
if os.path.exists(test_config.working_directory):
for entry in glob.glob(f'{test_config.working_directory}/*'):
os.unlink(entry)
os.rmdir(test_config.working_directory)
def test_organize_executes_chooser(test_config):
test_obs_id = tc.TestStorageName()
test_config.use_local_files = True
log_file_directory = os.path.join(tc.THIS_DIR, 'logs')
test_config.log_file_directory = log_file_directory
test_config.features.supports_composite = True
caom_client = Mock(autospec=True)
caom_client.read.side_effect = _read_obs2
test_config.task_types = [mc.TaskType.INGEST]
test_chooser = tc.TestChooser()
test_oe = ec.OrganizeExecutes(
test_config,
'command_name',
[],
[],
test_chooser,
cadc_client=Mock(autospec=True, return_value=None),
caom_client=caom_client,
)
executors = test_oe.choose(test_obs_id)
assert executors is not None
assert len(executors) == 1
assert isinstance(executors[0], ec.LocalMetaDeleteCreate)
assert executors[0].stream == 'TEST', 'stream'
assert executors[0].working_dir == os.path.join(
tc.THIS_DIR, 'test_obs_id'), 'working_dir'
assert caom_client.read.called, 'read should be called'
caom_client.read.reset()
test_config.use_local_files = False
test_config.task_types = [mc.TaskType.INGEST]
test_oe = ec.OrganizeExecutes(
test_config,
'command_name',
[],
[],
test_chooser,
cadc_client=Mock(autospec=True, return_value=None),
caom_client=caom_client,
)
executors = test_oe.choose(test_obs_id)
assert executors is not None
assert len(executors) == 1
assert isinstance(executors[0], ec.MetaDeleteCreate)
assert caom_client.read.called, 'read should be called'
def test_organize_executes_client_do_one(test_config):
test_obs_id = tc.TestStorageName()
test_config.use_local_files = True
log_file_directory = os.path.join(tc.THIS_DIR, 'logs')
test_config.log_file_directory = log_file_directory
success_log_file_name = 'success_log.txt'
test_config.success_log_file_name = success_log_file_name
failure_log_file_name = 'failure_log.txt'
test_config.failure_log_file_name = failure_log_file_name
test_config.features.use_clients = True
retry_file_name = 'retries.txt'
test_config.retry_file_name = retry_file_name
exec_cmd_orig = mc.exec_cmd_info
repo_client_mock = Mock(autospec=True)
repo_client_mock.read.return_value = None
try:
mc.exec_cmd_info = Mock(
return_value='INFO:cadc-data:info\n'
'File C170324_0054_SCI_prev.jpg:\n'
' archive: OMM\n'
' encoding: None\n'
' lastmod: Mon, 25 Jun 2018 16:52:07 GMT\n'
' md5sum: f37d21c53055498d1b5cb7753e1c6d6f\n'
' name: C120902_sh2-132_J_old_'
'SCIRED.fits.gz\n'
' size: 754408\n'
' type: image/jpeg\n'
' umd5sum: 704b494a972eed30b18b817e243ced7d\n'
' usize: 754408\n'.encode('utf-8')
)
test_config.task_types = [mc.TaskType.SCRAPE]
test_oe = ec.OrganizeExecutes(
test_config,
TEST_APP,
[],
[],
chooser=None,
cadc_client=Mock(autospec=True),
caom_client=repo_client_mock,
)
executors = test_oe.choose(test_obs_id)
assert executors is not None
assert len(executors) == 1
assert isinstance(executors[0], ec.ScrapeUpdate)
test_config.task_types = [
mc.TaskType.STORE, mc.TaskType.INGEST, mc.TaskType.MODIFY
]
test_oe = ec.OrganizeExecutes(
test_config,
TEST_APP,
[],
[],
chooser=None,
cadc_client=Mock(autospec=True),
caom_client=repo_client_mock,
)
executors = test_oe.choose(test_obs_id)
assert executors is not None
assert len(executors) == 3
assert isinstance(executors[0], ec.Store), type(executors[0])
assert isinstance(executors[1], ec.LocalMetaCreate)
assert isinstance(executors[2], ec.LocalDataVisit)
assert repo_client_mock.read.called, 'mock should be called'
assert repo_client_mock.read.reset()
test_config.use_local_files = False
test_config.task_types = [mc.TaskType.INGEST, mc.TaskType.MODIFY]
test_oe = ec.OrganizeExecutes(
test_config,
TEST_APP,
[],
[],
chooser=None,
cadc_client=Mock(autospec=True),
caom_client=repo_client_mock,
)
executors = test_oe.choose(test_obs_id)
assert executors is not None
assert len(executors) == 2
assert isinstance(executors[0], ec.MetaCreate)
assert isinstance(executors[1], ec.DataVisit)
assert repo_client_mock.read.called, 'mock should be called'
test_config.use_local_files = True
test_config.task_types = [mc.TaskType.INGEST, mc.TaskType.MODIFY]
test_oe = ec.OrganizeExecutes(
test_config,
TEST_APP,
[],
[],
chooser=None,
cadc_client=Mock(autospec=True),
caom_client=repo_client_mock,
)
executors = test_oe.choose(test_obs_id)
assert executors is not None
assert len(executors) == 2
assert isinstance(executors[0], ec.LocalMetaCreate)
assert isinstance(executors[1], ec.LocalDataVisit)
assert repo_client_mock.read.called, 'mock should be called'
assert repo_client_mock.read.reset()
repo_client_mock.read.side_effect = _read_obs2
test_config.task_types = [mc.TaskType.SCRAPE, mc.TaskType.MODIFY]
test_config.use_local_files = True
test_oe = ec.OrganizeExecutes(
test_config,
TEST_APP,
[],
[],
chooser=None,
cadc_client=Mock(autospec=True),
caom_client=repo_client_mock,
)
executors = test_oe.choose(test_obs_id)
assert executors is not None
assert len(executors) == 2
assert isinstance(executors[0], ec.ScrapeUpdate)
assert isinstance(executors[1], ec.DataScrape)
assert repo_client_mock.read.called, 'mock should be called'
assert repo_client_mock.read.reset()
test_config.task_types = [mc.TaskType.INGEST]
test_config.use_local_files = False
test_chooser = tc.TestChooser()
ec.CaomExecute.repo_cmd_get_client = Mock(return_value=_read_obs(None))
test_oe = ec.OrganizeExecutes(
test_config,
TEST_APP,
[],
[],
chooser=test_chooser,
cadc_client=Mock(autospec=True),
caom_client=repo_client_mock,
)
executors = test_oe.choose(test_obs_id)
assert executors is not None
assert len(executors) == 1
assert isinstance(executors[0], ec.MetaDeleteCreate)
assert repo_client_mock.read.called, 'mock should be called'
assert repo_client_mock.read.reset()
test_config.task_types = [mc.TaskType.INGEST_OBS]
test_config.use_local_files = False
ec.CaomExecute.repo_cmd_get_client = Mock(
return_value=_read_obs(test_obs_id)
)
test_oe = ec.OrganizeExecutes(
test_config,
TEST_APP,
[],
[],
cadc_client=Mock(autospec=True),
caom_client=repo_client_mock,
)
executors = test_oe.choose(test_obs_id)
assert executors is not None
assert len(executors) == 1
assert isinstance(executors[0], ec.MetaUpdateObservation)
assert repo_client_mock.read.called, 'mock should be called'
assert executors[0].url == 'https://test_url/', 'url'
assert executors[0].fname is None, 'file name'
assert executors[0].stream == 'TEST', 'stream'
assert executors[0].lineage is None, 'lineage'
assert executors[0].external_urls_param == '', 'external_url_params'
assert (
executors[0].working_dir == f'{tc.THIS_DIR}/test_obs_id'
), 'working_dir'
assert test_oe.todo_fqn == f'{tc.THIS_DIR}/todo.txt', 'wrong todo'
finally:
mc.exec_cmd_orig = exec_cmd_orig
def test_organize_executes_chooser(test_config):
test_obs_id = tc.TestStorageName()
test_config.use_local_files = True
log_file_directory = os.path.join(tc.THIS_DIR, 'logs')
test_config.log_file_directory = log_file_directory
test_config.features.supports_composite = True
exec_cmd_orig = mc.exec_cmd_info
caom_client = Mock(autospec=True)
caom_client.read.side_effect = _read_obs2
try:
mc.exec_cmd_info = \
Mock(
return_value='INFO:cadc-data:info\n'
'File C170324_0054_SCI_prev.jpg:\n'
' archive: OMM\n'
' encoding: None\n'
' lastmod: Mon, 25 Jun 2018 16:52:07 GMT\n'
' md5sum: f37d21c53055498d1b5cb7753e1c6d6f\n'
' name: C120902_sh2-132_J_old_'
'SCIRED.fits.gz\n'
' size: 754408\n'
' type: image/jpeg\n'
' umd5sum: 704b494a972eed30b18b817e243ced7d\n'
' usize: 754408\n'.encode('utf-8')
)
test_config.task_types = [mc.TaskType.INGEST]
test_chooser = tc.TestChooser()
test_oe = ec.OrganizeExecutes(
test_config,
'command_name',
[],
[],
test_chooser,
cadc_client=Mock(autospec=True, return_value=None),
caom_client=caom_client,
)
executors = test_oe.choose(test_obs_id)
assert executors is not None
assert len(executors) == 1
assert isinstance(executors[0], ec.LocalMetaDeleteCreate)
assert executors[0].fname == 'test_obs_id.fits', 'file name'
assert executors[0].stream == 'TEST', 'stream'
assert executors[0].working_dir == tc.THIS_DIR, 'working_dir'
assert caom_client.read.called, 'read should be called'
caom_client.read.reset()
test_config.use_local_files = False
test_config.task_types = [mc.TaskType.INGEST]
test_oe = ec.OrganizeExecutes(
test_config,
'command_name',
[],
[],
test_chooser,
cadc_client=Mock(autospec=True, return_value=None),
caom_client=caom_client,
)
executors = test_oe.choose(test_obs_id)
assert executors is not None
assert len(executors) == 1
assert isinstance(executors[0], ec.MetaDeleteCreate)
assert caom_client.read.called, 'read should be called'
finally:
mc.exec_cmd_orig = exec_cmd_orig
def test_organize_executes_client_do_one(test_config):
test_obs_id = tc.TestStorageName()
test_config.use_local_files = True
log_file_directory = os.path.join(tc.THIS_DIR, 'logs')
test_config.log_file_directory = log_file_directory
success_log_file_name = 'success_log.txt'
test_config.success_log_file_name = success_log_file_name
failure_log_file_name = 'failure_log.txt'
test_config.failure_log_file_name = failure_log_file_name
test_config.features.use_clients = True
retry_file_name = 'retries.txt'
test_config.retry_file_name = retry_file_name
repo_client_mock = Mock(autospec=True)
repo_client_mock.read.return_value = None
test_config.task_types = [mc.TaskType.SCRAPE]
test_oe = ec.OrganizeExecutes(
test_config,
TEST_APP,
[],
[],
chooser=None,
cadc_client=Mock(autospec=True),
caom_client=repo_client_mock,
)
executors = test_oe.choose(test_obs_id)
assert executors is not None
assert len(executors) == 1
assert isinstance(executors[0], ec.ScrapeUpdate), f'{type(executors[0])}'
test_config.task_types = [
mc.TaskType.STORE,
mc.TaskType.INGEST,
mc.TaskType.MODIFY,
]
test_oe = ec.OrganizeExecutes(
test_config,
TEST_APP,
[],
[],
chooser=None,
cadc_client=Mock(autospec=True),
caom_client=repo_client_mock,
)
executors = test_oe.choose(test_obs_id)
assert executors is not None
assert len(executors) == 3
assert (isinstance(executors[0], ec.Store), type(executors[0]))
assert isinstance(executors[1], ec.LocalMetaCreate)
assert isinstance(executors[2], ec.LocalDataVisit)
assert repo_client_mock.read.called, 'mock should be called'
assert repo_client_mock.read.reset()
test_config.use_local_files = False
test_config.task_types = [mc.TaskType.INGEST, mc.TaskType.MODIFY]
test_oe = ec.OrganizeExecutes(
test_config,
TEST_APP,
[],
[],
chooser=None,
cadc_client=Mock(autospec=True),
caom_client=repo_client_mock,
)
executors = test_oe.choose(test_obs_id)
assert executors is not None
assert len(executors) == 2
assert isinstance(executors[0], ec.MetaCreate)
assert isinstance(executors[1], ec.DataVisit)
assert repo_client_mock.read.called, 'mock should be called'
test_config.use_local_files = True
test_config.task_types = [mc.TaskType.INGEST, mc.TaskType.MODIFY]
test_oe = ec.OrganizeExecutes(
test_config,
TEST_APP,
[],
[],
chooser=None,
cadc_client=Mock(autospec=True),
caom_client=repo_client_mock,
)
executors = test_oe.choose(test_obs_id)
assert executors is not None
assert len(executors) == 2
assert isinstance(executors[0], ec.LocalMetaCreate)
assert isinstance(executors[1], ec.LocalDataVisit)
assert repo_client_mock.read.called, 'mock should be called'
assert repo_client_mock.read.reset()
repo_client_mock.read.side_effect = _read_obs2
test_config.task_types = [mc.TaskType.SCRAPE, mc.TaskType.MODIFY]
test_config.use_local_files = True
test_oe = ec.OrganizeExecutes(
test_config,
TEST_APP,
[],
[],
chooser=None,
cadc_client=Mock(autospec=True),
caom_client=repo_client_mock,
)
executors = test_oe.choose(test_obs_id)
assert executors is not None
assert len(executors) == 2
assert isinstance(executors[0], ec.ScrapeUpdate)
assert isinstance(executors[1], ec.DataScrape)
assert repo_client_mock.read.called, 'mock should be called'
assert repo_client_mock.read.reset()
test_config.task_types = [mc.TaskType.INGEST]
test_config.use_local_files = False
test_chooser = tc.TestChooser()
ec.CaomExecute.repo_cmd_get_client = Mock(return_value=_read_obs(None))
test_oe = ec.OrganizeExecutes(
test_config,
TEST_APP,
[],
[],
chooser=test_chooser,
cadc_client=Mock(autospec=True),
caom_client=repo_client_mock,
)
executors = test_oe.choose(test_obs_id)
assert executors is not None
assert len(executors) == 1
assert isinstance(executors[0], ec.MetaDeleteCreate)
assert repo_client_mock.read.called, 'mock should be called'
assert repo_client_mock.read.reset()
test_config.task_types = [mc.TaskType.INGEST_OBS]
test_config.use_local_files = False
ec.CaomExecute.repo_cmd_get_client = Mock(
return_value=_read_obs(test_obs_id))
test_oe = ec.OrganizeExecutes(
test_config,
TEST_APP,
[],
[],
cadc_client=Mock(autospec=True),
caom_client=repo_client_mock,
)
executors = test_oe.choose(test_obs_id)
assert executors is not None
assert len(executors) == 1
assert isinstance(executors[0], ec.MetaUpdateObservation)
assert repo_client_mock.read.called, 'mock should be called'
assert executors[0].stream == 'TEST', 'stream'
assert executors[0].external_urls_param == '', 'external_url_params'
assert (executors[0].working_dir == f'{tc.THIS_DIR}/test_obs_id'
), 'working_dir'
assert test_oe.todo_fqn == f'{tc.THIS_DIR}/todo.txt', 'wrong todo'