def test_data_visit_params():
test_wd = '/tmp/abc'
if os.path.exists(test_wd):
if os.path.isdir(test_wd):
os.rmdir(test_wd)
else:
os.unlink(test_wd)
storage_name = mc.StorageName(
obs_id='abc',
fname_on_disk='abc.fits.gz',
source_names=['vos:DAO/incoming/abc.fits.gz'],
destination_uris=['ad:TEST/abc.fits.gz'],
)
test_config = mc.Config()
test_config.task_types = [mc.TaskType.MODIFY]
test_config.working_directory = '/tmp'
test_config.logging_level = 'DEBUG'
test_cadc_client = Mock(autospec=True)
test_caom_client = Mock(autospec=True)
test_caom_client.read.side_effect = _read_obs2
data_visitor = Mock(autospec=True)
test_data_visitors = [data_visitor]
test_observable = Mock(autospec=True)
test_transferrer = Mock(autospec=True)
try:
test_config.use_local_files = False
test_subject = ec.DataVisit(
test_config,
storage_name,
test_cadc_client,
test_caom_client,
test_data_visitors,
test_config.task_types[0],
test_observable,
test_transferrer,
)
assert test_subject is not None, 'broken ctor'
test_subject.execute(context=None)
assert data_visitor.visit.called, 'expect visit call'
data_visitor.visit.assert_called_with(
ANY,
working_directory='/tmp/abc',
storage_name=storage_name,
log_file_directory=None,
cadc_client=ANY,
caom_repo_client=ANY,
stream=None,
observable=ANY,
), f'wrong visit params {storage_name.source_names}'
data_visitor.visit.reset_mock()
finally:
if os.path.exists(test_wd):
dir_listing = os.listdir(test_wd)
for f in dir_listing:
os.unlink(f)
os.rmdir(test_wd)
def test_data_execute_v(test_config):
test_config.features.supports_latest_client = True
test_obs_id = 'test_obs_id'
test_dir = os.path.join(tc.THIS_DIR, test_obs_id)
test_fits_fqn = os.path.join(test_dir, tc.TestStorageName().file_name)
try:
if not os.path.exists(test_dir):
os.mkdir(test_dir, mode=0o755)
# precondition = open(test_fits_fqn, 'w')
# precondition.close()
test_data_visitors = [TestVisit]
repo_client_mock = Mock(autospec=True)
cadc_client_mock = Mock(autospec=True)
cadc_client_mock.copy.side_effect = tc.mock_copy_md5
test_observer = Mock(autospec=True)
test_cred = ''
test_transferrer = transfer_composable.VoTransfer()
test_transferrer.cadc_client = cadc_client_mock
ec.CaomExecute._data_cmd_info = Mock(side_effect=_get_fname)
repo_client_mock.read.side_effect = tc.mock_read
# run the test
test_executor = ec.DataVisit(
test_config,
tc.TestStorageName(),
test_cred,
cadc_client_mock,
repo_client_mock,
test_data_visitors,
mc.TaskType.MODIFY,
test_observer,
test_transferrer,
)
test_executor.execute(None)
# check that things worked as expected
assert repo_client_mock.read.called, 'read call missed'
assert repo_client_mock.update.called, 'update call missed'
assert test_observer.metrics.observe.called, 'observe not called'
assert cadc_client_mock.copy.called, 'copy not called'
cadc_client_mock.copy.assert_called_with(
'ad:TEST/test_obs_id.fits.gz', test_fits_fqn, send_md5=True
), 'wrong call args'
finally:
if os.path.exists(test_fits_fqn):
os.unlink(test_fits_fqn)
if os.path.exists(test_dir):
os.rmdir(test_dir)
def test_data_execute(test_config):
test_obs_id = 'test_obs_id'
test_dir = os.path.join(tc.THIS_DIR, test_obs_id)
test_fits_fqn = os.path.join(test_dir, tc.TestStorageName().file_name)
try:
if not os.path.exists(test_dir):
os.mkdir(test_dir, mode=0o755)
precondition = open(test_fits_fqn, 'w')
precondition.close()
logging.error(test_fits_fqn)
test_data_visitors = [TestVisit]
repo_client_mock = Mock()
data_client_mock = Mock()
test_observer = Mock()
test_cred = ''
test_transferrer = transfer_composable.VoTransfer()
test_transferrer.cadc_client = data_client_mock
ec.CaomExecute._data_cmd_info = Mock(side_effect=_get_fname)
repo_client_mock.read.side_effect = tc.mock_read
# run the test
test_executor = ec.DataVisit(
test_config,
tc.TestStorageName(),
test_cred,
data_client_mock,
repo_client_mock,
test_data_visitors,
mc.TaskType.MODIFY,
test_observer,
test_transferrer,
)
test_executor.execute(None)
# check that things worked as expected
assert repo_client_mock.read.called, 'read call missed'
assert repo_client_mock.update.called, 'update call missed'
assert test_observer.metrics.observe.called, 'observe not called'
finally:
if os.path.exists(test_fits_fqn):
os.unlink(test_fits_fqn)
if os.path.exists(test_dir):
os.rmdir(test_dir)
def test_data_visit(get_mock, test_config):
get_mock.side_effect = Mock(autospec=True)
test_data_client = Mock(autospec=True)
test_repo_client = Mock(autospec=True)
test_repo_client.read.side_effect = tc.mock_read
dv_mock = Mock(autospec=True)
test_data_visitors = [dv_mock]
test_observable = Mock(autospec=True)
test_sn = mc.StorageName(
obs_id='test_obs_id', collection='TEST', collection_pattern='T[\\w+-]+'
)
test_cred_param = ''
test_transferrer = transfer_composable.CadcTransfer()
test_transferrer.cadc_client = test_data_client
test_transferrer.observable = test_observable
test_subject = ec.DataVisit(
test_config,
test_sn,
test_cred_param,
test_data_client,
test_repo_client,
test_data_visitors,
mc.TaskType.VISIT,
test_observable,
test_transferrer,
)
test_subject.execute(None)
assert get_mock.called, 'should be called'
args, kwargs = get_mock.call_args
assert args[1] == f'{tc.THIS_DIR}/test_obs_id', 'wrong directory'
assert args[2] == 'test_obs_id.fits', 'wrong file name'
assert args[3] == 'TEST', 'wrong archive'
test_repo_client.read.assert_called_with(
'OMM', 'test_obs_id'
), 'wrong values'
assert test_repo_client.update.called, 'expect an execution'
# TODO - why is the log file directory NOT the working directory?
args, kwargs = dv_mock.visit.call_args
assert kwargs.get('working_directory') == f'{tc.THIS_DIR}/test_obs_id'
assert kwargs.get('science_file') == 'test_obs_id.fits'
assert kwargs.get('log_file_directory') == tc.TEST_DATA_DIR
assert kwargs.get('stream') == 'TEST'
def test_data_visit(client_mock, test_config):
client_mock.get.side_effect = Mock(autospec=True)
test_repo_client = Mock(autospec=True)
test_repo_client.read.side_effect = tc.mock_read
dv_mock = Mock(autospec=True)
test_data_visitors = [dv_mock]
test_observable = Mock(autospec=True)
test_sn = mc.StorageName(
obs_id='test_obs_id',
collection='TEST',
collection_pattern='T[\\w+-]+',
)
test_sn.source_names = ['ad:TEST/test_obs_id.fits']
test_sn.destination_uris = test_sn.source_names
test_transferrer = transfer_composable.CadcTransfer()
test_transferrer.cadc_client = client_mock
test_transferrer.observable = test_observable
test_subject = ec.DataVisit(
test_config,
test_sn,
client_mock,
test_repo_client,
test_data_visitors,
mc.TaskType.VISIT,
test_observable,
test_transferrer,
)
test_subject.execute(None)
assert client_mock.get.called, 'should be called'
client_mock.get.assert_called_with(
f'{tc.THIS_DIR}/test_obs_id',
test_sn.destination_uris[0]), 'wrong get call args'
test_repo_client.read.assert_called_with('OMM',
'test_obs_id'), 'wrong values'
assert test_repo_client.update.called, 'expect an execution'
# TODO - why is the log file directory NOT the working directory?
args, kwargs = dv_mock.visit.call_args
assert kwargs.get('working_directory') == f'{tc.THIS_DIR}/test_obs_id'
assert (
kwargs.get('storage_name') == test_sn), 'wrong storage name parameter'
assert kwargs.get('log_file_directory') == tc.TEST_DATA_DIR
assert kwargs.get('stream') == 'TEST'