def _set_modify_transfer(modify_transfer, config):
if modify_transfer is None:
if not config.use_local_files:
if config.features.supports_latest_client:
modify_transfer = transfer_composable.VoTransfer()
else:
modify_transfer = transfer_composable.CadcTransfer()
return modify_transfer
def test_cadc_transfer(client_mock, caps_mock):
caps_mock.return_value = 'https://sc2.canfar.net/sc2repo'
test_subject = tc.CadcTransfer()
assert test_subject is not None, 'expect a result'
test_config = mc.Config()
test_config.rejected_fqn = '/tmp/rejected.yml'
test_observable = mc.Observable(mc.Rejected(test_config.rejected_fqn),
mc.Metrics(test_config))
test_subject.observable = test_observable
test_subject.cadc_client = client_mock
test_source = 'ad:TEST/test_file.fits'
test_destination = '/tmp/test_file.fits'
test_subject.get(test_source, test_destination)
assert client_mock.get.called, 'should have been called'
client_mock.get.assert_called_with(
'/tmp', 'ad:TEST/test_file.fits'), 'wrong parameters'
def test_data_visit(get_mock, test_config):
get_mock.side_effect = Mock(autospec=True)
test_data_client = Mock(autospec=True)
test_repo_client = Mock(autospec=True)
test_repo_client.read.side_effect = tc.mock_read
dv_mock = Mock(autospec=True)
test_data_visitors = [dv_mock]
test_observable = Mock(autospec=True)
test_sn = mc.StorageName(
obs_id='test_obs_id', collection='TEST', collection_pattern='T[\\w+-]+'
)
test_cred_param = ''
test_transferrer = transfer_composable.CadcTransfer()
test_transferrer.cadc_client = test_data_client
test_transferrer.observable = test_observable
test_subject = ec.DataVisit(
test_config,
test_sn,
test_cred_param,
test_data_client,
test_repo_client,
test_data_visitors,
mc.TaskType.VISIT,
test_observable,
test_transferrer,
)
test_subject.execute(None)
assert get_mock.called, 'should be called'
args, kwargs = get_mock.call_args
assert args[1] == f'{tc.THIS_DIR}/test_obs_id', 'wrong directory'
assert args[2] == 'test_obs_id.fits', 'wrong file name'
assert args[3] == 'TEST', 'wrong archive'
test_repo_client.read.assert_called_with(
'OMM', 'test_obs_id'
), 'wrong values'
assert test_repo_client.update.called, 'expect an execution'
# TODO - why is the log file directory NOT the working directory?
args, kwargs = dv_mock.visit.call_args
assert kwargs.get('working_directory') == f'{tc.THIS_DIR}/test_obs_id'
assert kwargs.get('science_file') == 'test_obs_id.fits'
assert kwargs.get('log_file_directory') == tc.TEST_DATA_DIR
assert kwargs.get('stream') == 'TEST'
def test_data_visit(client_mock, test_config):
client_mock.get.side_effect = Mock(autospec=True)
test_repo_client = Mock(autospec=True)
test_repo_client.read.side_effect = tc.mock_read
dv_mock = Mock(autospec=True)
test_data_visitors = [dv_mock]
test_observable = Mock(autospec=True)
test_sn = mc.StorageName(
obs_id='test_obs_id',
collection='TEST',
collection_pattern='T[\\w+-]+',
)
test_sn.source_names = ['ad:TEST/test_obs_id.fits']
test_sn.destination_uris = test_sn.source_names
test_transferrer = transfer_composable.CadcTransfer()
test_transferrer.cadc_client = client_mock
test_transferrer.observable = test_observable
test_subject = ec.DataVisit(
test_config,
test_sn,
client_mock,
test_repo_client,
test_data_visitors,
mc.TaskType.VISIT,
test_observable,
test_transferrer,
)
test_subject.execute(None)
assert client_mock.get.called, 'should be called'
client_mock.get.assert_called_with(
f'{tc.THIS_DIR}/test_obs_id',
test_sn.destination_uris[0]), 'wrong get call args'
test_repo_client.read.assert_called_with('OMM',
'test_obs_id'), 'wrong values'
assert test_repo_client.update.called, 'expect an execution'
# TODO - why is the log file directory NOT the working directory?
args, kwargs = dv_mock.visit.call_args
assert kwargs.get('working_directory') == f'{tc.THIS_DIR}/test_obs_id'
assert (
kwargs.get('storage_name') == test_sn), 'wrong storage name parameter'
assert kwargs.get('log_file_directory') == tc.TEST_DATA_DIR
assert kwargs.get('stream') == 'TEST'
def test_cadc_transfer(data_get_mock, caps_mock):
caps_mock.return_value = 'https://sc2.canfar.net/sc2repo'
data_get_mock.side_effect = Mock(autospec=True)
test_subject = tc.CadcTransfer()
assert test_subject is not None, 'expect a result'
test_config = mc.Config()
test_config.rejected_fqn = '/tmp/rejected.yml'
test_observable = mc.Observable(mc.Rejected(test_config.rejected_fqn),
mc.Metrics(test_config))
test_subject.observable = test_observable
test_subject.cadc_client = Mock()
test_source = 'ad:TEST/test_file.fits'
test_destination = '/tmp/test_file.fits'
test_subject.get(test_source, test_destination)
assert data_get_mock.called, 'should have been called'
args, kwargs = data_get_mock.call_args
assert args[1] == '/tmp', 'wrong dir name'
assert args[2] == 'test_file.fits', 'wrong file name'
assert args[3] == 'TEST', 'wrong archive name'
def _set_modify_transfer(modify_transfer, config, client):
if modify_transfer is None:
if not config.use_local_files:
modify_transfer = transfer_composable.CadcTransfer()
modify_transfer.client = client
return modify_transfer
def run_by_state_ad(
config=None,
name_builder=None,
command_name=None,
bookmark_name=None,
meta_visitors=[],
data_visitors=[],
end_time=None,
chooser=None,
source=None,
transferrer=None,
):
"""A default implementation for using the StateRunner.
:param config Config instance
:param name_builder NameBuilder extension that creates an instance of
a StorageName extension, from an entry from a DataSourceComposable
listing
:param command_name string that represents the specific pipeline
application name
:param bookmark_name string that represents the state.yml lookup value
:param meta_visitors list of modules with visit methods, that expect
the metadata of a work file to exist on disk
:param data_visitors list of modules with visit methods, that expect the
work file to exist on disk
:param end_time datetime for stopping a run, should be in UTC.
:param chooser OrganizerChooser, if there's strange rules about file
naming.
:param source DataSourceComposable extension that identifies work to be
done.
:param transferrer Transfer extension that identifies how to retrieve
data from a source.
"""
if config is None:
config = mc.Config()
config.get_executors()
_set_logging(config)
cadc_client, caom_client = _set_clients(config)
if name_builder is None:
name_builder = name_builder_composable.StorageNameInstanceBuilder(
config.collection)
if source is None:
source = data_source_composable.QueryTimeBoxDataSource(config)
if end_time is None:
end_time = get_utc_now()
if transferrer is None:
if config.use_local_files:
transferrer = transfer_composable.Transfer()
else:
transferrer = transfer_composable.CadcTransfer()
organizer = ec.OrganizeExecutes(
config,
command_name,
meta_visitors,
data_visitors,
chooser,
transferrer,
cadc_client=cadc_client,
caom_client=caom_client,
)
runner = StateRunner(config, organizer, name_builder, source,
bookmark_name, end_time)
result = runner.run()
result |= runner.run_retry()
runner.report()
return result