def _set_modify_transfer(modify_transfer, config): if modify_transfer is None: if not config.use_local_files: if config.features.supports_latest_client: modify_transfer = transfer_composable.VoTransfer() else: modify_transfer = transfer_composable.CadcTransfer() return modify_transfer
def test_cadc_transfer(client_mock, caps_mock): caps_mock.return_value = 'https://sc2.canfar.net/sc2repo' test_subject = tc.CadcTransfer() assert test_subject is not None, 'expect a result' test_config = mc.Config() test_config.rejected_fqn = '/tmp/rejected.yml' test_observable = mc.Observable(mc.Rejected(test_config.rejected_fqn), mc.Metrics(test_config)) test_subject.observable = test_observable test_subject.cadc_client = client_mock test_source = 'ad:TEST/test_file.fits' test_destination = '/tmp/test_file.fits' test_subject.get(test_source, test_destination) assert client_mock.get.called, 'should have been called' client_mock.get.assert_called_with( '/tmp', 'ad:TEST/test_file.fits'), 'wrong parameters'
def test_data_visit(get_mock, test_config): get_mock.side_effect = Mock(autospec=True) test_data_client = Mock(autospec=True) test_repo_client = Mock(autospec=True) test_repo_client.read.side_effect = tc.mock_read dv_mock = Mock(autospec=True) test_data_visitors = [dv_mock] test_observable = Mock(autospec=True) test_sn = mc.StorageName( obs_id='test_obs_id', collection='TEST', collection_pattern='T[\\w+-]+' ) test_cred_param = '' test_transferrer = transfer_composable.CadcTransfer() test_transferrer.cadc_client = test_data_client test_transferrer.observable = test_observable test_subject = ec.DataVisit( test_config, test_sn, test_cred_param, test_data_client, test_repo_client, test_data_visitors, mc.TaskType.VISIT, test_observable, test_transferrer, ) test_subject.execute(None) assert get_mock.called, 'should be called' args, kwargs = get_mock.call_args assert args[1] == f'{tc.THIS_DIR}/test_obs_id', 'wrong directory' assert args[2] == 'test_obs_id.fits', 'wrong file name' assert args[3] == 'TEST', 'wrong archive' test_repo_client.read.assert_called_with( 'OMM', 'test_obs_id' ), 'wrong values' assert test_repo_client.update.called, 'expect an execution' # TODO - why is the log file directory NOT the working directory? args, kwargs = dv_mock.visit.call_args assert kwargs.get('working_directory') == f'{tc.THIS_DIR}/test_obs_id' assert kwargs.get('science_file') == 'test_obs_id.fits' assert kwargs.get('log_file_directory') == tc.TEST_DATA_DIR assert kwargs.get('stream') == 'TEST'
def test_data_visit(client_mock, test_config): client_mock.get.side_effect = Mock(autospec=True) test_repo_client = Mock(autospec=True) test_repo_client.read.side_effect = tc.mock_read dv_mock = Mock(autospec=True) test_data_visitors = [dv_mock] test_observable = Mock(autospec=True) test_sn = mc.StorageName( obs_id='test_obs_id', collection='TEST', collection_pattern='T[\\w+-]+', ) test_sn.source_names = ['ad:TEST/test_obs_id.fits'] test_sn.destination_uris = test_sn.source_names test_transferrer = transfer_composable.CadcTransfer() test_transferrer.cadc_client = client_mock test_transferrer.observable = test_observable test_subject = ec.DataVisit( test_config, test_sn, client_mock, test_repo_client, test_data_visitors, mc.TaskType.VISIT, test_observable, test_transferrer, ) test_subject.execute(None) assert client_mock.get.called, 'should be called' client_mock.get.assert_called_with( f'{tc.THIS_DIR}/test_obs_id', test_sn.destination_uris[0]), 'wrong get call args' test_repo_client.read.assert_called_with('OMM', 'test_obs_id'), 'wrong values' assert test_repo_client.update.called, 'expect an execution' # TODO - why is the log file directory NOT the working directory? args, kwargs = dv_mock.visit.call_args assert kwargs.get('working_directory') == f'{tc.THIS_DIR}/test_obs_id' assert ( kwargs.get('storage_name') == test_sn), 'wrong storage name parameter' assert kwargs.get('log_file_directory') == tc.TEST_DATA_DIR assert kwargs.get('stream') == 'TEST'
def test_cadc_transfer(data_get_mock, caps_mock): caps_mock.return_value = 'https://sc2.canfar.net/sc2repo' data_get_mock.side_effect = Mock(autospec=True) test_subject = tc.CadcTransfer() assert test_subject is not None, 'expect a result' test_config = mc.Config() test_config.rejected_fqn = '/tmp/rejected.yml' test_observable = mc.Observable(mc.Rejected(test_config.rejected_fqn), mc.Metrics(test_config)) test_subject.observable = test_observable test_subject.cadc_client = Mock() test_source = 'ad:TEST/test_file.fits' test_destination = '/tmp/test_file.fits' test_subject.get(test_source, test_destination) assert data_get_mock.called, 'should have been called' args, kwargs = data_get_mock.call_args assert args[1] == '/tmp', 'wrong dir name' assert args[2] == 'test_file.fits', 'wrong file name' assert args[3] == 'TEST', 'wrong archive name'
def _set_modify_transfer(modify_transfer, config, client): if modify_transfer is None: if not config.use_local_files: modify_transfer = transfer_composable.CadcTransfer() modify_transfer.client = client return modify_transfer
def run_by_state_ad( config=None, name_builder=None, command_name=None, bookmark_name=None, meta_visitors=[], data_visitors=[], end_time=None, chooser=None, source=None, transferrer=None, ): """A default implementation for using the StateRunner. :param config Config instance :param name_builder NameBuilder extension that creates an instance of a StorageName extension, from an entry from a DataSourceComposable listing :param command_name string that represents the specific pipeline application name :param bookmark_name string that represents the state.yml lookup value :param meta_visitors list of modules with visit methods, that expect the metadata of a work file to exist on disk :param data_visitors list of modules with visit methods, that expect the work file to exist on disk :param end_time datetime for stopping a run, should be in UTC. :param chooser OrganizerChooser, if there's strange rules about file naming. :param source DataSourceComposable extension that identifies work to be done. :param transferrer Transfer extension that identifies how to retrieve data from a source. """ if config is None: config = mc.Config() config.get_executors() _set_logging(config) cadc_client, caom_client = _set_clients(config) if name_builder is None: name_builder = name_builder_composable.StorageNameInstanceBuilder( config.collection) if source is None: source = data_source_composable.QueryTimeBoxDataSource(config) if end_time is None: end_time = get_utc_now() if transferrer is None: if config.use_local_files: transferrer = transfer_composable.Transfer() else: transferrer = transfer_composable.CadcTransfer() organizer = ec.OrganizeExecutes( config, command_name, meta_visitors, data_visitors, chooser, transferrer, cadc_client=cadc_client, caom_client=caom_client, ) runner = StateRunner(config, organizer, name_builder, source, bookmark_name, end_time) result = runner.run() result |= runner.run_retry() runner.report() return result