def test_storage_name(): sn = mc.StorageName( obs_id='test_obs_id', collection='TEST', collection_pattern='T[\\w+-]+', ) assert sn.file_uri == 'ad:TEST/test_obs_id.fits.gz' assert sn.file_name == 'test_obs_id.fits' assert sn.compressed_file_name == 'test_obs_id.fits.gz' assert sn.model_file_name == 'test_obs_id.xml' assert sn.prev == 'test_obs_id_prev.jpg' assert sn.thumb == 'test_obs_id_prev_256.jpg' assert sn.prev_uri == 'ad:TEST/test_obs_id_prev.jpg' assert sn.thumb_uri == 'ad:TEST/test_obs_id_prev_256.jpg' assert sn.obs_id == 'test_obs_id' assert sn.log_file == 'test_obs_id.log' assert sn.product_id == 'test_obs_id' assert sn.fname_on_disk is None assert not sn.is_valid() sn = mc.StorageName( obs_id='Test_obs_id', collection='TEST', collection_pattern='T[\\w+-]+', ) assert sn.is_valid() x = mc.StorageName.remove_extensions('test_obs_id.fits.header.gz') assert x == 'test_obs_id'
def build(self, entry): return mc.StorageName( obs_id=mc.StorageName.remove_extensions(entry), collection=self._collection, fname_on_disk=entry, entry=entry, )
def _process_entry(self, entry): self._logger.debug(f'Begin _process_entry for {entry}.') storage_name = None try: storage_name = self._builder.build(entry) if storage_name.is_valid(): result = self._organizer.do_one(storage_name) else: self._logger.error( f'{storage_name.obs_id} failed naming validation check.') self._organizer.capture_failure( storage_name, BaseException('Invalid name format'), 'Invalid name format.', ) result = -1 except Exception as e: if storage_name is None: # keep going through storage name build failures self._logger.debug(traceback.format_exc()) self._logger.warning( f'StorageName construction failed. Using a default ' f'instance for {entry}, for logging only.') storage_name = mc.StorageName(obs_id=entry) self._organizer.capture_failure(storage_name, e, traceback.format_exc()) self._logger.info( f'Execution failed for {storage_name.entry} with {e}') self._logger.debug(traceback.format_exc()) # keep processing the rest of the entries, so don't throw # this or any other exception at this point result = -1 self._logger.debug(f'End _process_entry.') return result
def test_run_single(do_mock, test_config): _clean_up_log_files(test_config) progress_file = os.path.join(tc.TEST_DATA_DIR, 'progress.txt') test_config.features.expects_retry = False test_config.progress_fqn = progress_file test_config.state_fqn = STATE_FILE test_config.interval = 5 test_state = mc.State(test_config.state_fqn) test_state.save_state('gemini_timestamp', datetime.utcnow()) do_mock.return_value = -1 test_url = 'http://localhost/test_url.fits' test_storage_name = mc.StorageName(url=test_url) test_result = rc.run_single( test_config, test_storage_name, 'test_command', meta_visitors=None, data_visitors=None, ) assert test_result is not None, 'expect a result' assert test_result == -1, 'wrong result' assert do_mock.called, 'do mock not called' assert do_mock.call_count == 1, do_mock.call_count args, kwargs = do_mock.call_args test_storage = args[0] assert isinstance(test_storage, mc.StorageName), type(test_storage) assert test_storage.obs_id is None, 'wrong obs id' assert test_storage.url == test_url, test_storage.url
def cgps_run_single(): import sys config = mc.Config() config.get_executors() if config.features.run_in_airflow: temp = tempfile.NamedTemporaryFile() mc.write_to_file(temp.name, sys.argv[2]) config.proxy = temp.name else: config.proxy = sys.argv[2] if config.features.use_file_names: storage_name = mc.StorageName(file_name=sys.argv[1]) else: storage_name = mc.StorageName(obs_id=sys.argv[1]) ec.run_single(config, storage_name, APPLICATION, meta_visitors, data_visitors)
def test_data_visit_params(): test_wd = '/tmp/abc' if os.path.exists(test_wd): if os.path.isdir(test_wd): os.rmdir(test_wd) else: os.unlink(test_wd) storage_name = mc.StorageName( obs_id='abc', fname_on_disk='abc.fits.gz', source_names=['vos:DAO/incoming/abc.fits.gz'], destination_uris=['ad:TEST/abc.fits.gz'], ) test_config = mc.Config() test_config.task_types = [mc.TaskType.MODIFY] test_config.working_directory = '/tmp' test_config.logging_level = 'DEBUG' test_cadc_client = Mock(autospec=True) test_caom_client = Mock(autospec=True) test_caom_client.read.side_effect = _read_obs2 data_visitor = Mock(autospec=True) test_data_visitors = [data_visitor] test_observable = Mock(autospec=True) test_transferrer = Mock(autospec=True) try: test_config.use_local_files = False test_subject = ec.DataVisit( test_config, storage_name, test_cadc_client, test_caom_client, test_data_visitors, test_config.task_types[0], test_observable, test_transferrer, ) assert test_subject is not None, 'broken ctor' test_subject.execute(context=None) assert data_visitor.visit.called, 'expect visit call' data_visitor.visit.assert_called_with( ANY, working_directory='/tmp/abc', storage_name=storage_name, log_file_directory=None, cadc_client=ANY, caom_repo_client=ANY, stream=None, observable=ANY, ), f'wrong visit params {storage_name.source_names}' data_visitor.visit.reset_mock() finally: if os.path.exists(test_wd): dir_listing = os.listdir(test_wd) for f in dir_listing: os.unlink(f) os.rmdir(test_wd)
def test_data_visit(get_mock, test_config): get_mock.side_effect = Mock(autospec=True) test_data_client = Mock(autospec=True) test_repo_client = Mock(autospec=True) test_repo_client.read.side_effect = tc.mock_read dv_mock = Mock(autospec=True) test_data_visitors = [dv_mock] test_observable = Mock(autospec=True) test_sn = mc.StorageName( obs_id='test_obs_id', collection='TEST', collection_pattern='T[\\w+-]+' ) test_cred_param = '' test_transferrer = transfer_composable.CadcTransfer() test_transferrer.cadc_client = test_data_client test_transferrer.observable = test_observable test_subject = ec.DataVisit( test_config, test_sn, test_cred_param, test_data_client, test_repo_client, test_data_visitors, mc.TaskType.VISIT, test_observable, test_transferrer, ) test_subject.execute(None) assert get_mock.called, 'should be called' args, kwargs = get_mock.call_args assert args[1] == f'{tc.THIS_DIR}/test_obs_id', 'wrong directory' assert args[2] == 'test_obs_id.fits', 'wrong file name' assert args[3] == 'TEST', 'wrong archive' test_repo_client.read.assert_called_with( 'OMM', 'test_obs_id' ), 'wrong values' assert test_repo_client.update.called, 'expect an execution' # TODO - why is the log file directory NOT the working directory? args, kwargs = dv_mock.visit.call_args assert kwargs.get('working_directory') == f'{tc.THIS_DIR}/test_obs_id' assert kwargs.get('science_file') == 'test_obs_id.fits' assert kwargs.get('log_file_directory') == tc.TEST_DATA_DIR assert kwargs.get('stream') == 'TEST'
def test_data_visit(client_mock, test_config): client_mock.get.side_effect = Mock(autospec=True) test_repo_client = Mock(autospec=True) test_repo_client.read.side_effect = tc.mock_read dv_mock = Mock(autospec=True) test_data_visitors = [dv_mock] test_observable = Mock(autospec=True) test_sn = mc.StorageName( obs_id='test_obs_id', collection='TEST', collection_pattern='T[\\w+-]+', ) test_sn.source_names = ['ad:TEST/test_obs_id.fits'] test_sn.destination_uris = test_sn.source_names test_transferrer = transfer_composable.CadcTransfer() test_transferrer.cadc_client = client_mock test_transferrer.observable = test_observable test_subject = ec.DataVisit( test_config, test_sn, client_mock, test_repo_client, test_data_visitors, mc.TaskType.VISIT, test_observable, test_transferrer, ) test_subject.execute(None) assert client_mock.get.called, 'should be called' client_mock.get.assert_called_with( f'{tc.THIS_DIR}/test_obs_id', test_sn.destination_uris[0]), 'wrong get call args' test_repo_client.read.assert_called_with('OMM', 'test_obs_id'), 'wrong values' assert test_repo_client.update.called, 'expect an execution' # TODO - why is the log file directory NOT the working directory? args, kwargs = dv_mock.visit.call_args assert kwargs.get('working_directory') == f'{tc.THIS_DIR}/test_obs_id' assert ( kwargs.get('storage_name') == test_sn), 'wrong storage name parameter' assert kwargs.get('log_file_directory') == tc.TEST_DATA_DIR assert kwargs.get('stream') == 'TEST'