def test_storage_name():
sn = mc.StorageName(
obs_id='test_obs_id',
collection='TEST',
collection_pattern='T[\\w+-]+',
)
assert sn.file_uri == 'ad:TEST/test_obs_id.fits.gz'
assert sn.file_name == 'test_obs_id.fits'
assert sn.compressed_file_name == 'test_obs_id.fits.gz'
assert sn.model_file_name == 'test_obs_id.xml'
assert sn.prev == 'test_obs_id_prev.jpg'
assert sn.thumb == 'test_obs_id_prev_256.jpg'
assert sn.prev_uri == 'ad:TEST/test_obs_id_prev.jpg'
assert sn.thumb_uri == 'ad:TEST/test_obs_id_prev_256.jpg'
assert sn.obs_id == 'test_obs_id'
assert sn.log_file == 'test_obs_id.log'
assert sn.product_id == 'test_obs_id'
assert sn.fname_on_disk is None
assert not sn.is_valid()
sn = mc.StorageName(
obs_id='Test_obs_id',
collection='TEST',
collection_pattern='T[\\w+-]+',
)
assert sn.is_valid()
x = mc.StorageName.remove_extensions('test_obs_id.fits.header.gz')
assert x == 'test_obs_id'
def build(self, entry):
return mc.StorageName(
obs_id=mc.StorageName.remove_extensions(entry),
collection=self._collection,
fname_on_disk=entry,
entry=entry,
)
def _process_entry(self, entry):
self._logger.debug(f'Begin _process_entry for {entry}.')
storage_name = None
try:
storage_name = self._builder.build(entry)
if storage_name.is_valid():
result = self._organizer.do_one(storage_name)
else:
self._logger.error(
f'{storage_name.obs_id} failed naming validation check.')
self._organizer.capture_failure(
storage_name,
BaseException('Invalid name format'),
'Invalid name format.',
)
result = -1
except Exception as e:
if storage_name is None:
# keep going through storage name build failures
self._logger.debug(traceback.format_exc())
self._logger.warning(
f'StorageName construction failed. Using a default '
f'instance for {entry}, for logging only.')
storage_name = mc.StorageName(obs_id=entry)
self._organizer.capture_failure(storage_name, e,
traceback.format_exc())
self._logger.info(
f'Execution failed for {storage_name.entry} with {e}')
self._logger.debug(traceback.format_exc())
# keep processing the rest of the entries, so don't throw
# this or any other exception at this point
result = -1
self._logger.debug(f'End _process_entry.')
return result
def test_run_single(do_mock, test_config):
_clean_up_log_files(test_config)
progress_file = os.path.join(tc.TEST_DATA_DIR, 'progress.txt')
test_config.features.expects_retry = False
test_config.progress_fqn = progress_file
test_config.state_fqn = STATE_FILE
test_config.interval = 5
test_state = mc.State(test_config.state_fqn)
test_state.save_state('gemini_timestamp', datetime.utcnow())
do_mock.return_value = -1
test_url = 'http://localhost/test_url.fits'
test_storage_name = mc.StorageName(url=test_url)
test_result = rc.run_single(
test_config,
test_storage_name,
'test_command',
meta_visitors=None,
data_visitors=None,
)
assert test_result is not None, 'expect a result'
assert test_result == -1, 'wrong result'
assert do_mock.called, 'do mock not called'
assert do_mock.call_count == 1, do_mock.call_count
args, kwargs = do_mock.call_args
test_storage = args[0]
assert isinstance(test_storage, mc.StorageName), type(test_storage)
assert test_storage.obs_id is None, 'wrong obs id'
assert test_storage.url == test_url, test_storage.url
def cgps_run_single():
import sys
config = mc.Config()
config.get_executors()
if config.features.run_in_airflow:
temp = tempfile.NamedTemporaryFile()
mc.write_to_file(temp.name, sys.argv[2])
config.proxy = temp.name
else:
config.proxy = sys.argv[2]
if config.features.use_file_names:
storage_name = mc.StorageName(file_name=sys.argv[1])
else:
storage_name = mc.StorageName(obs_id=sys.argv[1])
ec.run_single(config, storage_name, APPLICATION, meta_visitors,
data_visitors)
def test_data_visit_params():
test_wd = '/tmp/abc'
if os.path.exists(test_wd):
if os.path.isdir(test_wd):
os.rmdir(test_wd)
else:
os.unlink(test_wd)
storage_name = mc.StorageName(
obs_id='abc',
fname_on_disk='abc.fits.gz',
source_names=['vos:DAO/incoming/abc.fits.gz'],
destination_uris=['ad:TEST/abc.fits.gz'],
)
test_config = mc.Config()
test_config.task_types = [mc.TaskType.MODIFY]
test_config.working_directory = '/tmp'
test_config.logging_level = 'DEBUG'
test_cadc_client = Mock(autospec=True)
test_caom_client = Mock(autospec=True)
test_caom_client.read.side_effect = _read_obs2
data_visitor = Mock(autospec=True)
test_data_visitors = [data_visitor]
test_observable = Mock(autospec=True)
test_transferrer = Mock(autospec=True)
try:
test_config.use_local_files = False
test_subject = ec.DataVisit(
test_config,
storage_name,
test_cadc_client,
test_caom_client,
test_data_visitors,
test_config.task_types[0],
test_observable,
test_transferrer,
)
assert test_subject is not None, 'broken ctor'
test_subject.execute(context=None)
assert data_visitor.visit.called, 'expect visit call'
data_visitor.visit.assert_called_with(
ANY,
working_directory='/tmp/abc',
storage_name=storage_name,
log_file_directory=None,
cadc_client=ANY,
caom_repo_client=ANY,
stream=None,
observable=ANY,
), f'wrong visit params {storage_name.source_names}'
data_visitor.visit.reset_mock()
finally:
if os.path.exists(test_wd):
dir_listing = os.listdir(test_wd)
for f in dir_listing:
os.unlink(f)
os.rmdir(test_wd)
def test_data_visit(get_mock, test_config):
get_mock.side_effect = Mock(autospec=True)
test_data_client = Mock(autospec=True)
test_repo_client = Mock(autospec=True)
test_repo_client.read.side_effect = tc.mock_read
dv_mock = Mock(autospec=True)
test_data_visitors = [dv_mock]
test_observable = Mock(autospec=True)
test_sn = mc.StorageName(
obs_id='test_obs_id', collection='TEST', collection_pattern='T[\\w+-]+'
)
test_cred_param = ''
test_transferrer = transfer_composable.CadcTransfer()
test_transferrer.cadc_client = test_data_client
test_transferrer.observable = test_observable
test_subject = ec.DataVisit(
test_config,
test_sn,
test_cred_param,
test_data_client,
test_repo_client,
test_data_visitors,
mc.TaskType.VISIT,
test_observable,
test_transferrer,
)
test_subject.execute(None)
assert get_mock.called, 'should be called'
args, kwargs = get_mock.call_args
assert args[1] == f'{tc.THIS_DIR}/test_obs_id', 'wrong directory'
assert args[2] == 'test_obs_id.fits', 'wrong file name'
assert args[3] == 'TEST', 'wrong archive'
test_repo_client.read.assert_called_with(
'OMM', 'test_obs_id'
), 'wrong values'
assert test_repo_client.update.called, 'expect an execution'
# TODO - why is the log file directory NOT the working directory?
args, kwargs = dv_mock.visit.call_args
assert kwargs.get('working_directory') == f'{tc.THIS_DIR}/test_obs_id'
assert kwargs.get('science_file') == 'test_obs_id.fits'
assert kwargs.get('log_file_directory') == tc.TEST_DATA_DIR
assert kwargs.get('stream') == 'TEST'
def test_data_visit(client_mock, test_config):
client_mock.get.side_effect = Mock(autospec=True)
test_repo_client = Mock(autospec=True)
test_repo_client.read.side_effect = tc.mock_read
dv_mock = Mock(autospec=True)
test_data_visitors = [dv_mock]
test_observable = Mock(autospec=True)
test_sn = mc.StorageName(
obs_id='test_obs_id',
collection='TEST',
collection_pattern='T[\\w+-]+',
)
test_sn.source_names = ['ad:TEST/test_obs_id.fits']
test_sn.destination_uris = test_sn.source_names
test_transferrer = transfer_composable.CadcTransfer()
test_transferrer.cadc_client = client_mock
test_transferrer.observable = test_observable
test_subject = ec.DataVisit(
test_config,
test_sn,
client_mock,
test_repo_client,
test_data_visitors,
mc.TaskType.VISIT,
test_observable,
test_transferrer,
)
test_subject.execute(None)
assert client_mock.get.called, 'should be called'
client_mock.get.assert_called_with(
f'{tc.THIS_DIR}/test_obs_id',
test_sn.destination_uris[0]), 'wrong get call args'
test_repo_client.read.assert_called_with('OMM',
'test_obs_id'), 'wrong values'
assert test_repo_client.update.called, 'expect an execution'
# TODO - why is the log file directory NOT the working directory?
args, kwargs = dv_mock.visit.call_args
assert kwargs.get('working_directory') == f'{tc.THIS_DIR}/test_obs_id'
assert (
kwargs.get('storage_name') == test_sn), 'wrong storage name parameter'
assert kwargs.get('log_file_directory') == tc.TEST_DATA_DIR
assert kwargs.get('stream') == 'TEST'