def _mock_file_info(uri):
sci_result = FileInfo(id=uri, lastmod='Fri, 28 Dec 2018 01:43:28 GMT')
reject_result = FileInfo(id=uri, lastmod='Thu, 14 May 2020 20:29:02 GMT')
result = reject_result
if '_SCI' in uri:
result = sci_result
return result
def test_visitor(test_name):
storage_name = OmmName(file_name=basename(test_name).replace(
'.header', '.gz'), )
file_info = FileInfo(id=storage_name.file_uri,
file_type='application/fits')
headers = ac.make_headers_from_file(test_name)
metadata_reader = rdc.FileMetadataReader()
metadata_reader._headers = {storage_name.file_uri: headers}
metadata_reader._file_info = {storage_name.file_uri: file_info}
kwargs = {
'storage_name': storage_name,
'metadata_reader': metadata_reader,
}
observation = None
input_file = f'{TEST_DATA_DIR}/in.{storage_name.product_id}.fits.xml'
if exists(input_file):
observation = mc.read_obs_from_file(input_file)
observation = fits2caom2_augmentation.visit(observation, **kwargs)
expected_fqn = (f'{TEST_DATA_DIR}/{storage_name.file_id}.expected.xml')
expected = mc.read_obs_from_file(expected_fqn)
compare_result = get_differences(expected, observation)
if compare_result is not None:
actual_fqn = expected_fqn.replace('expected', 'actual')
mc.write_obs_to_file(observation, actual_fqn)
compare_text = '\n'.join([r for r in compare_result])
msg = (f'Differences found in observation {expected.observation_id}\n'
f'{compare_text}')
raise AssertionError(msg)
def test_todo_local_common(
caom_mock,
access_mock,
fits2caom2_mock,
test_input_name,
):
if test_input_name not in ['OMM_TODO_LOCAL', 'NEOSSAT_TODO_LOCAL']:
return
test_input = INPUTS.get(test_input_name)
_cleanup()
_setup(test_input, local=True)
access_mock.return_value = 'https://localhost'
getcwd_orig = os.getcwd
os.getcwd = Mock(return_value=TEST_EXEC_DIR)
# import the module for execution
sys.path.append(test_input.test_path)
test_module = import_module('composable')
caom_mock.return_value.data_client.info.side_effect = [
None,
FileInfo(
id=test_input.test_uri,
md5sum='3d29f0edd984065a044d1376a11c6f08',
),
]
caom_mock.return_value.metadata_client.read.side_effect = [
None,
SimpleObservation(
'obs_id',
test_input.collection,
Algorithm(name='exposure'),
),
]
def _info(uri):
assert (uri == test_input.test_uri), 'wrong info uri'
return FileInfo(
id=uri,
md5sum='abc',
size=42,
)
fits2caom2_mock.return_value.info.side_effect = _info
try:
test_result = test_module._run()
assert test_result is not None, f'expect a result {test_input_name}'
assert test_result == 0, f'wrong test result {test_input_name}'
assert (caom_mock.return_value.data_client.put.called
), f'{test_input_name} put not called'
caom_mock.return_value.data_client.put.assert_called_with(
'/usr/src/app/integration_test/mock_test/data/test_files',
test_input.test_uri,
None,
), f'{test_input_name} wrong put args'
finally:
os.getcwd = getcwd_orig
def _mock_get_file_info(file_id):
return FileInfo(
id=file_id,
size=10290,
md5sum='{}'.format(md5('-37'.encode()).hexdigest()),
file_type='image/jpeg',
)
def info(self, uri):
"""
Retrieve the descriptive metdata associated with a file.
:param uri: str that is an Artifact URI, representing the file for
which to retrieve metadata
:return: cadcdata.FileInfo instance, no scheme for md5sum
"""
self._logger.debug(f'Begin info for {uri}')
try:
if self._use_si:
result = self._cadc_client.cadcinfo(uri)
# make the result look like the other possible ways to
# obtain metadata
result.md5sum = result.md5sum.replace('md5:', '')
else:
archive, f_name = StorageClientWrapper._decompose(uri)
temp = self._cadc_client.get_file_info(archive, f_name)
result = FileInfo(
id=uri,
size=temp.get('size'),
file_type=temp.get('type'),
md5sum=temp.get('md5sum').replace('md5:', '')
)
except exceptions.NotFoundException:
self._logger.info(f'cadcinfo:: {uri} not found')
result = None
self._logger.debug('End info')
return result
def _get_uris(collection, fnames, obs):
# NOTE: this function makes the assumption that the collection is
# the same with the AD archive, which in many cases is not true (CFHT
# for example has multiple collections)
uris = []
file_meta = {}
if fnames:
for fname in fnames:
f = os.path.basename(fname).replace('.header', '')
for p in obs.planes.values():
for a in p.artifacts.values():
if (f'ad:{collection}/{f}' in a.uri
or (a.uri.startswith('vos') and f in a.uri)):
uris.append(a.uri)
meta = FileInfo(id=a.uri,
file_type=a.content_type,
size=a.content_length,
md5sum=a.content_checksum.checksum)
file_url = urlparse(a.uri)
if file_url.scheme not in ['ad', 'vos', 'cadc']:
# TODO add hook to support other service providers
raise NotImplementedError(
'Only ad, vos type URIs supported')
archive, file_id = file_url.path.split('/')[-2:]
file_meta[(archive, file_id)] = meta
return uris, file_meta
else:
return None
def _info(uri):
assert uri == 'ad:CFHT/2460606o.fits.gz', 'wrong info uri'
return FileInfo(
id=uri,
md5sum='abc',
size=42,
)
def _info(uri):
assert (uri == 'gemini:GEMINI/S20191214S0301.fits'), 'wrong info uri'
return FileInfo(
id=uri,
md5sum='abc',
size=42,
)
def _info(uri):
assert (uri == test_input.test_uri), 'wrong info uri'
return FileInfo(
id=uri,
md5sum='abc',
size=42,
)
def mock_get_file_info(file_id):
if isinstance(file_id, GemName):
# the case if StorageClientReader is being mocked
file_id = file_id.source_names[0]
if '_prev' in file_id:
return FileInfo(
size=10290,
md5sum='{}'.format(md5(b'-37').hexdigest()),
file_type='image/jpeg',
id=file_id,
)
else:
return FileInfo(
size=665345,
md5sum='a347f2754ff2fd4b6209e7566637efad',
file_type='application/fits',
id=file_id,
)
def test_time_nan():
test_obs = 'C120712_NGC7790_H_SCIRED'
test_file = f'file://{TESTDATA_DIR}/{test_obs}.fits.header'
omm_name = OmmName(file_name=f'{test_obs}.fits', entry=test_file)
test_file_info = FileInfo(id=omm_name.file_uri)
test_xml = f'{TESTDATA_DIR}/{test_obs}.xml'
obs = mc.read_obs_from_file(test_xml)
headers = data_util.get_local_file_headers(test_file)
telescope = Telescope(omm_name.file_uri, headers)
with pytest.raises(mc.CadcException):
result = telescope.update(obs, omm_name, test_file_info)
assert result is None, 'should have returned nothing'
def get_local_file_info(fqn):
"""
Gets descriptive metadata for a file on disk.
:param fqn: Fully-qualified name of the file on disk.
:return: FileInfo
"""
s = stat(fqn)
meta = FileInfo(
id=path.basename(fqn),
size=s.st_size,
md5sum=md5(open(fqn, 'rb').read()).hexdigest(),
file_type=get_file_type(fqn),
)
return meta
def add_file_info_record(self, uri):
"""add_json_record has already been successfully called"""
if uri not in self._file_info.keys():
record = self._json_metadata[uri]
self._file_info[uri] = FileInfo(
id=uri,
size=record.get('data_size'),
name=record.get('filename'),
md5sum=record.get('data_md5'),
lastmod=mc.make_time_tz(record.get('lastmod')),
file_type=data_util.get_file_type(record.get('filename')),
encoding=data_util.get_file_encoding(record.get('filename')),
)
self._logger.debug(f'Adding FileInfo for {uri}')
def vault_info(client, uri):
"""
Translate Node properties into FileInfo.
:param client: Vault client
:param uri: VOS URI
:return: an instance of FileInfo
"""
node = client.get_node(uri, limit=None, force=False)
return FileInfo(
id=uri,
size=node.props.get('length'),
md5sum=node.props.get('MD5'),
lastmod=node.props.get('lastmod'),
)
def test_failure():
test_fqn = (
f'{test_fits2caom2_augmentation.TEST_DATA_DIR}/broken_data/'
f'dao_c122_2001_006946.fits.header'
)
dao_name = DAOName(basename(test_fqn).replace('.header', '.gz'))
file_info = FileInfo(id=dao_name.file_uri, file_type='application/fits')
headers = ac.make_headers_from_file(test_fqn)
metadata_reader = rdc.FileMetadataReader()
metadata_reader._headers = {dao_name.file_uri: headers}
metadata_reader._file_info = {dao_name.file_uri: file_info}
kwargs = {
'storage_name': dao_name,
'metadata_reader': metadata_reader,
}
observation = None
observation = fits2caom2_augmentation.visit(observation, **kwargs)
assert observation is None, 'failure, no observation creation'
def get_local_file_info(fqn):
"""
Gets descriptive metadata for a file on disk.
:param fqn: Fully-qualified name of the file on disk.
:return: FileInfo, no scheme on the md5sum value.
"""
s = stat(fqn)
# copy and paste from cadcdata/storageinventory.py
hash_md5 = md5()
with open(fqn, 'rb') as f:
for chunk in iter(lambda: f.read(4096), b''):
hash_md5.update(chunk)
meta = FileInfo(
id=path.basename(fqn),
size=s.st_size,
md5sum=hash_md5.hexdigest(),
file_type=get_file_type(fqn),
)
return meta
def test_look_pull_and_put(http_mock, mock_client):
test_storage_name = 'cadc:GEMINI/TEST.fits'
mock_client.info.return_value = FileInfo(
id=test_storage_name,
size=1234,
md5sum='9473fdd0d880a43c21b7778d34872157',
)
f_name = 'TEST.fits'
url = f'https://localhost/{f_name}'
test_config = mc.Config()
test_config.observe_execution = True
mock_client.info.return_value = None
test_fqn = os.path.join(TEST_DATA_DIR, f_name)
pull_augmentation.look_pull_and_put(
test_storage_name,
test_fqn,
url,
mock_client,
'md5:01234',
)
mock_client.data_client.put.assert_called_with(
TEST_DATA_DIR, test_storage_name), 'mock not called'
http_mock.assert_called_with(url, test_fqn), 'http mock not called'
def test_store_newer_files_only_flag(client_mock, test_config):
# first test case
# flag set to True, file is older at CADC, supports_latest_client = False
test_config.working_directory = tc.TEST_DATA_DIR
test_config.use_local_files = True
test_config.features.supports_latest_client = False
test_f_name = '1000003f.fits.fz'
sn = [os.path.join('/caom2pipe_test', test_f_name)]
du = [f'cadc:TEST/{test_f_name}']
test_sn = FlagStorageName(test_f_name, sn, du)
observable_mock = Mock(autospec=True)
client_mock.info.return_value = FileInfo(
id=du[0], lastmod='Mon, 4 Mar 2019 19:05:41 GMT')
test_subject = ec.LocalStore(
test_config,
test_sn,
'TEST_STORE',
client_mock,
observable_mock,
)
test_subject.execute(None)
assert client_mock.put.called, 'expect put call'
def test_store_newer_files_only_flag_client(client_mock, test_config):
# just like the previous test, except supports_latest_client = True
# first test case
# flag set to True, file is older at CADC, supports_latest_client = False
test_config.working_directory = tc.TEST_DATA_DIR
test_config.use_local_files = True
test_config.features.supports_latest_client = True
test_f_name = '1000003f.fits.fz'
sn = [os.path.join('/caom2pipe_test', test_f_name)]
du = [f'cadc:TEST/{test_f_name}']
test_sn = FlagStorageName(test_f_name, sn, du)
observable_mock = Mock(autospec=True)
client_mock.cadcinfo.return_value = FileInfo(
id=du[0], md5sum='d41d8cd98f00b204e9800998ecf8427e')
test_subject = ec.LocalStore(
test_config,
test_sn,
'TEST_STORE',
client_mock,
observable_mock,
)
test_subject.execute(None)
assert client_mock.put.called, 'expect copy call'
def test_si_client_get(mock_metrics, mock_client):
test_fqn = f'{tc.TEST_FILES_DIR}/TEST.fits'
if os.path.exists(test_fqn):
os.unlink(test_fqn)
test_source = 'gemini:GEMINI/TEST.fits'
test_fqn = os.path.join(tc.TEST_FILES_DIR, 'TEST.fits')
mock_client.cadcget.side_effect = tc.mock_si_get
mock_client.cadcinfo.return_value = FileInfo(
test_source, md5sum='9473fdd0d880a43c21b7778d34872157')
clc.si_client_get(
mock_client,
test_fqn,
test_source,
metrics=mock_metrics,
)
mock_client.cadcget.assert_called_with(test_source,
dest=test_fqn), 'mock not called'
assert mock_metrics.observe.called, 'mock not called'
args, kwargs = mock_metrics.observe.call_args
assert args[2] == 12, 'wrong size'
assert args[3] == 'cadcget', 'wrong endpoint'
assert args[4] == 'si', 'wrong service'
assert args[5] == 'TEST.fits', 'wrong id'
def test_dao_transfer_check_fits_verify(vault_info_mock):
test_match_file_info = FileInfo(
id='vos:abc/def.fits',
md5sum='ghi',
)
test_different_file_info = FileInfo(
id='vos:abc/def.fits',
md5sum='ghi',
)
test_file_info = [test_match_file_info, test_different_file_info]
test_data_client = Mock(autospec=True)
test_vos_client = Mock(autospec=True)
test_config = mc.Config()
test_config.data_source_extensions = ['.fits.gz']
test_config.data_sources = ['vos:DAO/Archive/Incoming']
test_config.cleanup_failure_destination = 'vos:DAO/failure'
test_config.cleanup_success_destination = 'vos:DAO/success'
test_config.recurse_data_sources = False
def _mock_listdir(entry):
if entry.endswith('Incoming'):
return [
'dao123.fits.gz', 'dao456.fits', 'Yesterday', '.dot.fits.gz'
]
else:
return []
test_vos_client.listdir.side_effect = _mock_listdir
test_vos_client.isdir.side_effect = [
False, False, True, False, False, False, True, False
]
vault_info_mock.return_value = test_file_info
test_data_client.info.return_value = test_file_info
for case in [True, False]:
test_config.cleanup_files_when_storing = case
test_subject = data_source.DAOVaultDataSource(test_config,
test_vos_client,
test_data_client)
assert test_subject is not None, 'expect ctor to work'
test_result = test_subject.get_work()
assert test_result is not None, 'expect a work list'
assert len(test_result) == 1, 'wrong work list entries'
assert (test_result[0] == 'vos:DAO/Archive/Incoming/dao123.fits.gz'
), 'wrong work entry'
assert test_vos_client.isdir.call_count == 4, 'wrong is_dir count'
test_vos_client.isdir.reset_mock()
# test the case when the md5sums are the same, so the transfer does
# not occur, but the file ends up in the success location
test_vos_client.isdir.side_effect = [False, False, True, False]
test_config.cleanup_files_when_storing = True
test_config.store_modified_files_only = True
vault_info_mock.return_value = test_match_file_info
test_data_client.info.return_value = test_different_file_info
test_vos_client.status.raises = exceptions.NotFoundException
second_test_subject = data_source.DAOVaultDataSource(
test_config, test_vos_client, test_data_client)
assert second_test_subject is not None, 'second ctor fails'
second_test_result = second_test_subject.get_work()
assert second_test_result is not None, 'expect a second result'
assert len(second_test_result) == 0, 'should be no successes'
assert test_vos_client.move.called, 'expect a success move call'
test_vos_client.move.assert_called_with(
'vos:DAO/Archive/Incoming/dao123.fits.gz',
'vos:DAO/success/dao123.fits.gz',
), 'wrong success move args'
assert test_vos_client.status.called, 'expect a status call'
def _file_info_mock(uri):
return FileInfo(
id=uri,
file_type='application/fits',
md5sum='md5:def',
)
def _get_info(uri):
assert uri == 'ad:DAO/dest_fqn.fits', f'wrong storage check {uri}'
return FileInfo(
id=uri,
md5sum='ghi',
)
def test_todo_local_move(
caom_mock,
access_mock,
fits2caom2_mock,
test_input_name,
):
if 'MOVE' not in test_input_name:
return
test_input = INPUTS.get(test_input_name)
_cleanup()
new_dir = TEST_DATA_DIR / 'new'
fail_dir = TEST_DATA_DIR / 'failure'
success_dir = TEST_DATA_DIR / 'success'
for entry in [new_dir, fail_dir, success_dir]:
for listing in entry.iterdir():
listing.unlink()
shutil.copy(test_input.test_file, new_dir / '2460606o.fits.gz')
access_mock.return_value = 'https://localhost'
# make sure the working directory TEST_EXEC_DIR has the correct things
# in it
config_file_target = TEST_EXEC_DIR / 'config.yml'
shutil.copy(test_input.config_file, config_file_target)
cache_file_target = TEST_EXEC_DIR / 'cache.yml'
shutil.copy(test_input.cache_file, cache_file_target)
with open(TEST_EXEC_DIR / 'cadcproxy.pem', 'w') as f:
f.write('test content')
getcwd_orig = os.getcwd
os.getcwd = Mock(return_value=TEST_EXEC_DIR)
# import the module for execution
sys.path.append(test_input.test_path)
test_module = import_module('composable')
caom_mock.return_value.data_client.info.side_effect = [
None,
FileInfo(
id='ad:CFHT/2460606o.fits.gz',
md5sum='3d29f0edd984065a044d1376a11c6f08',
),
]
caom_mock.return_value.metadata_client.read.side_effect = [
None,
SimpleObservation(
'obs_id',
'CFHT',
Algorithm(name='exposure'),
instrument=Instrument(name='ESPaDOnS'),
),
]
def _info(uri):
assert uri == 'ad:CFHT/2460606o.fits.gz', 'wrong info uri'
return FileInfo(
id=uri,
md5sum='abc',
size=42,
)
fits2caom2_mock.return_value.info.side_effect = _info
try:
test_result = test_module._run_by_builder()
assert test_result is not None, f'expect a result {test_input_name}'
assert test_result == 0, f'wrong test result {test_input_name}'
assert (caom_mock.return_value.data_client.put.called
), f'{test_input_name} put not called'
caom_mock.return_value.data_client.put.assert_called_with(
'/usr/src/app/integration_test/mock_test/data/test_files/new',
'ad:CFHT/2460606o.fits.gz',
'raw',
), f'{test_input_name} wrong put args'
count = 0
for entry in new_dir.iterdir():
count += 1
assert count == 0, 'wrong new dir content'
for entry in success_dir.iterdir():
count += 1
assert count == 1, 'wrong success dir content'
count = 0
for entry in fail_dir.iterdir():
logging.error(f'fail entry {entry}')
count += 1
assert count == 0, f'wrong fail dir content {count}'
finally:
os.getcwd = getcwd_orig
del sys.modules['composable']
def test_run_state_v(client_mock):
client_mock.metadata_client.read.side_effect = tc.mock_read
client_mock.data_client.info.return_value = FileInfo(
id='cadc:TEST/anything.fits',
size=42,
md5sum='9473fdd0d880a43c21b7778d34872157',
)
test_wd = '/usr/src/app/caom2pipe/int_test'
caom2pipe_bookmark = 'caom2_timestamp'
test_config = mc.Config()
test_config.working_directory = test_wd
test_config.collection = 'TEST'
test_config.interval = 10
test_config.log_file_directory = f'{test_wd}/logs'
test_config.failure_fqn = (
f'{test_config.log_file_directory}/failure_log.txt')
test_config.log_to_file = True
test_config.logging_level = 'INFO'
test_config.progress_file_name = 'progress.txt'
test_config.proxy_file_name = f'{test_wd}/cadcproxy.pem'
test_config.rejected_file_name = 'rejected.yml'
test_config.rejected_directory = f'{test_wd}/rejected'
test_config._report_fqn = (
f'{test_config.log_file_directory}/app_report.txt')
test_config.resource_id = 'ivo://cadc.nrc.ca/sc2repo'
test_config.retry_file_name = 'retries.txt'
test_config.retry_fqn = (
f'{test_config.log_file_directory}/{test_config.retry_file_name}')
test_config.state_file_name = 'state.yml'
test_config.success_fqn = (
f'{test_config.log_file_directory}/success_log.txt')
test_config.tap_id = 'ivo://cadc.nrc.ca/sc2tap'
test_config.task_types = [
mc.TaskType.STORE,
mc.TaskType.INGEST,
mc.TaskType.MODIFY,
]
test_config.features.use_file_names = True
test_config.features.use_urls = False
test_config.features.supports_latest_client = True
test_config.use_local_files = False
test_config.storage_inventory_resource_id = 'ivo://cadc.nrc.ca/test'
if not os.path.exists(test_wd):
os.mkdir(test_wd)
test_start_time, test_end_time = _get_times(test_config,
caom2pipe_bookmark)
with open(test_config.proxy_fqn, 'w') as f:
f.write('test content\n')
test_data_source = TestListDirTimeBoxDataSource()
test_builder = nbc.GuessingBuilder(tc.TestStorageName)
transferrer = TestTransfer()
try:
test_result = rc.run_by_state(
bookmark_name=caom2pipe_bookmark,
command_name='collection2caom2',
config=test_config,
end_time=test_end_time,
name_builder=test_builder,
source=test_data_source,
modify_transfer=transferrer,
store_transfer=transferrer,
clients=client_mock,
)
assert test_result is not None, 'expect a result'
assert test_result == 0, 'expect success'
assert client_mock.data_client.put.called, 'expect put call'
client_mock.data_client.put.assert_called_with(
'/usr/src/app/caom2pipe/int_test/test_obs_id',
'cadc:TEST/test_file.fits.gz',
None,
), 'wrong call args'
# state file checking
test_state = mc.State(test_config.state_fqn)
assert test_state is not None, 'expect state content'
test_checkpoint = test_state.get_bookmark(caom2pipe_bookmark)
assert test_checkpoint == test_end_time, 'wrong bookmark'
# success file testing
assert os.path.exists(test_config.log_file_directory), 'log directory'
assert os.path.exists(test_config.success_fqn), 'success fqn'
assert os.path.exists(test_config.progress_fqn), 'progress fqn'
log_file = f'{test_config.log_file_directory}/test_obs_id.log'
actual = glob.glob(f'{test_config.log_file_directory}/**')
assert os.path.exists(log_file), f'specific log file {actual}'
xml_file = f'{test_config.log_file_directory}/test_obs_id.xml'
assert os.path.exists(xml_file), f'xml file {actual}'
# reporting testing
report_file = f'{test_config.log_file_directory}/app_report.txt'
assert os.path.exists(report_file), f'report file {actual}'
pass_through_test = False
with open(report_file, 'r') as f:
for line in f:
pass_through_test = True
if 'Number' in line:
bits = line.split(':')
found = False
if 'Inputs' in bits[0]:
assert bits[1].strip() == '1', 'wrong inputs'
found = True
elif 'Successes' in bits[0]:
assert bits[1].strip() == '1', 'wrong successes'
found = True
elif 'Timeouts' in bits[0]:
assert bits[1].strip() == '0', 'wrong timeouts'
found = True
elif 'Retries' in bits[0]:
assert bits[1].strip() == '0', 'wrong retries'
found = True
elif 'Errors' in bits[0]:
assert bits[1].strip() == '0', 'wrong errors'
found = True
elif 'Rejections' in bits[0]:
assert bits[1].strip() == '0', 'wrong rejections'
found = True
assert found, f'{line}'
assert pass_through_test, 'found a report file and checked it'
finally:
f_list = glob.glob(f'{test_wd}/**', recursive=True)
for entry in f_list:
try:
if os.path.isdir(entry):
os.rmdir(entry)
else:
os.unlink(entry)
except OSError as e:
logging.error(f'failed to delete {e}')
def info_si_mock(ignore):
return FileInfo(id=test_uri, file_type='application/fits',
md5sum='abc', size=42)
def _info(uri):
return FileInfo(
id=uri,
md5sum='abc',
size=42,
)
def _info_mock(uri):
return FileInfo(id=uri,
size=12,
file_type='application/fits',
md5sum='abc')
def _get_file_info(uri):
return FileInfo(id=uri, file_type='application/fits')
def test_todo_vos(
vo_mock,
caom_mock,
access_mock,
transfer_mock,
fits2caom2_mock,
test_input_name,
):
if 'VOS' not in test_input_name:
return
test_input = INPUTS.get(test_input_name)
_cleanup()
access_mock.return_value = 'https://localhost'
# make sure the working directory TEST_EXEC_DIR has the correct things
# in it
config_file_target = TEST_EXEC_DIR / 'config.yml'
shutil.copy(test_input.config_file, config_file_target)
with open(TEST_EXEC_DIR / 'cadcproxy.pem', 'w') as f:
f.write('test content')
# import the module for execution
sys.path.append(test_input.test_path)
test_module = import_module('composable')
reload(test_module)
logging.error(dir(test_module))
def _vo_listdir(entry):
assert entry == 'vos:goliaths/test', 'wrong parameter'
return ['a2021_08_17_19_30_01.fits.gz']
vo_mock.return_value.listdir.side_effect = _vo_listdir
vo_mock.return_value.isdir.return_value = False
def _vo_get_node(uri, limit=None, force=False):
assert (uri == 'vos:goliaths/test/a2021_08_17_19_30_01.fits.gz'
), f'wrong get node uri {uri}'
test_start_time = (datetime.now(tz=dateutil.tz.UTC) -
timedelta(minutes=5))
node = type('', (), {})()
node.props = {
'length': 42,
'MD5': '1234',
'lastmod': test_start_time.isoformat(),
}
return node
vo_mock.return_value.get_node.side_effect = _vo_get_node
caom_mock.return_value.data_client.info.side_effect = [
None,
FileInfo(id='ad:DAO/a2021_08_17_19_30_01.fits.gz'),
]
caom_mock.return_value.metadata_client.read.side_effect = [
None,
SimpleObservation(
'obs_id',
'DAO',
Algorithm(name='exposure'),
),
]
def _transfer_get(src, dst):
logging.error(src)
logging.error(dst)
assert (src == 'vos:goliaths/test/a2021_08_17_19_30_01.fits.gz'
), 'wrong source'
assert (
dst == '/usr/src/app/integration_test/mock_test/data/execution/'
'a2021_08_17_19_30_01/a2021_08_17_19_30_01.fits.gz'), 'wrong dst'
with open(dst, 'w') as f2:
f2.write('test content')
transfer_mock.return_value.get.side_effect = _transfer_get
def _get_head(ignore):
x = """SIMPLE = T / Written by IDL: Fri Oct 6 01:48:35 2017
BITPIX = -32 / Bits per pixel
NAXIS = 2 / Number of dimensions
NAXIS1 = 2048 /
NAXIS2 = 2048 /
EXPTIME = 1.23
NCOMBINE= 1
DATATYPE= 'REDUC ' /Data type, SCIENCE/CALIB/REJECT/FOCUS/TEST
END
"""
delim = '\nEND'
extensions = \
[e + delim for e in x.split(delim) if e.strip()]
headers = [fits.Header.fromstring(e, sep='\n') for e in extensions]
return headers
fits2caom2_mock.return_value.get_head.side_effect = _get_head
def _info(uri):
return FileInfo(
id=uri,
md5sum='abc',
size=42,
)
fits2caom2_mock.return_value.info.side_effect = _info
getcwd_orig = os.getcwd
os.getcwd = Mock(return_value=TEST_EXEC_DIR)
try:
test_result = test_module._run_vo()
assert test_result is not None, f'expect a result {test_input_name}'
assert test_result == 0, f'wrong test result {test_input_name}'
assert (caom_mock.return_value.data_client.put.called
), f'{test_input_name} put not called'
caom_mock.return_value.data_client.put.assert_called_with(
'/usr/src/app/integration_test/mock_test/data/execution/'
'a2021_08_17_19_30_01',
'ad:DAO/a2021_08_17_19_30_01.fits.gz',
'raw',
), f'{test_input_name} wrong put args'
finally:
os.getcwd = getcwd_orig
del sys.modules['composable']
