def test_visitor(test_name):
storage_name = OmmName(file_name=basename(test_name).replace(
'.header', '.gz'), )
file_info = FileInfo(id=storage_name.file_uri,
file_type='application/fits')
headers = ac.make_headers_from_file(test_name)
metadata_reader = rdc.FileMetadataReader()
metadata_reader._headers = {storage_name.file_uri: headers}
metadata_reader._file_info = {storage_name.file_uri: file_info}
kwargs = {
'storage_name': storage_name,
'metadata_reader': metadata_reader,
}
observation = None
input_file = f'{TEST_DATA_DIR}/in.{storage_name.product_id}.fits.xml'
if exists(input_file):
observation = mc.read_obs_from_file(input_file)
observation = fits2caom2_augmentation.visit(observation, **kwargs)
expected_fqn = (f'{TEST_DATA_DIR}/{storage_name.file_id}.expected.xml')
expected = mc.read_obs_from_file(expected_fqn)
compare_result = get_differences(expected, observation)
if compare_result is not None:
actual_fqn = expected_fqn.replace('expected', 'actual')
mc.write_obs_to_file(observation, actual_fqn)
compare_text = '\n'.join([r for r in compare_result])
msg = (f'Differences found in observation {expected.observation_id}\n'
f'{compare_text}')
raise AssertionError(msg)
def test_main_app(header_mock, test_name):
header_mock.side_effect = ac.make_headers_from_file
storage_name = main_app.BlankName(entry=test_name)
metadata_reader = rdc.FileMetadataReader()
metadata_reader.set(storage_name)
file_type = 'application/fits'
metadata_reader.file_info[storage_name.file_uri].file_type = file_type
kwargs = {
'storage_name': storage_name,
'metadata_reader': metadata_reader,
}
expected_fqn = f'{TEST_DATA_DIR}/{test_name}.expected.xml'
expected = mc.read_obs_from_file(expected_fqn)
in_fqn = expected_fqn.replace('.expected', '.in')
actual_fqn = expected_fqn.replace('expected', 'actual')
if os.path.exists(actual_fqn):
os.unlink(actual_fqn)
observation = None
if os.path.exists(in_fqn):
observation = mc.read_obs_from_file(in_fqn)
observation = fits2caom2_augmentation.visit(observation, **kwargs)
try:
compare_result = get_differences(expected, observation)
except Exception as e:
mc.write_obs_to_file(observation, actual_fqn)
raise e
if compare_result is not None:
mc.write_obs_to_file(observation, actual_fqn)
compare_text = '\n'.join([r for r in compare_result])
msg = (f'Differences found in observation {expected.observation_id}\n'
f'{compare_text}')
raise AssertionError(msg)
def test_visit(header_mock, test_files):
obs_id = test_files[0]
header_mock.side_effect = ac.make_headers_from_file
expected_fqn = f'{TEST_DATA_DIR}/{obs_id}.xml'
expected = mc.read_obs_from_file(expected_fqn)
in_fqn = expected_fqn.replace('.xml', '.in.xml')
actual_fqn = expected_fqn.replace('.xml', '.actual.xml')
observation = None
if os.path.exists(in_fqn):
observation = mc.read_obs_from_file(in_fqn)
for f_name in test_files[1:]:
temp_fqn = f'{TEST_DATA_DIR}/{f_name}'
vlass_name = storage_name.VlassName(entry=temp_fqn)
metadata_reader = rdc.FileMetadataReader()
metadata_reader.set(vlass_name)
file_type = 'application/fits'
metadata_reader.file_info[vlass_name.file_uri].file_type = file_type
kwargs = {
'storage_name': vlass_name,
'metadata_reader': metadata_reader,
}
observation = fits2caom2_augmentation.visit(observation, **kwargs)
try:
compare_result = get_differences(expected, observation)
except Exception as e:
mc.write_obs_to_file(observation, actual_fqn)
raise e
if compare_result is not None:
mc.write_obs_to_file(observation, actual_fqn)
compare_text = '\n'.join([r for r in compare_result])
msg = (f'Differences found in observation {expected.observation_id}\n'
f'{compare_text}')
raise AssertionError(msg)
def test_main_app(test_name):
basename = os.path.basename(test_name)
vlite_name = VliteName(file_name=basename)
output_file = '{}/actual.{}.xml'.format(TEST_DATA_DIR, basename)
obs_path = '{}/{}'.format(TEST_DATA_DIR,
'expected.{}.xml'.format(vlite_name.obs_id))
expected = mc.read_obs_from_file(obs_path)
with patch('caom2utils.fits2caom2.CadcDataClient') as data_client_mock:
def get_file_info(archive, file_id):
return {'type': 'application/fits'}
data_client_mock.return_value.get_file_info.side_effect = get_file_info
sys.argv = \
('{} --no_validate --local {} --observation {} {} -o {} '
'--plugin {} --module {} --lineage {}'.
format(APPLICATION, _get_local(test_name), COLLECTION,
test_name, output_file, PLUGIN, PLUGIN,
_get_lineage(test_name))).split()
print(sys.argv)
vlite_main_app()
actual = mc.read_obs_from_file(output_file)
result = get_differences(expected, actual, 'Observation')
if result:
msg = 'Differences found in observation {} test name {}\n{}'. \
format(expected.observation_id, test_name, '\n'.join(
[r for r in result]))
raise AssertionError(msg)
def compare(ex_fqn, act_fqn, entry):
ex = mc.read_obs_from_file(ex_fqn)
act = mc.read_obs_from_file(act_fqn)
result = get_differences(ex, act, 'Observation')
if result:
result_str = '\n'.join([r for r in result])
msg = f'Differences found obs id {ex.observation_id} ' \
f'file id {entry} ' \
f'instr {ex.instrument.name}\n{result_str}'
return msg
return None
def test_aug_visit_quality_works():
rejected_uri = 'ad:VLASS/VLASS1.1.ql.T10t12.J075402-033000.10.2048.v1' \
'.I.iter1.image.pbcor.tt0.subim.fits'
test_file = os.path.join(TESTDATA_DIR,
'VLASS1.1.T01t01.J000228-363000.xml')
test_obs = mc.read_obs_from_file(test_file)
assert test_obs is not None, 'unexpected None'
data_dir = os.path.join(THIS_DIR, '../../data')
kwargs = {'working_directory': data_dir}
test_result = vlass_quality_augmentation.visit(test_obs, **kwargs)
assert test_obs is not None, 'unexpected modification'
assert test_result is not None, 'should have a result status'
assert len(test_result) == 1, 'modified artifacts count'
assert test_result['observations'] == 0, 'observation count'
assert test_obs.requirements is None, 'status value should not be set'
for plane in test_obs.planes:
for artifact in test_obs.planes[plane].artifacts:
test_obs.planes[plane].artifacts[artifact].uri = rejected_uri
test_result = vlass_quality_augmentation.visit(test_obs, **kwargs)
assert test_obs is not None, 'unexpected modification'
assert test_result is not None, 'should have a result status'
assert len(test_result) == 1, 'modified artifacts count'
assert test_result['observations'] == 2, 'observation count'
assert test_obs.requirements.flag == Status.FAIL, 'wrong status value'
mc.write_obs_to_file(test_obs, os.path.join(TESTDATA_DIR, 'x.xml'))
def test_preview_augment_delete_preview():
# plane starts with a preview artifact, but it represents a non-existent
# file, so remove the artifact from the CAOM observation
test_product_id = 'S20080610S0045'
fqn = os.path.join(TEST_DATA_DIR, 'GS-2008A-C-5-35-002.fits.xml')
obs = mc.read_obs_from_file(fqn)
assert len(obs.planes[test_product_id].artifacts) == 2, 'initial condition'
test_rejected = mc.Rejected('/tmp/nonexistent')
test_rejected.content = {
'bad_metadata': [],
'no_preview':
['S20080610S0043.jpg',
'S20080610S0041.jpg',
'S20080610S0044.jpg',
'S20080610S0045.jpg']}
test_config = mc.Config()
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
kwargs = {'working_directory': TEST_DATA_DIR,
'cadc_client': None,
'stream': 'stream',
'observable': test_observable}
result = preview_augmentation.visit(obs, **kwargs)
assert result is not None, 'expect a result'
assert result['artifacts'] == 1, 'wrong result'
assert len(obs.planes[test_product_id].artifacts) == 1, 'post condition'
def test_build_temporal_wcs(query_mock):
def _mock_query(arg1, arg2):
if 'N20160102S0296' in arg1:
return Table.read(
'val,delta,cunit,naxis\n'
'57389.66314699074,0.000115798611111111,d,1\n'.split('\n'),
format='csv',
)
else:
return Table.read(
'val,delta,cunit,naxis\n'
'57389.66342476852,0.000115798611111111,d,1\n'.split('\n'),
format='csv',
)
query_mock.side_effect = _mock_query
test_tap_client = Mock()
test_observation = mc.read_obs_from_file(
f'{tc.TEST_DATA_DIR}/build_temporal_wcs_start.xml')
test_plane = test_observation.planes['GN2001BQ013-04']
test_part = test_plane.artifacts['cadc:GEMINI/test.fits'].parts['0']
assert test_part.chunks[0].time is None, 'temporal wcs ic'
test_collection = 'TEST'
cc.build_temporal_wcs_bounds(test_tap_client, test_plane, test_collection)
assert test_part.chunks[0].time is not None, 'temporal wcs ic change'
test_result = test_part.chunks[0].time
assert test_result.axis is not None, 'expect axis'
assert test_result.axis.bounds is not None, 'expect bounds'
assert len(test_result.axis.bounds.samples) == 2, 'expect two samples'
def _invoke_gem2caom2(obs_id):
try:
plugin = os.path.join(gem2caom2.__path__[0], 'main_app.py')
command_line_bits = em.gofr.get_args(obs_id)
if len(command_line_bits) > 1:
msg = ' '.join(ii.obs_id for ii in command_line_bits)
logging.warning(
'Wanted one observation for {}, got {} with {}'.format(
obs_id, len(command_line_bits), msg))
for bit in command_line_bits:
output_temp_file = tempfile.NamedTemporaryFile(delete=False)
sys.argv = ('{} --no_validate --observation {} '
'--external_url {} '
'--plugin {} --module {} --out {} --lineage {}'.format(
APPLICATION, bit.obs_id, bit.urls, plugin, plugin,
output_temp_file.name, bit.lineage)).split()
logger.info(sys.argv)
gem2caom2.main_app2()
out_file_stat = os.stat(output_temp_file.name)
if out_file_stat.st_size > 0:
obs = mc.read_obs_from_file(output_temp_file.name)
_update_last_modified(obs)
else:
logging.error(
'Did not generate an observation for {}'.format(obs_id))
obs = None
os.unlink(output_temp_file.name)
return obs
except Exception as e:
logger.error('main_app {} failed for {} with {}'.format(
APPLICATION, obs_id, e))
import traceback
logging.error(traceback.format_exc())
return None
def test_pull_augmentation():
obs = mc.read_obs_from_file(TEST_OBS_FILE)
obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow()
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition'
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
cadc_client_mock = Mock()
kwargs = {'working_directory': TEST_DATA_DIR,
'cadc_client': cadc_client_mock,
'stream': 'stream',
'observable': test_observable}
with patch('caom2pipe.manage_composable.http_get') as http_mock, \
patch('caom2pipe.manage_composable.data_put') as ad_put_mock:
cadc_client_mock.return_value.data_get.return_value = mc.CadcException(
'test')
# no scheme from cadc client
cadc_client_mock.get_file_info.return_value = {'md5sum': '1234'}
result = pull_augmentation.visit(obs, **kwargs)
test_url = f'{pull_augmentation.FILE_URL}/{TEST_PRODUCT_ID}.fits'
test_prev = f'{TEST_DATA_DIR}/{TEST_PRODUCT_ID}.fits'
http_mock.assert_called_with(test_url, test_prev), 'mock not called'
assert ad_put_mock.called, 'ad put mock not called'
assert result is not None, 'expect a result'
assert result['observation'] == 0, 'no updated metadata'
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, \
'no new artifacts'
def test_provenance_augmentation(obs_id_mock, repo_get_mock, headers_mock,
test_fqn):
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
headers_mock.side_effect = _get_headers_mock
repo_get_mock.side_effect = _repo_get_mock
obs_id_mock.side_effect = _get_obs_id_mock
getcwd_orig = os.getcwd
os.getcwd = Mock(return_value=test_main_app.TEST_DATA_DIR)
try:
test_obs = mc.read_obs_from_file(test_fqn)
assert not test_obs.target.moving, 'initial conditions moving target'
kwargs = {
'science_file':
os.path.basename(test_fqn).replace('.expected.xml', '.fits'),
'working_directory':
test_main_app.TEST_DATA_DIR,
'observable':
test_observable,
'caom_repo_client':
Mock()
}
test_result = provenance_augmentation.visit(test_obs, **kwargs)
assert test_result is not None, 'expect a result'
assert test_result.get('provenance') == 2, 'wrong result'
assert len(test_obs.members) == 1, 'wrong membership'
assert test_obs.target.moving, 'should be changed'
finally:
os.getcwd = getcwd_orig
def test_add_headers_to_obs_by_blueprint(parser_mock):
test_blueprint = ObsBlueprint()
test_fqn = f'{test_conf.TEST_DATA_DIR}/translate_test/after_aug.xml'
test_obs = mc.read_obs_from_file(test_fqn)
test_product_id = '2515996g'
test_uri = 'ad:CFHT/2515996g.fits'
assert len(test_obs.planes) == 1, 'wrong number of planes'
assert (len(test_obs.planes[test_product_id].artifacts) == 1
), 'wrong number of artifacts'
assert (len(
test_obs.planes[test_product_id].artifacts[test_uri].parts) == 5
), 'wrong number of parts'
assert (len(
test_obs.planes[test_product_id].artifacts[test_uri].parts['0'].chunks)
== 1), 'wrong number of chunks'
assert (test_obs.planes[test_product_id].artifacts[test_uri].parts['0'].
chunks[0].naxis == 3), 'track initial value'
assert (test_obs.planes[test_product_id].artifacts[test_uri].
parts['IMAGE DATA'].chunks[0].naxis is None), 'track initial value'
tc.add_headers_to_obs_by_blueprint(test_obs, [], test_blueprint, test_uri,
test_product_id)
assert (len(
test_obs.planes[test_product_id].artifacts[test_uri].parts) == 5
), 'wrong number of parts'
assert (len(
test_obs.planes[test_product_id].artifacts[test_uri].parts['0'].chunks)
== 0), 'wrong number of chunks'
assert (len(
test_obs.planes[test_product_id].artifacts[test_uri].parts['1'].chunks)
== 1), 'wrong number of chunks'
assert (len(test_obs.planes[test_product_id].artifacts[test_uri].
parts['IMAGE DATA'].chunks) == 1), 'wrong number of chunks'
assert (test_obs.planes[test_product_id].artifacts[test_uri].
parts['IMAGE DATA'].chunks[0].naxis == 3), 'track initial value'
def test_provenance_augmentation(dmf_mock, headers_mock, access_mock,
builder_mock, repo_get_mock, test_fqn):
builder_mock.return_value._get_obs_id.return_value = None
access_mock.return_value = 'https://localhost'
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_config.task_types = [mc.TaskType.VISIT]
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
headers_mock.return_value.get_head.side_effect = _get_headers_mock
dmf_mock.get.side_effect = _get_obs_id_mock
repo_get_mock.side_effect = _repo_get_mock
getcwd_orig = os.getcwd
os.getcwd = Mock(return_value=test_main_app.TEST_DATA_DIR)
temp = os.path.basename(test_fqn).replace('.expected.xml', '.fits')
test_storage_name = GemProcName(entry=temp)
try:
test_obs = mc.read_obs_from_file(test_fqn)
assert not test_obs.target.moving, 'initial conditions moving target'
kwargs = {
'storage_name': test_storage_name,
'working_directory': test_main_app.TEST_DATA_DIR,
'observable': test_observable,
'caom_repo_client': Mock(),
}
test_result = provenance_augmentation.visit(test_obs, **kwargs)
assert test_result is not None, 'expect a result'
assert test_result.get('provenance') == 2, 'wrong result'
assert len(test_obs.members) == 1, 'wrong membership'
assert test_obs.target.moving, 'should be changed'
finally:
os.getcwd = getcwd_orig
def test_update_cal_provenance():
test_obs = 'C170323_domeflat_K_CALRED'
test_obs_file = os.path.join(TESTDATA_DIR, '{}.fits.xml'.format(test_obs))
test_header_file = os.path.join(TESTDATA_DIR,
'{}.fits.header'.format(test_obs))
test_obs = mc.read_obs_from_file(test_obs_file)
fits_header = open(test_header_file).read()
headers = ac.make_headers_from_string(fits_header)
_update_cal_provenance(test_obs, headers)
assert test_obs is not None, 'no test_obs'
assert test_obs.members is not None, 'no members'
assert len(test_obs.members) == 22, 'wrong obs members length'
test_member = test_obs.members.pop()
assert test_member.uri.find('caom:OMM/C170323') == 0, 'wrong member value'
assert test_member.uri.endswith('_CAL'), 'wrong member value'
for ii in test_obs.planes:
plane = test_obs.planes[ii]
assert plane.provenance is not None, 'no provenance'
assert plane.provenance.inputs is not None, 'no provenance inputs'
assert len(plane.provenance.inputs) == 22, \
'wrong provenance inputs length'
test_plane_input = plane.provenance.inputs.pop()
assert test_plane_input.uri.find(
'caom:OMM/C170323') == 0, 'wrong input value'
assert test_plane_input.uri.endswith('_CAL'), 'wrong input value'
def test_meta_update_observation_direct(test_config):
test_cred_param = '--cert /usr/src/app/cadcproxy.pem'
data_client_mock = Mock()
repo_client_mock = Mock()
test_sn = tc.TestStorageName()
test_observation = mc.read_obs_from_file(
f'{tc.TEST_DATA_DIR}/fpf_start_obs.xml'
)
try:
test_observable = mc.Observable(
mc.Rejected(test_config.rejected_fqn), mc.Metrics(test_config)
)
test_executor = ec.MetaUpdateObservation(
test_config,
test_sn,
TEST_APP,
test_cred_param,
data_client_mock,
repo_client_mock,
test_observation,
[],
observable=test_observable,
)
test_executor.execute(None)
assert repo_client_mock.update.called, 'repo update called'
finally:
pass
def test_pull_v_augmentation(put_mock, http_mock):
obs = mc.read_obs_from_file(TEST_OBS_FILE)
obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow()
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition'
test_uri = f'{SCHEME}:{COLLECTION}/{TEST_PRODUCT_ID}.fits'
for plane in obs.planes.values():
for artifact in plane.artifacts.values():
artifact.uri = test_uri
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
cadc_client_mock = Mock()
kwargs = {'working_directory': TEST_DATA_DIR,
'cadc_client': cadc_client_mock,
'observable': test_observable}
result = pull_v_augmentation.visit(obs, **kwargs)
test_url = f'{pull_augmentation.FILE_URL}/{TEST_PRODUCT_ID}.fits'
test_prev = f'{TEST_DATA_DIR}/{TEST_PRODUCT_ID}.fits'
http_mock.assert_called_with(test_url, test_prev), 'mock not called'
assert put_mock.called, 'put mock not called'
args, kwargs = put_mock.call_args
assert args[1] == TEST_DATA_DIR, 'wrong working dir'
assert args[2] == f'{TEST_PRODUCT_ID}.fits', 'wrong file name'
assert args[3] == test_uri, 'wrong storage name'
assert result is not None, 'expect a result'
assert result['observation'] == 0, 'no updated metadata'
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, \
'no new artifacts'
def test_visit_augmentation():
f_list = {
# positionAxis1 or positionAxis2 is null
'20090629-a96458f347efa3cbcd0f28171743e9cb.expected.xml': 6,
# missing axis 1
's04bu10_20041103_0009_project.xml': 2,
# observableAxis found (1) but metadata not found
'20150612-418df74888cfeff1651599d5703218a1.xml': 6,
# positionAxis1 found (1) but metadata not found
'acsis_00015_20070529T090717.xml': 4,
# this is the trouble-some one that had no position information
# but apparently should have
# 'M10AN02-76bdd3b5c47be9be5dcbd940b6e922f4.xml': 31
}
kwargs = {}
for key, value in f_list.items():
fqn = f'{test_composable.TEST_DATA_DIR}/{key}'
test_obs = mc.read_obs_from_file(fqn)
test_result = visit_augmentation.visit(test_obs, **kwargs)
assert test_result is not None, 'expect a result'
changed = test_result.get('chunks')
assert changed == value, 'wrong result'
mc.write_obs_to_file(
test_obs, f'{test_composable.TEST_DATA_DIR}/{key}.actual.xml')
def test_reset():
test_obs_file = 'fpf_start_obs.xml'
test_obs = mc.read_obs_from_file(
os.path.join(tc.TEST_DATA_DIR, test_obs_file))
test_chunk = (
test_obs.planes['VLASS1.2.T07t14.J084202-123000.quicklook.v1'].
artifacts[
'ad:VLASS/VLASS1.2.ql.T07t14.J084202-123000.10.2048.v1.I.iter1.'
'image.pbcor.tt0.subim.fits'].parts['0'].chunks.pop())
assert test_chunk is not None, 'chunk expected'
assert test_chunk.position is not None, 'position expected'
assert test_chunk.position.axis is not None, 'axis expected'
assert test_chunk.position.axis.bounds is None, 'bounds not expected'
assert test_chunk.energy is not None, 'energy expected'
assert test_chunk.energy_axis is not None, 'energy axis expected'
cc.reset_position(test_chunk)
assert test_chunk.position is None, 'position not expected'
assert test_chunk.position_axis_1 is None, 'axis 1 not expected'
assert test_chunk.position_axis_2 is None, 'axis 2 not expected'
cc.reset_energy(test_chunk)
assert test_chunk.energy is None, 'energy not expected'
assert test_chunk.energy_axis is None, 'energy axis not expected'
cc.reset_observable(test_chunk)
assert test_chunk.observable is None, 'observable not expected'
assert test_chunk.observable_axis is None, 'observable axis not expected'
all_archive_keys = cc.get_all_artifact_keys(test_obs)
assert len(all_archive_keys) == 2, 'wrong artifact count'
def test_aug_visit_works(query_endpoint_mock, get_mock):
get_mock.return_value.__enter__.return_value.raw = test_scrape.WL_INDEX
query_endpoint_mock.side_effect = test_scrape._query_endpoint
test_config = mc.Config()
test_config.get_executors()
test_state = mc.State(test_config.state_fqn)
work.init_web_log(test_state, test_config)
test_name = sn.VlassName(
file_name='VLASS1.2.ql.T07t13.J081828-133000.10.2048.v1.I.iter1.'
'image.pbcor.tt0.subim.fits',
entry='VLASS1.2.ql.T07t13.J081828-133000.10.2048.v1.I.iter1.'
'image.pbcor.tt0.subim.fits')
test_file = os.path.join(TEST_DATA_DIR, f'{test_name.obs_id}.xml')
test_obs = mc.read_obs_from_file(test_file)
assert test_obs is not None, 'unexpected None'
data_dir = os.path.join(THIS_DIR, '../../data')
kwargs = {'working_directory': data_dir, 'cadc_client': Mock()}
test_result = time_bounds_augmentation.visit(test_obs, **kwargs)
assert test_obs is not None, 'unexpected modification'
assert test_result is not None, 'should have a result status'
assert len(test_result) == 1, 'modified artifacts count'
assert test_result['artifacts'] == 2, 'artifact count'
plane = test_obs.planes[test_name.product_id]
chunk = plane.artifacts[test_name.file_uri].parts['0'].chunks[0]
assert chunk is not None
assert chunk.time is not None, 'no time information'
assert chunk.time.axis is not None, 'no axis information'
assert chunk.time.axis.bounds is not None, 'no bounds information'
assert len(chunk.time.axis.bounds.samples) == 1, \
'wrong amount of bounds info'
assert chunk.time.exposure == 234.0, 'wrong exposure value'
def test_main_app():
# test_name = ''
# basename = os.path.basename(test_name)
# product_id = basename.split('.fits')[0]
# lineage = _get_lineage(product_id, basename)
# output_file = '{}.actual.xml'.format(test_name)
# local = _get_local(test_name)
# plugin = PLUGIN
#
# with patch('caom2utils.fits2caom2.CadcDataClient') as data_client_mock:
# def get_file_info(archive, file_id):
# if '_prev' in file_id:
# return {'size': 10290,
# 'md5sum': md5('-37'.encode()).hexdigest(),
# 'type': 'image/jpeg'}
# else:
# return {'size': 37,
# 'md5sum': md5('-37'.encode()).hexdigest(),
# 'type': 'application/octet-stream'}
# data_client_mock.return_value.get_file_info.side_effect = \
# get_file_info
obs_id = 'f060_s00001'
file_name = '20190805T024026_f060_s00001.h5'
local = os.path.join(TEST_DATA_DIR, file_name)
plugin = None
product_id = 'pixel'
output_file = '{}/{}.xml'.format(TEST_DATA_DIR, obs_id)
lineage = '{}/ad:TAOSII/{}'.format(product_id, file_name)
sys.argv = \
('{} --no_validate --local {} '
'--plugin {} --module {} --observation TAOSII {} -o {} '
'--lineage {}'.
format(main_app.APPLICATION, local, plugin, plugin, obs_id,
output_file, lineage)).split()
print(sys.argv)
main_app.main_app2()
obs_path = os.path.join(THIS_DIR, 'stuff.xml')
expected = mc.read_obs_from_file(obs_path)
actual = mc.read_obs_from_file(output_file)
result = get_differences(actual, expected, 'Observation')
if result:
msg = 'Differences found in observation {}\n{}'. \
format(expected.observation_id, '\n'.join(
[r for r in result]))
raise AssertionError(msg)
def compare(ex_fqn, act_fqn, entry):
"""Run get_differences for two files on disk.
:param ex_fqn: Fully-qualified file name for the expected observation
:param act_fqn: Fully-qualified file name for the actual observation
:param entry: String for logging information.
:return:
"""
ex = mc.read_obs_from_file(ex_fqn)
act = mc.read_obs_from_file(act_fqn)
result = get_differences(ex, act, 'Observation')
if result:
result_str = '\n'.join([r for r in result])
msg = (f'Differences found obs id {ex.observation_id} file id {entry} '
f'instr {ex.instrument.name}\n{result_str}')
return msg
return None
def test_main_app(test_name):
basename = os.path.basename(test_name)
obs_id = AskapName.get_obs_id(basename)
product_id = AskapName.get_product_id(basename)
lineage = mc.get_lineage(COLLECTION, product_id,
basename.replace('.header', ''),
AskapName.scheme())
output_file = '{}.actual.xml'.format(test_name)
local = test_name
plugin = PLUGIN
with patch('caom2utils.fits2caom2.CadcDataClient') as data_client_mock:
def get_file_info(archive, file_id):
if '_prev' in file_id:
return {
'size': 10290,
'md5sum': md5('-37'.encode()).hexdigest(),
'type': 'image/jpeg'
}
else:
return {
'size': 37,
'md5sum': md5('-37'.encode()).hexdigest(),
'type': 'application/fits'
}
data_client_mock.return_value.get_file_info.side_effect = \
get_file_info
sys.argv = \
('{} --no_validate --local {} '
'--plugin {} --module {} --observation {} {} -o {} --lineage {}'.
format(APPLICATION, local, plugin, plugin, COLLECTION, obs_id,
output_file, lineage)).split()
print(sys.argv)
caom_main()
obs_path = _get_expected_name(test_name)
expected = mc.read_obs_from_file(obs_path)
actual = mc.read_obs_from_file(output_file)
result = get_differences(expected, actual, 'Observation')
if result:
msg = 'Differences found in observation {}\n{}'. \
format(expected.observation_id, '\n'.join(
[r for r in result]))
raise AssertionError(msg)
def compare(expected_fqn, actual_fqn, observation):
expected = mc.read_obs_from_file(expected_fqn)
compare_result = get_differences(expected, observation)
if compare_result is not None:
mc.write_obs_to_file(observation, actual_fqn)
compare_text = '\n'.join([r for r in compare_result])
msg = (f'Differences found in observation {expected.observation_id}\n'
f'{compare_text}. Check {actual_fqn}')
raise AssertionError(msg)
def _do_botched_compare(actual_fqn, expected_fqn):
"""Compare artifacts for .fits files only. Rationalize the keys so
that the schema change from GEM to GEMINI is irrelevant."""
actual = mc.read_obs_from_file(actual_fqn)
expected = mc.read_obs_from_file(expected_fqn)
result = _compare_keys(expected.planes.keys(), actual.planes.keys(),
'plane')
if result is None:
for plane in actual.planes.values():
e_plane = expected.planes[plane.product_id]
expected_a_keys = _rationalize_keys(e_plane.artifacts.keys())
actual_a_keys = _rationalize_keys(plane.artifacts.keys())
result = _compare_keys(expected_a_keys, actual_a_keys,
f'artifact.{plane.product_id}')
if result is None:
for artifact in plane.artifacts.values():
if 'gemini:GEM/' in artifact.uri:
# ignore the artifacts that are named with the
# 'old' URIs
continue
expected_id = artifact.uri.replace('GEMINI/', 'GEM/')
expected_artifact = e_plane.artifacts[expected_id]
expected_artifact.uri = artifact.uri
if expected_id in e_plane.artifacts.keys():
temp = get_differences(artifact, expected_artifact,
'Artifact')
if temp is not None:
if result is None:
result = temp
else:
result = f'{result}\n{temp}'
else:
temp = f'Artifact {expected_id} in actual ' \
f'but not expected.'
if result is None:
result = temp
else:
result = f'{result}\n{temp}'
else:
result = f'Got {len(actual.planes)} planes. Expected ' \
f'{len(expected.planes)} planes.'
return result
def mock_repo_read(arg1, arg2):
# arg1 GEMINI arg2 GS-CAL20191010-3-034
global read_call_count
if arg1 == 'GEMINI' and arg2 == 'GS-2004A-Q-6-27-0255':
return ''
elif read_call_count == 0:
read_call_count = 1
return None
else:
return mc.read_obs_from_file(
f'{TEST_DATA_DIR}/GS-2019B-Q-222-181-001.expected.xml')
def test_aug_visit_quality_works(query_endpoint_mock):
query_endpoint_mock.side_effect = test_scrape._query_endpoint
test_file = os.path.join(TEST_DATA_DIR,
'VLASS1.2.T08t19.J123816-103000.xml')
test_obs = mc.read_obs_from_file(test_file)
assert test_obs is not None, 'unexpected None'
kwargs = {}
test_obs = quality_augmentation.visit(test_obs, **kwargs)
assert test_obs is not None, 'unexpected modification'
assert test_obs.requirements.flag == Status.FAIL, 'wrong status value'
def test_visit(query_endpoint_mock, get_mock):
get_mock.return_value.__enter__.return_value.raw = WL_INDEX
test_id = ('VLASS1.2_T07t14.J084202-123000_P35696v1_2019_03_11T23_06_04.'
'128/')
query_endpoint_mock.side_effect = _query_endpoint
scrape.init_web_log_content({'VLASS1.2': TEST_START_TIME})
scrape.web_log_content[test_id] = TEST_START_TIME - timedelta(hours=1)
test_fqn = os.path.join(TEST_DATA_DIR, TEST_OBS_ID)
test_obs = mc.read_obs_from_file(f'{test_fqn}.xml')
kwargs = {'cadc_client': Mock()}
test_obs = time_bounds_augmentation.visit(test_obs, **kwargs)
assert test_obs is not None, 'expected a result'
obs_path = os.path.join(TEST_DATA_DIR, 'visit.xml')
expected_obs = mc.read_obs_from_file(obs_path)
result = get_differences(expected_obs, test_obs, 'Observation')
if result:
msg = 'Differences found in observation {}\n{}'.format(
TEST_OBS_ID, '\n'.join([r for r in result]))
raise AssertionError(msg)
def test_preview_augment_plane():
omm_name = OmmName(file_name=TEST_FILE, entry=TEST_FILE)
preview = os.path.join(TEST_FILES_DIR, omm_name.prev)
thumb = os.path.join(TEST_FILES_DIR, omm_name.thumb)
if os.path.exists(preview):
os.remove(preview)
if os.path.exists(thumb):
os.remove(thumb)
test_fqn = os.path.join(TEST_DATA_DIR,
f'{omm_name.product_id}.expected.xml')
test_obs = mc.read_obs_from_file(test_fqn)
assert len(test_obs.planes[omm_name.product_id].artifacts) == 1
preva = 'cadc:OMM/C170324_0054_SCI_prev.jpg'
thumba = 'cadc:OMM/C170324_0054_SCI_prev_256.jpg'
test_config = mc.Config()
test_config.observe_execution = True
test_metrics = mc.Metrics(test_config)
test_observable = mc.Observable(rejected=None, metrics=test_metrics)
test_kwargs = {
'working_directory': TEST_FILES_DIR,
'cadc_client': None,
'observable': test_observable,
'storage_name': omm_name,
}
test_result = preview_augmentation.visit(test_obs, **test_kwargs)
assert test_result is not None, 'expected a visit return value'
assert os.path.exists(preview)
assert os.path.exists(thumb)
test_plane = test_result.planes[omm_name.product_id]
assert test_plane.artifacts[preva].content_checksum == ChecksumURI(
'md5:f37d21c53055498d1b5cb7753e1c6d6f'), 'prev checksum failure'
assert test_plane.artifacts[thumba].content_checksum == ChecksumURI(
'md5:19661c3c2508ecc22425ee2a05881ed4'), 'thumb checksum failure'
# now do updates
test_obs.planes[
omm_name.product_id].artifacts[preva].content_checksum = ChecksumURI(
'de9f39804f172682ea9b001f8ca11f15')
test_obs.planes[
omm_name.product_id].artifacts[thumba].content_checksum = ChecksumURI(
'cd118dae04391f6bea93ba4bf2711adf')
test_result = preview_augmentation.visit(test_obs, **test_kwargs)
assert test_result is not None, 'expected update visit return value'
assert len(test_result.planes[omm_name.product_id].artifacts) == 3
assert os.path.exists(preview)
assert os.path.exists(thumb)
assert test_plane.artifacts[preva].content_checksum == ChecksumURI(
'md5:f37d21c53055498d1b5cb7753e1c6d6f'), 'prev update failed'
assert test_plane.artifacts[thumba].content_checksum == ChecksumURI(
'md5:19661c3c2508ecc22425ee2a05881ed4'), 'prev_256 update failed'
assert len(test_metrics.history) == 0, 'wrong history, client is not None'
def test_preview_augment(http_mock):
# this should result in two new artifacts being added to the plane
# one for a thumbnail and one for a preview
obs = mc.read_obs_from_file(TEST_OBS_FILE)
obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow()
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition'
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_metrics = mc.Metrics(test_config)
test_observable = mc.Observable(test_rejected, test_metrics)
cadc_client_mock = Mock()
clients_mock = Mock()
clients_mock.data_client = cadc_client_mock
test_storage_name = gem_name.GemName(file_name=f'{TEST_PRODUCT_ID}.fits')
kwargs = {
'working_directory': '/test_files',
'clients': clients_mock,
'observable': test_observable,
'storage_name': test_storage_name,
}
test_prev = f'/test_files/{TEST_PRODUCT_ID}.jpg'
if os.path.exists(test_prev):
os.unlink(test_prev)
try:
cadc_client_mock.get.side_effect = exceptions.UnexpectedException(
'test')
http_mock.side_effect = _get_mock
obs = preview_augmentation.visit(obs, **kwargs)
test_url = (f'{preview_augmentation.PREVIEW_URL}'
f'{TEST_PRODUCT_ID}.fits')
assert http_mock.called, 'http mock should be called'
http_mock.assert_called_with(test_url, test_prev), 'mock not called'
assert cadc_client_mock.put.called, 'put mock not called'
cadc_client_mock.put.assert_called_with(
'/test_files',
'cadc:GEMINI/GN2001BQ013-04_th.jpg',
), 'wrong put arguments'
assert obs is not None, 'expect a result'
assert (len(
obs.planes[TEST_PRODUCT_ID].artifacts) == 3), 'two new artifacts'
prev_uri = mc.build_uri(COLLECTION, f'{TEST_PRODUCT_ID}.jpg', SCHEME)
thumb_uri = mc.build_uri(COLLECTION, f'{TEST_PRODUCT_ID}_th.jpg',
'cadc')
assert (prev_uri
in obs.planes[TEST_PRODUCT_ID].artifacts.keys()), 'no preview'
assert (thumb_uri
in obs.planes[TEST_PRODUCT_ID].artifacts), 'no thumbnail'
finally:
if os.path.exists(test_prev):
os.unlink(test_prev)
def test_time_nan():
test_obs = 'C120712_NGC7790_H_SCIRED'
test_file = f'file://{TESTDATA_DIR}/{test_obs}.fits.header'
omm_name = OmmName(file_name=f'{test_obs}.fits', entry=test_file)
test_file_info = FileInfo(id=omm_name.file_uri)
test_xml = f'{TESTDATA_DIR}/{test_obs}.xml'
obs = mc.read_obs_from_file(test_xml)
headers = data_util.get_local_file_headers(test_file)
telescope = Telescope(omm_name.file_uri, headers)
with pytest.raises(mc.CadcException):
result = telescope.update(obs, omm_name, test_file_info)
assert result is None, 'should have returned nothing'
