def test_visitor(test_name):
storage_name = OmmName(file_name=basename(test_name).replace(
'.header', '.gz'), )
file_info = FileInfo(id=storage_name.file_uri,
file_type='application/fits')
headers = ac.make_headers_from_file(test_name)
metadata_reader = rdc.FileMetadataReader()
metadata_reader._headers = {storage_name.file_uri: headers}
metadata_reader._file_info = {storage_name.file_uri: file_info}
kwargs = {
'storage_name': storage_name,
'metadata_reader': metadata_reader,
}
observation = None
input_file = f'{TEST_DATA_DIR}/in.{storage_name.product_id}.fits.xml'
if exists(input_file):
observation = mc.read_obs_from_file(input_file)
observation = fits2caom2_augmentation.visit(observation, **kwargs)
expected_fqn = (f'{TEST_DATA_DIR}/{storage_name.file_id}.expected.xml')
expected = mc.read_obs_from_file(expected_fqn)
compare_result = get_differences(expected, observation)
if compare_result is not None:
actual_fqn = expected_fqn.replace('expected', 'actual')
mc.write_obs_to_file(observation, actual_fqn)
compare_text = '\n'.join([r for r in compare_result])
msg = (f'Differences found in observation {expected.observation_id}\n'
f'{compare_text}')
raise AssertionError(msg)
def main_app2():
args = get_gen_proc_arg_parser().parse_args()
try:
uri = _get_uri(args)
# blueprints = _build_blueprints(uri)
# gen_proc(args, blueprints)
obs = stuff(args)
mc.write_obs_to_file(obs, './stuff.xml')
except Exception as e:
logging.error('Failed {} execution for {} with {}.'.format(
APPLICATION, args, e))
tb = traceback.format_exc()
logging.error(tb)
sys.exit(-1)
logging.debug('Done {} processing.'.format(APPLICATION))
# bash-5.0# /usr/local/hdf5/bin/h5dump 20190805T024026_f060_s00001.h5 | grep DATASET
# DATASET "catalog" {
# DATASET "header" {
# DATASET "image" {
# DATASET "imager" {
# DATASET "moment" {
# DATASET "a" {
# DATASET "ap" {
# DATASET "b" {
# DATASET "bp" {
# DATASET "cdmatrix" {
# DATASET "order" {
# DATASET "window" {
# bash-5.0# ls^C
def test_aug_visit_quality_works():
rejected_uri = 'ad:VLASS/VLASS1.1.ql.T10t12.J075402-033000.10.2048.v1' \
'.I.iter1.image.pbcor.tt0.subim.fits'
test_file = os.path.join(TESTDATA_DIR,
'VLASS1.1.T01t01.J000228-363000.xml')
test_obs = mc.read_obs_from_file(test_file)
assert test_obs is not None, 'unexpected None'
data_dir = os.path.join(THIS_DIR, '../../data')
kwargs = {'working_directory': data_dir}
test_result = vlass_quality_augmentation.visit(test_obs, **kwargs)
assert test_obs is not None, 'unexpected modification'
assert test_result is not None, 'should have a result status'
assert len(test_result) == 1, 'modified artifacts count'
assert test_result['observations'] == 0, 'observation count'
assert test_obs.requirements is None, 'status value should not be set'
for plane in test_obs.planes:
for artifact in test_obs.planes[plane].artifacts:
test_obs.planes[plane].artifacts[artifact].uri = rejected_uri
test_result = vlass_quality_augmentation.visit(test_obs, **kwargs)
assert test_obs is not None, 'unexpected modification'
assert test_result is not None, 'should have a result status'
assert len(test_result) == 1, 'modified artifacts count'
assert test_result['observations'] == 2, 'observation count'
assert test_obs.requirements.flag == Status.FAIL, 'wrong status value'
mc.write_obs_to_file(test_obs, os.path.join(TESTDATA_DIR, 'x.xml'))
def test_visit(header_mock, test_files):
obs_id = test_files[0]
header_mock.side_effect = ac.make_headers_from_file
expected_fqn = f'{TEST_DATA_DIR}/{obs_id}.xml'
expected = mc.read_obs_from_file(expected_fqn)
in_fqn = expected_fqn.replace('.xml', '.in.xml')
actual_fqn = expected_fqn.replace('.xml', '.actual.xml')
observation = None
if os.path.exists(in_fqn):
observation = mc.read_obs_from_file(in_fqn)
for f_name in test_files[1:]:
temp_fqn = f'{TEST_DATA_DIR}/{f_name}'
vlass_name = storage_name.VlassName(entry=temp_fqn)
metadata_reader = rdc.FileMetadataReader()
metadata_reader.set(vlass_name)
file_type = 'application/fits'
metadata_reader.file_info[vlass_name.file_uri].file_type = file_type
kwargs = {
'storage_name': vlass_name,
'metadata_reader': metadata_reader,
}
observation = fits2caom2_augmentation.visit(observation, **kwargs)
try:
compare_result = get_differences(expected, observation)
except Exception as e:
mc.write_obs_to_file(observation, actual_fqn)
raise e
if compare_result is not None:
mc.write_obs_to_file(observation, actual_fqn)
compare_text = '\n'.join([r for r in compare_result])
msg = (f'Differences found in observation {expected.observation_id}\n'
f'{compare_text}')
raise AssertionError(msg)
def test_main_app(header_mock, test_name):
header_mock.side_effect = ac.make_headers_from_file
storage_name = main_app.BlankName(entry=test_name)
metadata_reader = rdc.FileMetadataReader()
metadata_reader.set(storage_name)
file_type = 'application/fits'
metadata_reader.file_info[storage_name.file_uri].file_type = file_type
kwargs = {
'storage_name': storage_name,
'metadata_reader': metadata_reader,
}
expected_fqn = f'{TEST_DATA_DIR}/{test_name}.expected.xml'
expected = mc.read_obs_from_file(expected_fqn)
in_fqn = expected_fqn.replace('.expected', '.in')
actual_fqn = expected_fqn.replace('expected', 'actual')
if os.path.exists(actual_fqn):
os.unlink(actual_fqn)
observation = None
if os.path.exists(in_fqn):
observation = mc.read_obs_from_file(in_fqn)
observation = fits2caom2_augmentation.visit(observation, **kwargs)
try:
compare_result = get_differences(expected, observation)
except Exception as e:
mc.write_obs_to_file(observation, actual_fqn)
raise e
if compare_result is not None:
mc.write_obs_to_file(observation, actual_fqn)
compare_text = '\n'.join([r for r in compare_result])
msg = (f'Differences found in observation {expected.observation_id}\n'
f'{compare_text}')
raise AssertionError(msg)
def _cmd_direct_mock():
from caom2 import SimpleObservation, Algorithm
obs = SimpleObservation(observation_id='VLASS1.2.T07t13.J083838-153000',
collection=COLLECTION,
algorithm=Algorithm(name='testing'))
mc.write_obs_to_file(
obs, '/usr/src/app/logs/VLASS1.2.T07t13.J083838-153000.fits.xml')
def test_visit_augmentation():
f_list = {
# positionAxis1 or positionAxis2 is null
'20090629-a96458f347efa3cbcd0f28171743e9cb.expected.xml': 6,
# missing axis 1
's04bu10_20041103_0009_project.xml': 2,
# observableAxis found (1) but metadata not found
'20150612-418df74888cfeff1651599d5703218a1.xml': 6,
# positionAxis1 found (1) but metadata not found
'acsis_00015_20070529T090717.xml': 4,
# this is the trouble-some one that had no position information
# but apparently should have
# 'M10AN02-76bdd3b5c47be9be5dcbd940b6e922f4.xml': 31
}
kwargs = {}
for key, value in f_list.items():
fqn = f'{test_composable.TEST_DATA_DIR}/{key}'
test_obs = mc.read_obs_from_file(fqn)
test_result = visit_augmentation.visit(test_obs, **kwargs)
assert test_result is not None, 'expect a result'
changed = test_result.get('chunks')
assert changed == value, 'wrong result'
mc.write_obs_to_file(
test_obs, f'{test_composable.TEST_DATA_DIR}/{key}.actual.xml')
def _write_obs_mock():
args = get_gen_proc_arg_parser().parse_args()
obs = SimpleObservation(
collection=args.observation[0],
observation_id=args.observation[1],
algorithm=Algorithm(name='exposure'),
)
mc.write_obs_to_file(obs, args.out_obs_xml)
def mock_repo_create(arg1):
# arg1 is an Observation instance
act_fqn = f'{TEST_DATA_DIR}/{arg1.observation_id}.actual.xml'
ex_fqn = f'{TEST_DATA_DIR}/{arg1.observation_id}.expected.xml'
mc.write_obs_to_file(arg1, act_fqn)
result = compare(ex_fqn, act_fqn, arg1.observation_id)
if result is not None:
assert False, result
def compare(expected_fqn, actual_fqn, observation):
expected = mc.read_obs_from_file(expected_fqn)
compare_result = get_differences(expected, observation)
if compare_result is not None:
mc.write_obs_to_file(observation, actual_fqn)
compare_text = '\n'.join([r for r in compare_result])
msg = (f'Differences found in observation {expected.observation_id}\n'
f'{compare_text}. Check {actual_fqn}')
raise AssertionError(msg)
def to_caom2():
plugin = ('/usr/local/lib/python3.9/site-packages/test_execute_composable/'
'test_execute_composable.py')
assert sys.argv is not None, 'expect sys.argv to be set'
local_meta_create_answer = [
'test_execute_composable',
'--verbose',
'--observation',
'OMM',
'test_obs_id',
'--local',
f'{tc.TEST_DATA_DIR}/test_file.fits.gz',
'--out',
f'{tc.THIS_DIR}/test_obs_id/test_obs_id.xml',
'--plugin',
f'{plugin}',
'--module',
f'{plugin}',
'--lineage',
'test_obs_id/cadc:TEST/test_obs_id.fits.gz',
]
scrape_answer = [
'test_execute_composable',
'--verbose',
'--not_connected',
'--observation',
'OMM',
'test_obs_id',
'--local',
f'{tc.TEST_DATA_DIR}/test_file.fits.gz',
'--out',
f'{tc.TEST_DATA_DIR}/test_obs_id/test_obs_id.xml',
'--plugin',
f'{plugin}',
'--module',
f'{plugin}',
'--lineage',
'test_obs_id/cadc:TEST/test_obs_id.fits.gz',
]
# TaskType.SCRAPE (Scrape)
if sys.argv != scrape_answer:
# TaskType.INGEST (LocalMetaCreate)
if sys.argv != local_meta_create_answer:
assert False, (f'wrong sys.argv values \n{sys.argv} '
f'\n{local_meta_create_answer}')
fqn_index = sys.argv.index('--out') + 1
fqn = sys.argv[fqn_index]
mc.write_obs_to_file(
SimpleObservation(
collection='test_collection',
observation_id='test_obs_id',
algorithm=Algorithm(str('exposure')),
),
fqn,
)
def _mock_write():
fqn = f'{tc.THIS_DIR}/NEOS_SCI_2015347000000_clean/' \
f'NEOS_SCI_2015347000000_clean.fits.xml'
mc.write_obs_to_file(
SimpleObservation(
collection='test_collection',
observation_id='ghi',
algorithm=Algorithm(str('test'))
),
fqn,
)
def to_caom2():
"""This function is called by pipeline execution. It must have this name.
"""
args = get_gen_proc_arg_parser().parse_args()
# uris = _get_uris(args)
# blueprints = _build_blueprints(uris)
# result = gen_proc(args, blueprints)
obs = build_from_hdf5(args)
mc.write_obs_to_file(obs, f'./{obs.observation_id}.actual.xml')
logging.debug(f'Done {APPLICATION} processing.')
return 1
def test_read_write_obs_with_file():
if os.path.exists(TEST_OBS_FILE):
os.unlink(TEST_OBS_FILE)
mc.write_obs_to_file(
SimpleObservation(
collection='test_collection',
observation_id='test_obs_id',
algorithm=Algorithm(str('exposure')),
),
TEST_OBS_FILE,
)
test_subject = mc.read_obs_from_file(TEST_OBS_FILE)
assert test_subject is not None, 'expect a result'
assert isinstance(test_subject, SimpleObservation), 'wrong read'
def test_aug_visit_works():
test_file = os.path.join(TESTDATA_DIR,
'VLASS1.1.T01t01.J000228-363000.xml')
test_obs = mc.read_obs_from_file(test_file)
assert test_obs is not None, 'unexpected None'
data_dir = os.path.join(THIS_DIR, '../../data')
kwargs = {'working_directory': data_dir}
test_result = vlass_time_bounds_augmentation.visit(test_obs, **kwargs)
assert test_obs is not None, 'unexpected modification'
assert test_result is not None, 'should have a result status'
assert len(test_result) == 1, 'modified artifacts count'
assert test_result['artifacts'] == 2, 'artifact count'
plane = test_obs.planes['VLASS1.1.T01t01.J000228-363000.quicklook.v1']
chunk = plane.artifacts[TEST_URI].parts['0'].chunks[0]
assert chunk is not None
assert chunk.time is not None, 'no time information'
assert chunk.time.axis is not None, 'no axis information'
assert chunk.time.axis.bounds is not None, 'no bounds information'
assert len(chunk.time.axis.bounds.samples) == 1, \
'wrong amount of bounds info'
assert chunk.time.exposure == 401.0, \
'wrong exposure value'
mc.write_obs_to_file(test_obs, os.path.join(TESTDATA_DIR, 'x.xml'))
else:
plane._energy = None
if not hasattr(plane, '_polarization'):
plane._polarization = None
if not hasattr(plane, '_time'):
plane._time = None
if not hasattr(plane, '_position'):
plane._position = None
# plane.artifacts = caom2.caom_util.TypedOrderedDict(caom2.Artifact, )
for artifact in plane.artifacts.values():
artifact._id = caom2.AbstractCaomEntity._gen_id(fulluuid=False)
artifact._last_modified = datetime.datetime.now()
artifact._max_last_modified = datetime.datetime.now()
artifact._meta_checksum = None
artifact._acc_meta_checksum = None
artifact.parts = caom2.caom_util.TypedOrderedDict(caom2.Part, )
# u = jsonpickle.unpickler.Unpickler()
# u.register_classes(caom2.caom_util.TypeOrderedDict)
# y = u.restore(js)
# print(y)
# without the register_classes, this gives the js as a very
# nicely-formatted string ;)
# y = u.restore(js)
# print(y)
mc.write_obs_to_file(x, './draost.xml')
print(x.planes['RN43-C74'].polarization)
def test_gen():
# from astroquery.alma import Alma
# from astropy.table import Table
# db_table = Alma().query(payload={'project_code': '2016.1.00010.S'})
# # db_table.write('{}/alma_query.xml'.format(TEST_DATA_DIR),
# # format='votable')
# db_table.write('{}/alma_query2.xml'.format(TEST_DATA_DIR), format='html')
# FYI - this has values
# import logging
# logging.error(db_table['Release date'])
# assert False
import logging
from caom2.diff import get_differences
from caom2pipe import manage_composable as mc
from almaca2caom2 import main_app as ma
from astropy.table import Table
db_fqn = '{}/alma_query.html'.format(ALMA_QUERY_DIR)
db_content = Table.read(db_fqn, format='html')
obs_lookup = {}
obs = None
entries = glob.glob('/data/calibrated/*ms.split.cal')
for entry in entries:
fqn = f'{entry}/md.pk'
if not os.path.exists(fqn):
logging.error('missing md for {}'.format(fqn))
continue
logging.error(entry)
try:
almaca_name = ma.AlmacaName(os.path.basename(entry))
except mc.CadcException:
# skip the input MS's
continue
if obs is not None and almaca_name.obs_id != obs.observation_id:
# logging.error('storing {}'.format(obs.observation_id))
obs_lookup[obs.observation_id] = obs
obs = obs_lookup.get(almaca_name.obs_id)
try:
obs = ma.build_observation(db_content, obs, fqn)
except Exception as e:
import traceback
logging.error(traceback.format_exc())
logging.error(fqn)
assert False
errors_found = False
for obs in obs_lookup:
obs_fqn = f'{ALMA_QUERY_DIR}/{obs}.actual.xml'
logging.error(f'checking fqn {obs_fqn}')
actual = obs_lookup.get(obs)
mc.write_obs_to_file(actual, obs_fqn)
expected_fqn = f'{ALMA_QUERY_DIR}/{actual.observation_id}.expected.xml'
if os.path.exists(expected_fqn):
expected = mc.read_obs_from_file(expected_fqn)
result = get_differences(expected, actual, 'Observation')
if result:
msg_str = '\n'.join([r for r in result])
msg = f'Differences found {actual.observation_id}\n{msg_str}'
logging.error(msg)
errors_found = True
# assert False, obs_fqn
else:
msg = f'Could not find {expected_fqn}'
logging.error(msg)
raise AssertionError(msg)
if errors_found:
raise AssertionError('Final failure.')
def _build_observation(args):
config = mc.Config()
config.get_executors()
existing = None
if args.in_obs_xml:
existing = mc.read_obs_from_file(args.in_obs_xml.name)
drao_name, drao_dir = _get_name(args)
json_fqn = f'{drao_dir}/{drao_name.obs_id}.json'
logging.error(f'Looking for metadata in {json_fqn}')
if not os.path.exists(json_fqn):
raise mc.CadcException(
f'Could not find {json_fqn}. Cannot continue without it.')
with open(json_fqn) as f:
js = f.read()
# get the skeleton of the CAOM2 observation
jsonpickle.handlers.register(TypedOrderedDict, TypedOrderedDictHandler)
jsonpickle.handlers.register(datetime, DateTimeHandler)
obs = jsonpickle.decode(js)
# add the bits of the CAOM2 observation that are required for a
# structure that's acceptable to /ams - this mostly amounts to
# ensuring that attributes have been defined on the 'un-pickled'
_set_common(obs, existing)
if obs._proposal is not None:
if not hasattr(obs._proposal, '_project'):
obs._proposal._project = None
if not hasattr(obs._proposal, '_name'):
obs._proposal._name = None
if not hasattr(obs._proposal, '_keywords'):
obs._proposal._keywords = set()
if not hasattr(obs._proposal, '_title'):
obs._proposal._title = None
if obs._target is not None:
if not hasattr(obs._target, '_target_type'):
obs._target._target_type = None
if not hasattr(obs._target, '_standard'):
obs._target._standard = None
if not hasattr(obs._target, '_redshift'):
obs._target._redshift = None
if not hasattr(obs._target, '_moving'):
obs._target._moving = None
if not hasattr(obs._target, '_target_id'):
obs._target._target_id = None
obs._target._keywords = set()
obs._requirements = None
if obs._telescope is not None:
obs._telescope._keywords = set()
if obs._instrument is not None:
obs._instrument._keywords = set()
obs._environment = None
if not hasattr(obs, '_meta_read_groups'):
obs._meta_read_groups = None
for plane in obs.planes.values():
if existing is not None:
_set_common(plane, existing.planes[plane.product_id])
else:
_set_common(plane, None)
plane._acc_meta_checksum = None
if not hasattr(plane, '_data_read_groups'):
plane._data_read_groups = None
if not hasattr(plane, '_meta_read_groups'):
plane._meta_read_groups = None
plane._metrics = None
plane._quality = None
if plane._provenance is not None:
plane._provenance._keywords = set()
plane._provenance._inputs = TypedSet(PlaneURI, )
if not hasattr(plane._provenance, '_run_id'):
plane._provenance._run_id = None
# plane._provenance._last_executed = None
if hasattr(plane, '_position'):
if plane._position is not None:
plane._position._dimension = None
plane._position._resolution = None
plane._position._sample_size = None
if not hasattr(plane._position, '_resolution_bounds'):
plane._position._resolution_bounds = None
else:
plane._position = None
if hasattr(plane, '_energy'):
if plane._energy is not None:
if not hasattr(plane._energy, '_sample_size'):
plane._energy._sample_size = None
if not hasattr(plane._energy, '_bandpass_name'):
plane._energy._bandpass_name = None
if not hasattr(plane._energy, '_transition'):
plane._energy._transition = None
if not hasattr(plane._energy, '_resolving_power'):
plane._energy._resolving_power = None
if not hasattr(plane._energy, '_resolving_power_bounds'):
plane._energy._resolving_power_bounds = None
if hasattr(plane._energy, '_em_band'):
plane._energy.energy_bands = caom_util.TypedSet(EnergyBand)
plane._energy.energy_bands.add(plane._energy._em_band)
else:
plane._energy = None
if not hasattr(plane, '_polarization'):
plane._polarization = None
if hasattr(plane, '_time'):
if plane._time is not None:
if not hasattr(plane._time, '_resolution'):
plane._time._resolution = None
if not hasattr(plane._time, '_resolution_bounds'):
plane._time._resolution_bounds = None
else:
plane._time = None
if not hasattr(plane, '_position'):
plane._position = None
if not hasattr(plane, '_custom'):
plane._custom = None
for artifact in plane.artifacts.values():
if existing is not None:
_set_common(
artifact,
existing.planes[plane.product_id].artifacts[artifact.uri])
else:
_set_common(artifact, None)
artifact._acc_meta_checksum = None
artifact.parts = TypedOrderedDict(Part, )
if not hasattr(artifact, '_content_release'):
artifact._content_release = None
if not hasattr(artifact, '_content_read_groups'):
artifact._content_read_groups = None
if args.out_obs_xml:
mc.write_obs_to_file(obs, args.out_obs_xml)
else:
raise mc.CadcException(f'No where to write for {obs.observation_id}')
return 0
ops_buffer = io.StringIO()
ops_client.query(ops_query, output_file=ops_buffer, data_only=True, response_format='csv')
ops_table = Table.read(ops_buffer.getvalue().split('\n'), format='csv')
if len(ops_table) == 1:
obs_id = ops_table[0]['observationID']
uri = ops_table[0]['uri']
ignore_scheme, ignore_path, f_name = mc.decompose_uri(uri)
print(f':::Looking for {obs_id} and {f_name}')
else:
print(f':::No observation records found for collection {archive}')
sys.exit(-1)
obs = caom_client.read(archive, obs_id)
obs_fqn = f'/usr/src/app/expected.{obs_id}.xml'
mc.write_obs_to_file(obs, obs_fqn)
print(f':::2 - Get {f_name}')
config = mc.Config()
config.get_executors()
data_client = CadcDataClient(subject)
metrics = mc.Metrics(config)
mc.data_get(data_client, '/usr/src/app', f_name, collection, metrics)
print(':::3 - Update config.yml to say task types are scrape and modify, and use local files.')
config.task_types = [mc.TaskType.SCRAPE, mc.TaskType.MODIFY]
config.use_local_files = True
config.logging_level = logging.INFO
mc.Config.write_to_file(config)
print(':::4 - Run the application.')
actual_fqn = f'/usr/src/app/logs/{obs_id}.xml'
round_trip_fqn = f'/usr/src/app/round_trip.{obs_id}.xml'
print(f'::: read obs from file {actual_fqn}')
actual_obs = mc.read_obs_from_file(actual_fqn)
# write to sc2repo
# read from sc2repo, to get the plane-level metadata if calculated by
# service
try:
caom_client.delete(collection, obs_id)
except exceptions.NotFoundException as e:
pass
print(f'::: create observation {collection} {obs_id}')
caom_client.create(actual_obs)
print(f'::: read observation from sc2repo')
obs_from_service = caom_client.read(collection, obs_id)
mc.write_obs_to_file(obs_from_service, round_trip_fqn)
try:
msg = mc.compare_observations(round_trip_fqn, expected_fqn)
print(msg)
except Exception as e:
print(f'comparison of {round_trip_fqn} and {expected_fqn} failed')
print(e)
for plane in obs_from_service.planes.values():
for artifact in plane.artifacts.values():
if '.fits' in artifact.uri:
f_name = mc.CaomName(uri=artifact.uri).file_name
todo_list.append(f_name)
# check that no clean up occurred, because this was supposed to be
# a SCRAPE + MODIFY configuration, where cleaning up doesn't make
# sense
