def test_visitor(test_name): storage_name = OmmName(file_name=basename(test_name).replace( '.header', '.gz'), ) file_info = FileInfo(id=storage_name.file_uri, file_type='application/fits') headers = ac.make_headers_from_file(test_name) metadata_reader = rdc.FileMetadataReader() metadata_reader._headers = {storage_name.file_uri: headers} metadata_reader._file_info = {storage_name.file_uri: file_info} kwargs = { 'storage_name': storage_name, 'metadata_reader': metadata_reader, } observation = None input_file = f'{TEST_DATA_DIR}/in.{storage_name.product_id}.fits.xml' if exists(input_file): observation = mc.read_obs_from_file(input_file) observation = fits2caom2_augmentation.visit(observation, **kwargs) expected_fqn = (f'{TEST_DATA_DIR}/{storage_name.file_id}.expected.xml') expected = mc.read_obs_from_file(expected_fqn) compare_result = get_differences(expected, observation) if compare_result is not None: actual_fqn = expected_fqn.replace('expected', 'actual') mc.write_obs_to_file(observation, actual_fqn) compare_text = '\n'.join([r for r in compare_result]) msg = (f'Differences found in observation {expected.observation_id}\n' f'{compare_text}') raise AssertionError(msg)
def main_app2(): args = get_gen_proc_arg_parser().parse_args() try: uri = _get_uri(args) # blueprints = _build_blueprints(uri) # gen_proc(args, blueprints) obs = stuff(args) mc.write_obs_to_file(obs, './stuff.xml') except Exception as e: logging.error('Failed {} execution for {} with {}.'.format( APPLICATION, args, e)) tb = traceback.format_exc() logging.error(tb) sys.exit(-1) logging.debug('Done {} processing.'.format(APPLICATION)) # bash-5.0# /usr/local/hdf5/bin/h5dump 20190805T024026_f060_s00001.h5 | grep DATASET # DATASET "catalog" { # DATASET "header" { # DATASET "image" { # DATASET "imager" { # DATASET "moment" { # DATASET "a" { # DATASET "ap" { # DATASET "b" { # DATASET "bp" { # DATASET "cdmatrix" { # DATASET "order" { # DATASET "window" { # bash-5.0# ls^C
def test_aug_visit_quality_works(): rejected_uri = 'ad:VLASS/VLASS1.1.ql.T10t12.J075402-033000.10.2048.v1' \ '.I.iter1.image.pbcor.tt0.subim.fits' test_file = os.path.join(TESTDATA_DIR, 'VLASS1.1.T01t01.J000228-363000.xml') test_obs = mc.read_obs_from_file(test_file) assert test_obs is not None, 'unexpected None' data_dir = os.path.join(THIS_DIR, '../../data') kwargs = {'working_directory': data_dir} test_result = vlass_quality_augmentation.visit(test_obs, **kwargs) assert test_obs is not None, 'unexpected modification' assert test_result is not None, 'should have a result status' assert len(test_result) == 1, 'modified artifacts count' assert test_result['observations'] == 0, 'observation count' assert test_obs.requirements is None, 'status value should not be set' for plane in test_obs.planes: for artifact in test_obs.planes[plane].artifacts: test_obs.planes[plane].artifacts[artifact].uri = rejected_uri test_result = vlass_quality_augmentation.visit(test_obs, **kwargs) assert test_obs is not None, 'unexpected modification' assert test_result is not None, 'should have a result status' assert len(test_result) == 1, 'modified artifacts count' assert test_result['observations'] == 2, 'observation count' assert test_obs.requirements.flag == Status.FAIL, 'wrong status value' mc.write_obs_to_file(test_obs, os.path.join(TESTDATA_DIR, 'x.xml'))
def test_visit(header_mock, test_files): obs_id = test_files[0] header_mock.side_effect = ac.make_headers_from_file expected_fqn = f'{TEST_DATA_DIR}/{obs_id}.xml' expected = mc.read_obs_from_file(expected_fqn) in_fqn = expected_fqn.replace('.xml', '.in.xml') actual_fqn = expected_fqn.replace('.xml', '.actual.xml') observation = None if os.path.exists(in_fqn): observation = mc.read_obs_from_file(in_fqn) for f_name in test_files[1:]: temp_fqn = f'{TEST_DATA_DIR}/{f_name}' vlass_name = storage_name.VlassName(entry=temp_fqn) metadata_reader = rdc.FileMetadataReader() metadata_reader.set(vlass_name) file_type = 'application/fits' metadata_reader.file_info[vlass_name.file_uri].file_type = file_type kwargs = { 'storage_name': vlass_name, 'metadata_reader': metadata_reader, } observation = fits2caom2_augmentation.visit(observation, **kwargs) try: compare_result = get_differences(expected, observation) except Exception as e: mc.write_obs_to_file(observation, actual_fqn) raise e if compare_result is not None: mc.write_obs_to_file(observation, actual_fqn) compare_text = '\n'.join([r for r in compare_result]) msg = (f'Differences found in observation {expected.observation_id}\n' f'{compare_text}') raise AssertionError(msg)
def test_main_app(header_mock, test_name): header_mock.side_effect = ac.make_headers_from_file storage_name = main_app.BlankName(entry=test_name) metadata_reader = rdc.FileMetadataReader() metadata_reader.set(storage_name) file_type = 'application/fits' metadata_reader.file_info[storage_name.file_uri].file_type = file_type kwargs = { 'storage_name': storage_name, 'metadata_reader': metadata_reader, } expected_fqn = f'{TEST_DATA_DIR}/{test_name}.expected.xml' expected = mc.read_obs_from_file(expected_fqn) in_fqn = expected_fqn.replace('.expected', '.in') actual_fqn = expected_fqn.replace('expected', 'actual') if os.path.exists(actual_fqn): os.unlink(actual_fqn) observation = None if os.path.exists(in_fqn): observation = mc.read_obs_from_file(in_fqn) observation = fits2caom2_augmentation.visit(observation, **kwargs) try: compare_result = get_differences(expected, observation) except Exception as e: mc.write_obs_to_file(observation, actual_fqn) raise e if compare_result is not None: mc.write_obs_to_file(observation, actual_fqn) compare_text = '\n'.join([r for r in compare_result]) msg = (f'Differences found in observation {expected.observation_id}\n' f'{compare_text}') raise AssertionError(msg)
def _cmd_direct_mock(): from caom2 import SimpleObservation, Algorithm obs = SimpleObservation(observation_id='VLASS1.2.T07t13.J083838-153000', collection=COLLECTION, algorithm=Algorithm(name='testing')) mc.write_obs_to_file( obs, '/usr/src/app/logs/VLASS1.2.T07t13.J083838-153000.fits.xml')
def test_visit_augmentation(): f_list = { # positionAxis1 or positionAxis2 is null '20090629-a96458f347efa3cbcd0f28171743e9cb.expected.xml': 6, # missing axis 1 's04bu10_20041103_0009_project.xml': 2, # observableAxis found (1) but metadata not found '20150612-418df74888cfeff1651599d5703218a1.xml': 6, # positionAxis1 found (1) but metadata not found 'acsis_00015_20070529T090717.xml': 4, # this is the trouble-some one that had no position information # but apparently should have # 'M10AN02-76bdd3b5c47be9be5dcbd940b6e922f4.xml': 31 } kwargs = {} for key, value in f_list.items(): fqn = f'{test_composable.TEST_DATA_DIR}/{key}' test_obs = mc.read_obs_from_file(fqn) test_result = visit_augmentation.visit(test_obs, **kwargs) assert test_result is not None, 'expect a result' changed = test_result.get('chunks') assert changed == value, 'wrong result' mc.write_obs_to_file( test_obs, f'{test_composable.TEST_DATA_DIR}/{key}.actual.xml')
def _write_obs_mock(): args = get_gen_proc_arg_parser().parse_args() obs = SimpleObservation( collection=args.observation[0], observation_id=args.observation[1], algorithm=Algorithm(name='exposure'), ) mc.write_obs_to_file(obs, args.out_obs_xml)
def mock_repo_create(arg1): # arg1 is an Observation instance act_fqn = f'{TEST_DATA_DIR}/{arg1.observation_id}.actual.xml' ex_fqn = f'{TEST_DATA_DIR}/{arg1.observation_id}.expected.xml' mc.write_obs_to_file(arg1, act_fqn) result = compare(ex_fqn, act_fqn, arg1.observation_id) if result is not None: assert False, result
def compare(expected_fqn, actual_fqn, observation): expected = mc.read_obs_from_file(expected_fqn) compare_result = get_differences(expected, observation) if compare_result is not None: mc.write_obs_to_file(observation, actual_fqn) compare_text = '\n'.join([r for r in compare_result]) msg = (f'Differences found in observation {expected.observation_id}\n' f'{compare_text}. Check {actual_fqn}') raise AssertionError(msg)
def to_caom2(): plugin = ('/usr/local/lib/python3.9/site-packages/test_execute_composable/' 'test_execute_composable.py') assert sys.argv is not None, 'expect sys.argv to be set' local_meta_create_answer = [ 'test_execute_composable', '--verbose', '--observation', 'OMM', 'test_obs_id', '--local', f'{tc.TEST_DATA_DIR}/test_file.fits.gz', '--out', f'{tc.THIS_DIR}/test_obs_id/test_obs_id.xml', '--plugin', f'{plugin}', '--module', f'{plugin}', '--lineage', 'test_obs_id/cadc:TEST/test_obs_id.fits.gz', ] scrape_answer = [ 'test_execute_composable', '--verbose', '--not_connected', '--observation', 'OMM', 'test_obs_id', '--local', f'{tc.TEST_DATA_DIR}/test_file.fits.gz', '--out', f'{tc.TEST_DATA_DIR}/test_obs_id/test_obs_id.xml', '--plugin', f'{plugin}', '--module', f'{plugin}', '--lineage', 'test_obs_id/cadc:TEST/test_obs_id.fits.gz', ] # TaskType.SCRAPE (Scrape) if sys.argv != scrape_answer: # TaskType.INGEST (LocalMetaCreate) if sys.argv != local_meta_create_answer: assert False, (f'wrong sys.argv values \n{sys.argv} ' f'\n{local_meta_create_answer}') fqn_index = sys.argv.index('--out') + 1 fqn = sys.argv[fqn_index] mc.write_obs_to_file( SimpleObservation( collection='test_collection', observation_id='test_obs_id', algorithm=Algorithm(str('exposure')), ), fqn, )
def _mock_write(): fqn = f'{tc.THIS_DIR}/NEOS_SCI_2015347000000_clean/' \ f'NEOS_SCI_2015347000000_clean.fits.xml' mc.write_obs_to_file( SimpleObservation( collection='test_collection', observation_id='ghi', algorithm=Algorithm(str('test')) ), fqn, )
def to_caom2(): """This function is called by pipeline execution. It must have this name. """ args = get_gen_proc_arg_parser().parse_args() # uris = _get_uris(args) # blueprints = _build_blueprints(uris) # result = gen_proc(args, blueprints) obs = build_from_hdf5(args) mc.write_obs_to_file(obs, f'./{obs.observation_id}.actual.xml') logging.debug(f'Done {APPLICATION} processing.') return 1
def test_read_write_obs_with_file(): if os.path.exists(TEST_OBS_FILE): os.unlink(TEST_OBS_FILE) mc.write_obs_to_file( SimpleObservation( collection='test_collection', observation_id='test_obs_id', algorithm=Algorithm(str('exposure')), ), TEST_OBS_FILE, ) test_subject = mc.read_obs_from_file(TEST_OBS_FILE) assert test_subject is not None, 'expect a result' assert isinstance(test_subject, SimpleObservation), 'wrong read'
def test_aug_visit_works(): test_file = os.path.join(TESTDATA_DIR, 'VLASS1.1.T01t01.J000228-363000.xml') test_obs = mc.read_obs_from_file(test_file) assert test_obs is not None, 'unexpected None' data_dir = os.path.join(THIS_DIR, '../../data') kwargs = {'working_directory': data_dir} test_result = vlass_time_bounds_augmentation.visit(test_obs, **kwargs) assert test_obs is not None, 'unexpected modification' assert test_result is not None, 'should have a result status' assert len(test_result) == 1, 'modified artifacts count' assert test_result['artifacts'] == 2, 'artifact count' plane = test_obs.planes['VLASS1.1.T01t01.J000228-363000.quicklook.v1'] chunk = plane.artifacts[TEST_URI].parts['0'].chunks[0] assert chunk is not None assert chunk.time is not None, 'no time information' assert chunk.time.axis is not None, 'no axis information' assert chunk.time.axis.bounds is not None, 'no bounds information' assert len(chunk.time.axis.bounds.samples) == 1, \ 'wrong amount of bounds info' assert chunk.time.exposure == 401.0, \ 'wrong exposure value' mc.write_obs_to_file(test_obs, os.path.join(TESTDATA_DIR, 'x.xml'))
else: plane._energy = None if not hasattr(plane, '_polarization'): plane._polarization = None if not hasattr(plane, '_time'): plane._time = None if not hasattr(plane, '_position'): plane._position = None # plane.artifacts = caom2.caom_util.TypedOrderedDict(caom2.Artifact, ) for artifact in plane.artifacts.values(): artifact._id = caom2.AbstractCaomEntity._gen_id(fulluuid=False) artifact._last_modified = datetime.datetime.now() artifact._max_last_modified = datetime.datetime.now() artifact._meta_checksum = None artifact._acc_meta_checksum = None artifact.parts = caom2.caom_util.TypedOrderedDict(caom2.Part, ) # u = jsonpickle.unpickler.Unpickler() # u.register_classes(caom2.caom_util.TypeOrderedDict) # y = u.restore(js) # print(y) # without the register_classes, this gives the js as a very # nicely-formatted string ;) # y = u.restore(js) # print(y) mc.write_obs_to_file(x, './draost.xml') print(x.planes['RN43-C74'].polarization)
def test_gen(): # from astroquery.alma import Alma # from astropy.table import Table # db_table = Alma().query(payload={'project_code': '2016.1.00010.S'}) # # db_table.write('{}/alma_query.xml'.format(TEST_DATA_DIR), # # format='votable') # db_table.write('{}/alma_query2.xml'.format(TEST_DATA_DIR), format='html') # FYI - this has values # import logging # logging.error(db_table['Release date']) # assert False import logging from caom2.diff import get_differences from caom2pipe import manage_composable as mc from almaca2caom2 import main_app as ma from astropy.table import Table db_fqn = '{}/alma_query.html'.format(ALMA_QUERY_DIR) db_content = Table.read(db_fqn, format='html') obs_lookup = {} obs = None entries = glob.glob('/data/calibrated/*ms.split.cal') for entry in entries: fqn = f'{entry}/md.pk' if not os.path.exists(fqn): logging.error('missing md for {}'.format(fqn)) continue logging.error(entry) try: almaca_name = ma.AlmacaName(os.path.basename(entry)) except mc.CadcException: # skip the input MS's continue if obs is not None and almaca_name.obs_id != obs.observation_id: # logging.error('storing {}'.format(obs.observation_id)) obs_lookup[obs.observation_id] = obs obs = obs_lookup.get(almaca_name.obs_id) try: obs = ma.build_observation(db_content, obs, fqn) except Exception as e: import traceback logging.error(traceback.format_exc()) logging.error(fqn) assert False errors_found = False for obs in obs_lookup: obs_fqn = f'{ALMA_QUERY_DIR}/{obs}.actual.xml' logging.error(f'checking fqn {obs_fqn}') actual = obs_lookup.get(obs) mc.write_obs_to_file(actual, obs_fqn) expected_fqn = f'{ALMA_QUERY_DIR}/{actual.observation_id}.expected.xml' if os.path.exists(expected_fqn): expected = mc.read_obs_from_file(expected_fqn) result = get_differences(expected, actual, 'Observation') if result: msg_str = '\n'.join([r for r in result]) msg = f'Differences found {actual.observation_id}\n{msg_str}' logging.error(msg) errors_found = True # assert False, obs_fqn else: msg = f'Could not find {expected_fqn}' logging.error(msg) raise AssertionError(msg) if errors_found: raise AssertionError('Final failure.')
def _build_observation(args): config = mc.Config() config.get_executors() existing = None if args.in_obs_xml: existing = mc.read_obs_from_file(args.in_obs_xml.name) drao_name, drao_dir = _get_name(args) json_fqn = f'{drao_dir}/{drao_name.obs_id}.json' logging.error(f'Looking for metadata in {json_fqn}') if not os.path.exists(json_fqn): raise mc.CadcException( f'Could not find {json_fqn}. Cannot continue without it.') with open(json_fqn) as f: js = f.read() # get the skeleton of the CAOM2 observation jsonpickle.handlers.register(TypedOrderedDict, TypedOrderedDictHandler) jsonpickle.handlers.register(datetime, DateTimeHandler) obs = jsonpickle.decode(js) # add the bits of the CAOM2 observation that are required for a # structure that's acceptable to /ams - this mostly amounts to # ensuring that attributes have been defined on the 'un-pickled' _set_common(obs, existing) if obs._proposal is not None: if not hasattr(obs._proposal, '_project'): obs._proposal._project = None if not hasattr(obs._proposal, '_name'): obs._proposal._name = None if not hasattr(obs._proposal, '_keywords'): obs._proposal._keywords = set() if not hasattr(obs._proposal, '_title'): obs._proposal._title = None if obs._target is not None: if not hasattr(obs._target, '_target_type'): obs._target._target_type = None if not hasattr(obs._target, '_standard'): obs._target._standard = None if not hasattr(obs._target, '_redshift'): obs._target._redshift = None if not hasattr(obs._target, '_moving'): obs._target._moving = None if not hasattr(obs._target, '_target_id'): obs._target._target_id = None obs._target._keywords = set() obs._requirements = None if obs._telescope is not None: obs._telescope._keywords = set() if obs._instrument is not None: obs._instrument._keywords = set() obs._environment = None if not hasattr(obs, '_meta_read_groups'): obs._meta_read_groups = None for plane in obs.planes.values(): if existing is not None: _set_common(plane, existing.planes[plane.product_id]) else: _set_common(plane, None) plane._acc_meta_checksum = None if not hasattr(plane, '_data_read_groups'): plane._data_read_groups = None if not hasattr(plane, '_meta_read_groups'): plane._meta_read_groups = None plane._metrics = None plane._quality = None if plane._provenance is not None: plane._provenance._keywords = set() plane._provenance._inputs = TypedSet(PlaneURI, ) if not hasattr(plane._provenance, '_run_id'): plane._provenance._run_id = None # plane._provenance._last_executed = None if hasattr(plane, '_position'): if plane._position is not None: plane._position._dimension = None plane._position._resolution = None plane._position._sample_size = None if not hasattr(plane._position, '_resolution_bounds'): plane._position._resolution_bounds = None else: plane._position = None if hasattr(plane, '_energy'): if plane._energy is not None: if not hasattr(plane._energy, '_sample_size'): plane._energy._sample_size = None if not hasattr(plane._energy, '_bandpass_name'): plane._energy._bandpass_name = None if not hasattr(plane._energy, '_transition'): plane._energy._transition = None if not hasattr(plane._energy, '_resolving_power'): plane._energy._resolving_power = None if not hasattr(plane._energy, '_resolving_power_bounds'): plane._energy._resolving_power_bounds = None if hasattr(plane._energy, '_em_band'): plane._energy.energy_bands = caom_util.TypedSet(EnergyBand) plane._energy.energy_bands.add(plane._energy._em_band) else: plane._energy = None if not hasattr(plane, '_polarization'): plane._polarization = None if hasattr(plane, '_time'): if plane._time is not None: if not hasattr(plane._time, '_resolution'): plane._time._resolution = None if not hasattr(plane._time, '_resolution_bounds'): plane._time._resolution_bounds = None else: plane._time = None if not hasattr(plane, '_position'): plane._position = None if not hasattr(plane, '_custom'): plane._custom = None for artifact in plane.artifacts.values(): if existing is not None: _set_common( artifact, existing.planes[plane.product_id].artifacts[artifact.uri]) else: _set_common(artifact, None) artifact._acc_meta_checksum = None artifact.parts = TypedOrderedDict(Part, ) if not hasattr(artifact, '_content_release'): artifact._content_release = None if not hasattr(artifact, '_content_read_groups'): artifact._content_read_groups = None if args.out_obs_xml: mc.write_obs_to_file(obs, args.out_obs_xml) else: raise mc.CadcException(f'No where to write for {obs.observation_id}') return 0
ops_buffer = io.StringIO() ops_client.query(ops_query, output_file=ops_buffer, data_only=True, response_format='csv') ops_table = Table.read(ops_buffer.getvalue().split('\n'), format='csv') if len(ops_table) == 1: obs_id = ops_table[0]['observationID'] uri = ops_table[0]['uri'] ignore_scheme, ignore_path, f_name = mc.decompose_uri(uri) print(f':::Looking for {obs_id} and {f_name}') else: print(f':::No observation records found for collection {archive}') sys.exit(-1) obs = caom_client.read(archive, obs_id) obs_fqn = f'/usr/src/app/expected.{obs_id}.xml' mc.write_obs_to_file(obs, obs_fqn) print(f':::2 - Get {f_name}') config = mc.Config() config.get_executors() data_client = CadcDataClient(subject) metrics = mc.Metrics(config) mc.data_get(data_client, '/usr/src/app', f_name, collection, metrics) print(':::3 - Update config.yml to say task types are scrape and modify, and use local files.') config.task_types = [mc.TaskType.SCRAPE, mc.TaskType.MODIFY] config.use_local_files = True config.logging_level = logging.INFO mc.Config.write_to_file(config) print(':::4 - Run the application.')
actual_fqn = f'/usr/src/app/logs/{obs_id}.xml' round_trip_fqn = f'/usr/src/app/round_trip.{obs_id}.xml' print(f'::: read obs from file {actual_fqn}') actual_obs = mc.read_obs_from_file(actual_fqn) # write to sc2repo # read from sc2repo, to get the plane-level metadata if calculated by # service try: caom_client.delete(collection, obs_id) except exceptions.NotFoundException as e: pass print(f'::: create observation {collection} {obs_id}') caom_client.create(actual_obs) print(f'::: read observation from sc2repo') obs_from_service = caom_client.read(collection, obs_id) mc.write_obs_to_file(obs_from_service, round_trip_fqn) try: msg = mc.compare_observations(round_trip_fqn, expected_fqn) print(msg) except Exception as e: print(f'comparison of {round_trip_fqn} and {expected_fqn} failed') print(e) for plane in obs_from_service.planes.values(): for artifact in plane.artifacts.values(): if '.fits' in artifact.uri: f_name = mc.CaomName(uri=artifact.uri).file_name todo_list.append(f_name) # check that no clean up occurred, because this was supposed to be # a SCRAPE + MODIFY configuration, where cleaning up doesn't make # sense