def test_main_app(header_mock, test_name): header_mock.side_effect = ac.make_headers_from_file storage_name = main_app.BlankName(entry=test_name) metadata_reader = rdc.FileMetadataReader() metadata_reader.set(storage_name) file_type = 'application/fits' metadata_reader.file_info[storage_name.file_uri].file_type = file_type kwargs = { 'storage_name': storage_name, 'metadata_reader': metadata_reader, } expected_fqn = f'{TEST_DATA_DIR}/{test_name}.expected.xml' expected = mc.read_obs_from_file(expected_fqn) in_fqn = expected_fqn.replace('.expected', '.in') actual_fqn = expected_fqn.replace('expected', 'actual') if os.path.exists(actual_fqn): os.unlink(actual_fqn) observation = None if os.path.exists(in_fqn): observation = mc.read_obs_from_file(in_fqn) observation = fits2caom2_augmentation.visit(observation, **kwargs) try: compare_result = get_differences(expected, observation) except Exception as e: mc.write_obs_to_file(observation, actual_fqn) raise e if compare_result is not None: mc.write_obs_to_file(observation, actual_fqn) compare_text = '\n'.join([r for r in compare_result]) msg = (f'Differences found in observation {expected.observation_id}\n' f'{compare_text}') raise AssertionError(msg)
def test_main_app(test_name): basename = os.path.basename(test_name) vlite_name = VliteName(file_name=basename) output_file = '{}/actual.{}.xml'.format(TEST_DATA_DIR, basename) obs_path = '{}/{}'.format(TEST_DATA_DIR, 'expected.{}.xml'.format(vlite_name.obs_id)) expected = mc.read_obs_from_file(obs_path) with patch('caom2utils.fits2caom2.CadcDataClient') as data_client_mock: def get_file_info(archive, file_id): return {'type': 'application/fits'} data_client_mock.return_value.get_file_info.side_effect = get_file_info sys.argv = \ ('{} --no_validate --local {} --observation {} {} -o {} ' '--plugin {} --module {} --lineage {}'. format(APPLICATION, _get_local(test_name), COLLECTION, test_name, output_file, PLUGIN, PLUGIN, _get_lineage(test_name))).split() print(sys.argv) vlite_main_app() actual = mc.read_obs_from_file(output_file) result = get_differences(expected, actual, 'Observation') if result: msg = 'Differences found in observation {} test name {}\n{}'. \ format(expected.observation_id, test_name, '\n'.join( [r for r in result])) raise AssertionError(msg)
def test_visit(header_mock, test_files): obs_id = test_files[0] header_mock.side_effect = ac.make_headers_from_file expected_fqn = f'{TEST_DATA_DIR}/{obs_id}.xml' expected = mc.read_obs_from_file(expected_fqn) in_fqn = expected_fqn.replace('.xml', '.in.xml') actual_fqn = expected_fqn.replace('.xml', '.actual.xml') observation = None if os.path.exists(in_fqn): observation = mc.read_obs_from_file(in_fqn) for f_name in test_files[1:]: temp_fqn = f'{TEST_DATA_DIR}/{f_name}' vlass_name = storage_name.VlassName(entry=temp_fqn) metadata_reader = rdc.FileMetadataReader() metadata_reader.set(vlass_name) file_type = 'application/fits' metadata_reader.file_info[vlass_name.file_uri].file_type = file_type kwargs = { 'storage_name': vlass_name, 'metadata_reader': metadata_reader, } observation = fits2caom2_augmentation.visit(observation, **kwargs) try: compare_result = get_differences(expected, observation) except Exception as e: mc.write_obs_to_file(observation, actual_fqn) raise e if compare_result is not None: mc.write_obs_to_file(observation, actual_fqn) compare_text = '\n'.join([r for r in compare_result]) msg = (f'Differences found in observation {expected.observation_id}\n' f'{compare_text}') raise AssertionError(msg)
def test_visitor(test_name): storage_name = OmmName(file_name=basename(test_name).replace( '.header', '.gz'), ) file_info = FileInfo(id=storage_name.file_uri, file_type='application/fits') headers = ac.make_headers_from_file(test_name) metadata_reader = rdc.FileMetadataReader() metadata_reader._headers = {storage_name.file_uri: headers} metadata_reader._file_info = {storage_name.file_uri: file_info} kwargs = { 'storage_name': storage_name, 'metadata_reader': metadata_reader, } observation = None input_file = f'{TEST_DATA_DIR}/in.{storage_name.product_id}.fits.xml' if exists(input_file): observation = mc.read_obs_from_file(input_file) observation = fits2caom2_augmentation.visit(observation, **kwargs) expected_fqn = (f'{TEST_DATA_DIR}/{storage_name.file_id}.expected.xml') expected = mc.read_obs_from_file(expected_fqn) compare_result = get_differences(expected, observation) if compare_result is not None: actual_fqn = expected_fqn.replace('expected', 'actual') mc.write_obs_to_file(observation, actual_fqn) compare_text = '\n'.join([r for r in compare_result]) msg = (f'Differences found in observation {expected.observation_id}\n' f'{compare_text}') raise AssertionError(msg)
def compare(expected_fqn, actual_fqn, observation): expected = mc.read_obs_from_file(expected_fqn) compare_result = get_differences(expected, observation) if compare_result is not None: mc.write_obs_to_file(observation, actual_fqn) compare_text = '\n'.join([r for r in compare_result]) msg = (f'Differences found in observation {expected.observation_id}\n' f'{compare_text}. Check {actual_fqn}') raise AssertionError(msg)
def _compare_observations(expected, actual, output_dir): result = get_differences(expected, actual, 'Observation') if result: msg = 'Differences found observation {} in {}\n{}'.\ format(expected.observation_id, output_dir, '\n'.join([r for r in result])) raise AssertionError(msg) else: logging.info('Observation {} in {} match'.format( expected.observation_id, output_dir))
def compare(ex_fqn, act_fqn, entry): ex = mc.read_obs_from_file(ex_fqn) act = mc.read_obs_from_file(act_fqn) result = get_differences(ex, act, 'Observation') if result: result_str = '\n'.join([r for r in result]) msg = f'Differences found obs id {ex.observation_id} ' \ f'file id {entry} ' \ f'instr {ex.instrument.name}\n{result_str}' return msg return None
def _compare_observations(expected, actual, output_dir): result = get_differences(expected, actual, 'Observation') if result: tmp = '\n'.join([r for r in result]) msg = f'Differences found observation {expected.observation_id} in ' \ f'{output_dir}\n{tmp}' _write_observation(actual) raise AssertionError(msg) else: logging.info('Observation {} in {} match'.format( expected.observation_id, output_dir))
def compare(ex_fqn, act_fqn, entry): """Run get_differences for two files on disk. :param ex_fqn: Fully-qualified file name for the expected observation :param act_fqn: Fully-qualified file name for the actual observation :param entry: String for logging information. :return: """ ex = mc.read_obs_from_file(ex_fqn) act = mc.read_obs_from_file(act_fqn) result = get_differences(ex, act, 'Observation') if result: result_str = '\n'.join([r for r in result]) msg = (f'Differences found obs id {ex.observation_id} file id {entry} ' f'instr {ex.instrument.name}\n{result_str}') return msg return None
def test_main_app(test_name): basename = os.path.basename(test_name) obs_id = AskapName.get_obs_id(basename) product_id = AskapName.get_product_id(basename) lineage = mc.get_lineage(COLLECTION, product_id, basename.replace('.header', ''), AskapName.scheme()) output_file = '{}.actual.xml'.format(test_name) local = test_name plugin = PLUGIN with patch('caom2utils.fits2caom2.CadcDataClient') as data_client_mock: def get_file_info(archive, file_id): if '_prev' in file_id: return { 'size': 10290, 'md5sum': md5('-37'.encode()).hexdigest(), 'type': 'image/jpeg' } else: return { 'size': 37, 'md5sum': md5('-37'.encode()).hexdigest(), 'type': 'application/fits' } data_client_mock.return_value.get_file_info.side_effect = \ get_file_info sys.argv = \ ('{} --no_validate --local {} ' '--plugin {} --module {} --observation {} {} -o {} --lineage {}'. format(APPLICATION, local, plugin, plugin, COLLECTION, obs_id, output_file, lineage)).split() print(sys.argv) caom_main() obs_path = _get_expected_name(test_name) expected = mc.read_obs_from_file(obs_path) actual = mc.read_obs_from_file(output_file) result = get_differences(expected, actual, 'Observation') if result: msg = 'Differences found in observation {}\n{}'. \ format(expected.observation_id, '\n'.join( [r for r in result])) raise AssertionError(msg)
def test_visit(query_endpoint_mock, get_mock): get_mock.return_value.__enter__.return_value.raw = WL_INDEX test_id = ('VLASS1.2_T07t14.J084202-123000_P35696v1_2019_03_11T23_06_04.' '128/') query_endpoint_mock.side_effect = _query_endpoint scrape.init_web_log_content({'VLASS1.2': TEST_START_TIME}) scrape.web_log_content[test_id] = TEST_START_TIME - timedelta(hours=1) test_fqn = os.path.join(TEST_DATA_DIR, TEST_OBS_ID) test_obs = mc.read_obs_from_file(f'{test_fqn}.xml') kwargs = {'cadc_client': Mock()} test_obs = time_bounds_augmentation.visit(test_obs, **kwargs) assert test_obs is not None, 'expected a result' obs_path = os.path.join(TEST_DATA_DIR, 'visit.xml') expected_obs = mc.read_obs_from_file(obs_path) result = get_differences(expected_obs, test_obs, 'Observation') if result: msg = 'Differences found in observation {}\n{}'.format( TEST_OBS_ID, '\n'.join([r for r in result])) raise AssertionError(msg)
def test_main_app(test_files): obs_id = test_files[0] obs_path = os.path.join(TEST_DATA_DIR, '{}.xml'.format(obs_id)) expected = mc.read_obs_from_file(obs_path) if obs_id.endswith('r'): obs_path = os.path.join(TEST_DATA_DIR, '{}.in.xml'.format(obs_id)) input_param = '--in {}'.format(obs_path) else: input_param = '--observation {} {}'.format(COLLECTION, obs_id) lineage = _get_lineage(obs_id, test_files) output_file = '{}.actual.xml'.format(obs_id) local = _get_local(test_files[1:]) with patch('caom2utils.fits2caom2.CadcDataClient') as data_client_mock: def get_file_info(archive, file_id): if file_id == a: return {'size': 55425600, 'md5sum': 'ae2a33238c5051611133e7090560fd8a', 'type': 'application/fits'} else: return {'size': 55425600, 'md5sum': '40f7c2763f92ea6e9c6b0304c569097e', 'type': 'application/fits'} data_client_mock.return_value.get_file_info.side_effect = \ get_file_info sys.argv = \ ('vlass2caom2 --local {} {} -o {} --plugin {} --module {} ' '--lineage {}'.format(local, input_param, output_file, PLUGIN, PLUGIN, lineage)).split() print(sys.argv) to_caom2() actual = mc.read_obs_from_file(output_file) result = get_differences(expected, actual, 'Observation') if result: msg = 'Differences found in observation {}\n{}'. \ format(expected.observation_id, '\n'.join( [r for r in result])) raise AssertionError(msg)
def test_gen(): # from astroquery.alma import Alma # from astropy.table import Table # db_table = Alma().query(payload={'project_code': '2016.1.00010.S'}) # # db_table.write('{}/alma_query.xml'.format(TEST_DATA_DIR), # # format='votable') # db_table.write('{}/alma_query2.xml'.format(TEST_DATA_DIR), format='html') # FYI - this has values # import logging # logging.error(db_table['Release date']) # assert False import logging from caom2.diff import get_differences from caom2pipe import manage_composable as mc from almaca2caom2 import main_app as ma from astropy.table import Table db_fqn = '{}/alma_query.html'.format(ALMA_QUERY_DIR) db_content = Table.read(db_fqn, format='html') obs_lookup = {} obs = None entries = glob.glob('/data/calibrated/*ms.split.cal') for entry in entries: fqn = f'{entry}/md.pk' if not os.path.exists(fqn): logging.error('missing md for {}'.format(fqn)) continue logging.error(entry) try: almaca_name = ma.AlmacaName(os.path.basename(entry)) except mc.CadcException: # skip the input MS's continue if obs is not None and almaca_name.obs_id != obs.observation_id: # logging.error('storing {}'.format(obs.observation_id)) obs_lookup[obs.observation_id] = obs obs = obs_lookup.get(almaca_name.obs_id) try: obs = ma.build_observation(db_content, obs, fqn) except Exception as e: import traceback logging.error(traceback.format_exc()) logging.error(fqn) assert False errors_found = False for obs in obs_lookup: obs_fqn = f'{ALMA_QUERY_DIR}/{obs}.actual.xml' logging.error(f'checking fqn {obs_fqn}') actual = obs_lookup.get(obs) mc.write_obs_to_file(actual, obs_fqn) expected_fqn = f'{ALMA_QUERY_DIR}/{actual.observation_id}.expected.xml' if os.path.exists(expected_fqn): expected = mc.read_obs_from_file(expected_fqn) result = get_differences(expected, actual, 'Observation') if result: msg_str = '\n'.join([r for r in result]) msg = f'Differences found {actual.observation_id}\n{msg_str}' logging.error(msg) errors_found = True # assert False, obs_fqn else: msg = f'Could not find {expected_fqn}' logging.error(msg) raise AssertionError(msg) if errors_found: raise AssertionError('Final failure.')
def test_main_app(test_name): obs_id = test_name lineage = _get_lineage(obs_id) input_file = '{}/{}/{}.in.xml'.format(TEST_DATA_DIR, DIR_NAME, obs_id) actual_fqn = '{}/{}/{}.actual.xml'.format(TEST_DATA_DIR, DIR_NAME, obs_id) local = _get_local(test_name) plugin = PLUGIN with patch('caom2utils.fits2caom2.CadcDataClient') as data_client_mock, \ patch('gem2caom2.external_metadata.get_obs_metadata') as gemini_client_mock, \ patch('gem2caom2.external_metadata.get_pi_metadata') as gemini_pi_mock, \ patch('gem2caom2.svofps.get_vo_table') as svofps_mock: def get_file_info(archive, file_id): if '_prev' in file_id: return {'size': 10290, 'md5sum': 'md5:{}'.format( md5('-37'.encode()).hexdigest()), 'type': 'image/jpeg'} else: return {'size': 665345, 'md5sum': 'md5:a347f2754ff2fd4b6209e7566637efad', 'type': 'application/fits'} def get_obs_metadata(file_id): try: logging.error('obs metadata file_id {}'.format(file_id)) fname = '{}/{}/json/{}.json'.format(TEST_DATA_DIR, DIR_NAME, file_id) with open(fname) as f: y = json.loads(f.read()) em.om.add(y, file_id) except Exception as e: logging.error(e) import traceback tb = traceback.format_exc() logging.error(tb) def get_pi_metadata(program_id): try: logging.error('program id is {}'.format(program_id)) fname = '{}/{}/program/{}.xml'.format(TEST_DATA_DIR, DIR_NAME, program_id) with open(fname) as f: y = f.read() from bs4 import BeautifulSoup soup = BeautifulSoup(y, 'lxml') tds = soup.find_all('td') if len(tds) > 0: title = tds[1].contents[0].replace('\n', ' ') pi_name = tds[3].contents[0] metadata = {'title': title, 'pi_name': pi_name} return metadata return None except Exception as e: logging.error(e) import traceback tb = traceback.format_exc() logging.error(tb) def mock_get_votable(url): try: x = url.split('/') filter_name = x[-1].replace('&VERB=0', '') votable = parse_single_table( '{}/votable/{}.xml'.format(TEST_DATA_DIR, filter_name)) return votable, None except Exception as e: logging.error('get_vo_table failure for url {}'.format(url)) logging.error(e) return None, None data_client_mock.return_value.get_file_info.side_effect = \ get_file_info gemini_client_mock.side_effect = get_obs_metadata gemini_pi_mock.side_effect = get_pi_metadata svofps_mock.side_effect = mock_get_votable if os.path.exists(actual_fqn): os.remove(actual_fqn) logging.error('{}'.format(os.path.join(DIR_NAME, input_file))) sys.argv = \ ('{} --quiet --no_validate --local {} ' '--plugin {} --module {} --in {} --out {} --lineage {}'. format(APPLICATION, local, plugin, plugin, input_file, actual_fqn, lineage)).split() print(sys.argv) main_app2() expected_fqn = '{}/{}/{}.xml'.format(TEST_DATA_DIR, DIR_NAME, obs_id) expected = mc.read_obs_from_file(expected_fqn) actual = mc.read_obs_from_file(actual_fqn) result = get_differences(expected, actual, 'Observation') if result: msg = 'Differences found obs id {} n{}'.format( expected.observation_id, '\n'.join([r for r in result])) raise AssertionError(msg)