def test_main_app(header_mock, test_name):
header_mock.side_effect = ac.make_headers_from_file
storage_name = main_app.BlankName(entry=test_name)
metadata_reader = rdc.FileMetadataReader()
metadata_reader.set(storage_name)
file_type = 'application/fits'
metadata_reader.file_info[storage_name.file_uri].file_type = file_type
kwargs = {
'storage_name': storage_name,
'metadata_reader': metadata_reader,
}
expected_fqn = f'{TEST_DATA_DIR}/{test_name}.expected.xml'
expected = mc.read_obs_from_file(expected_fqn)
in_fqn = expected_fqn.replace('.expected', '.in')
actual_fqn = expected_fqn.replace('expected', 'actual')
if os.path.exists(actual_fqn):
os.unlink(actual_fqn)
observation = None
if os.path.exists(in_fqn):
observation = mc.read_obs_from_file(in_fqn)
observation = fits2caom2_augmentation.visit(observation, **kwargs)
try:
compare_result = get_differences(expected, observation)
except Exception as e:
mc.write_obs_to_file(observation, actual_fqn)
raise e
if compare_result is not None:
mc.write_obs_to_file(observation, actual_fqn)
compare_text = '\n'.join([r for r in compare_result])
msg = (f'Differences found in observation {expected.observation_id}\n'
f'{compare_text}')
raise AssertionError(msg)
def test_main_app(test_name):
basename = os.path.basename(test_name)
vlite_name = VliteName(file_name=basename)
output_file = '{}/actual.{}.xml'.format(TEST_DATA_DIR, basename)
obs_path = '{}/{}'.format(TEST_DATA_DIR,
'expected.{}.xml'.format(vlite_name.obs_id))
expected = mc.read_obs_from_file(obs_path)
with patch('caom2utils.fits2caom2.CadcDataClient') as data_client_mock:
def get_file_info(archive, file_id):
return {'type': 'application/fits'}
data_client_mock.return_value.get_file_info.side_effect = get_file_info
sys.argv = \
('{} --no_validate --local {} --observation {} {} -o {} '
'--plugin {} --module {} --lineage {}'.
format(APPLICATION, _get_local(test_name), COLLECTION,
test_name, output_file, PLUGIN, PLUGIN,
_get_lineage(test_name))).split()
print(sys.argv)
vlite_main_app()
actual = mc.read_obs_from_file(output_file)
result = get_differences(expected, actual, 'Observation')
if result:
msg = 'Differences found in observation {} test name {}\n{}'. \
format(expected.observation_id, test_name, '\n'.join(
[r for r in result]))
raise AssertionError(msg)
def test_visit(header_mock, test_files):
obs_id = test_files[0]
header_mock.side_effect = ac.make_headers_from_file
expected_fqn = f'{TEST_DATA_DIR}/{obs_id}.xml'
expected = mc.read_obs_from_file(expected_fqn)
in_fqn = expected_fqn.replace('.xml', '.in.xml')
actual_fqn = expected_fqn.replace('.xml', '.actual.xml')
observation = None
if os.path.exists(in_fqn):
observation = mc.read_obs_from_file(in_fqn)
for f_name in test_files[1:]:
temp_fqn = f'{TEST_DATA_DIR}/{f_name}'
vlass_name = storage_name.VlassName(entry=temp_fqn)
metadata_reader = rdc.FileMetadataReader()
metadata_reader.set(vlass_name)
file_type = 'application/fits'
metadata_reader.file_info[vlass_name.file_uri].file_type = file_type
kwargs = {
'storage_name': vlass_name,
'metadata_reader': metadata_reader,
}
observation = fits2caom2_augmentation.visit(observation, **kwargs)
try:
compare_result = get_differences(expected, observation)
except Exception as e:
mc.write_obs_to_file(observation, actual_fqn)
raise e
if compare_result is not None:
mc.write_obs_to_file(observation, actual_fqn)
compare_text = '\n'.join([r for r in compare_result])
msg = (f'Differences found in observation {expected.observation_id}\n'
f'{compare_text}')
raise AssertionError(msg)
def test_visitor(test_name):
storage_name = OmmName(file_name=basename(test_name).replace(
'.header', '.gz'), )
file_info = FileInfo(id=storage_name.file_uri,
file_type='application/fits')
headers = ac.make_headers_from_file(test_name)
metadata_reader = rdc.FileMetadataReader()
metadata_reader._headers = {storage_name.file_uri: headers}
metadata_reader._file_info = {storage_name.file_uri: file_info}
kwargs = {
'storage_name': storage_name,
'metadata_reader': metadata_reader,
}
observation = None
input_file = f'{TEST_DATA_DIR}/in.{storage_name.product_id}.fits.xml'
if exists(input_file):
observation = mc.read_obs_from_file(input_file)
observation = fits2caom2_augmentation.visit(observation, **kwargs)
expected_fqn = (f'{TEST_DATA_DIR}/{storage_name.file_id}.expected.xml')
expected = mc.read_obs_from_file(expected_fqn)
compare_result = get_differences(expected, observation)
if compare_result is not None:
actual_fqn = expected_fqn.replace('expected', 'actual')
mc.write_obs_to_file(observation, actual_fqn)
compare_text = '\n'.join([r for r in compare_result])
msg = (f'Differences found in observation {expected.observation_id}\n'
f'{compare_text}')
raise AssertionError(msg)
def compare(expected_fqn, actual_fqn, observation):
expected = mc.read_obs_from_file(expected_fqn)
compare_result = get_differences(expected, observation)
if compare_result is not None:
mc.write_obs_to_file(observation, actual_fqn)
compare_text = '\n'.join([r for r in compare_result])
msg = (f'Differences found in observation {expected.observation_id}\n'
f'{compare_text}. Check {actual_fqn}')
raise AssertionError(msg)
def _compare_observations(expected, actual, output_dir):
result = get_differences(expected, actual, 'Observation')
if result:
msg = 'Differences found observation {} in {}\n{}'.\
format(expected.observation_id,
output_dir, '\n'.join([r for r in result]))
raise AssertionError(msg)
else:
logging.info('Observation {} in {} match'.format(
expected.observation_id, output_dir))
def _compare_observations(expected, actual, output_dir):
result = get_differences(expected, actual, 'Observation')
if result:
msg = 'Differences found observation {} in {}\n{}'.\
format(expected.observation_id,
output_dir, '\n'.join([r for r in result]))
raise AssertionError(msg)
else:
logging.info('Observation {} in {} match'.format(
expected.observation_id, output_dir))
def compare(ex_fqn, act_fqn, entry):
ex = mc.read_obs_from_file(ex_fqn)
act = mc.read_obs_from_file(act_fqn)
result = get_differences(ex, act, 'Observation')
if result:
result_str = '\n'.join([r for r in result])
msg = f'Differences found obs id {ex.observation_id} ' \
f'file id {entry} ' \
f'instr {ex.instrument.name}\n{result_str}'
return msg
return None
def _compare_observations(expected, actual, output_dir):
result = get_differences(expected, actual, 'Observation')
if result:
tmp = '\n'.join([r for r in result])
msg = f'Differences found observation {expected.observation_id} in ' \
f'{output_dir}\n{tmp}'
_write_observation(actual)
raise AssertionError(msg)
else:
logging.info('Observation {} in {} match'.format(
expected.observation_id, output_dir))
def compare(ex_fqn, act_fqn, entry):
"""Run get_differences for two files on disk.
:param ex_fqn: Fully-qualified file name for the expected observation
:param act_fqn: Fully-qualified file name for the actual observation
:param entry: String for logging information.
:return:
"""
ex = mc.read_obs_from_file(ex_fqn)
act = mc.read_obs_from_file(act_fqn)
result = get_differences(ex, act, 'Observation')
if result:
result_str = '\n'.join([r for r in result])
msg = (f'Differences found obs id {ex.observation_id} file id {entry} '
f'instr {ex.instrument.name}\n{result_str}')
return msg
return None
def test_main_app(test_name):
basename = os.path.basename(test_name)
obs_id = AskapName.get_obs_id(basename)
product_id = AskapName.get_product_id(basename)
lineage = mc.get_lineage(COLLECTION, product_id,
basename.replace('.header', ''),
AskapName.scheme())
output_file = '{}.actual.xml'.format(test_name)
local = test_name
plugin = PLUGIN
with patch('caom2utils.fits2caom2.CadcDataClient') as data_client_mock:
def get_file_info(archive, file_id):
if '_prev' in file_id:
return {
'size': 10290,
'md5sum': md5('-37'.encode()).hexdigest(),
'type': 'image/jpeg'
}
else:
return {
'size': 37,
'md5sum': md5('-37'.encode()).hexdigest(),
'type': 'application/fits'
}
data_client_mock.return_value.get_file_info.side_effect = \
get_file_info
sys.argv = \
('{} --no_validate --local {} '
'--plugin {} --module {} --observation {} {} -o {} --lineage {}'.
format(APPLICATION, local, plugin, plugin, COLLECTION, obs_id,
output_file, lineage)).split()
print(sys.argv)
caom_main()
obs_path = _get_expected_name(test_name)
expected = mc.read_obs_from_file(obs_path)
actual = mc.read_obs_from_file(output_file)
result = get_differences(expected, actual, 'Observation')
if result:
msg = 'Differences found in observation {}\n{}'. \
format(expected.observation_id, '\n'.join(
[r for r in result]))
raise AssertionError(msg)
def test_visit(query_endpoint_mock, get_mock):
get_mock.return_value.__enter__.return_value.raw = WL_INDEX
test_id = ('VLASS1.2_T07t14.J084202-123000_P35696v1_2019_03_11T23_06_04.'
'128/')
query_endpoint_mock.side_effect = _query_endpoint
scrape.init_web_log_content({'VLASS1.2': TEST_START_TIME})
scrape.web_log_content[test_id] = TEST_START_TIME - timedelta(hours=1)
test_fqn = os.path.join(TEST_DATA_DIR, TEST_OBS_ID)
test_obs = mc.read_obs_from_file(f'{test_fqn}.xml')
kwargs = {'cadc_client': Mock()}
test_obs = time_bounds_augmentation.visit(test_obs, **kwargs)
assert test_obs is not None, 'expected a result'
obs_path = os.path.join(TEST_DATA_DIR, 'visit.xml')
expected_obs = mc.read_obs_from_file(obs_path)
result = get_differences(expected_obs, test_obs, 'Observation')
if result:
msg = 'Differences found in observation {}\n{}'.format(
TEST_OBS_ID, '\n'.join([r for r in result]))
raise AssertionError(msg)
def test_main_app(test_files):
obs_id = test_files[0]
obs_path = os.path.join(TEST_DATA_DIR, '{}.xml'.format(obs_id))
expected = mc.read_obs_from_file(obs_path)
if obs_id.endswith('r'):
obs_path = os.path.join(TEST_DATA_DIR, '{}.in.xml'.format(obs_id))
input_param = '--in {}'.format(obs_path)
else:
input_param = '--observation {} {}'.format(COLLECTION, obs_id)
lineage = _get_lineage(obs_id, test_files)
output_file = '{}.actual.xml'.format(obs_id)
local = _get_local(test_files[1:])
with patch('caom2utils.fits2caom2.CadcDataClient') as data_client_mock:
def get_file_info(archive, file_id):
if file_id == a:
return {'size': 55425600,
'md5sum': 'ae2a33238c5051611133e7090560fd8a',
'type': 'application/fits'}
else:
return {'size': 55425600,
'md5sum': '40f7c2763f92ea6e9c6b0304c569097e',
'type': 'application/fits'}
data_client_mock.return_value.get_file_info.side_effect = \
get_file_info
sys.argv = \
('vlass2caom2 --local {} {} -o {} --plugin {} --module {} '
'--lineage {}'.format(local, input_param, output_file, PLUGIN,
PLUGIN, lineage)).split()
print(sys.argv)
to_caom2()
actual = mc.read_obs_from_file(output_file)
result = get_differences(expected, actual, 'Observation')
if result:
msg = 'Differences found in observation {}\n{}'. \
format(expected.observation_id, '\n'.join(
[r for r in result]))
raise AssertionError(msg)
def test_gen():
# from astroquery.alma import Alma
# from astropy.table import Table
# db_table = Alma().query(payload={'project_code': '2016.1.00010.S'})
# # db_table.write('{}/alma_query.xml'.format(TEST_DATA_DIR),
# # format='votable')
# db_table.write('{}/alma_query2.xml'.format(TEST_DATA_DIR), format='html')
# FYI - this has values
# import logging
# logging.error(db_table['Release date'])
# assert False
import logging
from caom2.diff import get_differences
from caom2pipe import manage_composable as mc
from almaca2caom2 import main_app as ma
from astropy.table import Table
db_fqn = '{}/alma_query.html'.format(ALMA_QUERY_DIR)
db_content = Table.read(db_fqn, format='html')
obs_lookup = {}
obs = None
entries = glob.glob('/data/calibrated/*ms.split.cal')
for entry in entries:
fqn = f'{entry}/md.pk'
if not os.path.exists(fqn):
logging.error('missing md for {}'.format(fqn))
continue
logging.error(entry)
try:
almaca_name = ma.AlmacaName(os.path.basename(entry))
except mc.CadcException:
# skip the input MS's
continue
if obs is not None and almaca_name.obs_id != obs.observation_id:
# logging.error('storing {}'.format(obs.observation_id))
obs_lookup[obs.observation_id] = obs
obs = obs_lookup.get(almaca_name.obs_id)
try:
obs = ma.build_observation(db_content, obs, fqn)
except Exception as e:
import traceback
logging.error(traceback.format_exc())
logging.error(fqn)
assert False
errors_found = False
for obs in obs_lookup:
obs_fqn = f'{ALMA_QUERY_DIR}/{obs}.actual.xml'
logging.error(f'checking fqn {obs_fqn}')
actual = obs_lookup.get(obs)
mc.write_obs_to_file(actual, obs_fqn)
expected_fqn = f'{ALMA_QUERY_DIR}/{actual.observation_id}.expected.xml'
if os.path.exists(expected_fqn):
expected = mc.read_obs_from_file(expected_fqn)
result = get_differences(expected, actual, 'Observation')
if result:
msg_str = '\n'.join([r for r in result])
msg = f'Differences found {actual.observation_id}\n{msg_str}'
logging.error(msg)
errors_found = True
# assert False, obs_fqn
else:
msg = f'Could not find {expected_fqn}'
logging.error(msg)
raise AssertionError(msg)
if errors_found:
raise AssertionError('Final failure.')
def test_main_app(test_name):
obs_id = test_name
lineage = _get_lineage(obs_id)
input_file = '{}/{}/{}.in.xml'.format(TEST_DATA_DIR, DIR_NAME, obs_id)
actual_fqn = '{}/{}/{}.actual.xml'.format(TEST_DATA_DIR, DIR_NAME, obs_id)
local = _get_local(test_name)
plugin = PLUGIN
with patch('caom2utils.fits2caom2.CadcDataClient') as data_client_mock, \
patch('gem2caom2.external_metadata.get_obs_metadata') as gemini_client_mock, \
patch('gem2caom2.external_metadata.get_pi_metadata') as gemini_pi_mock, \
patch('gem2caom2.svofps.get_vo_table') as svofps_mock:
def get_file_info(archive, file_id):
if '_prev' in file_id:
return {'size': 10290,
'md5sum': 'md5:{}'.format(
md5('-37'.encode()).hexdigest()),
'type': 'image/jpeg'}
else:
return {'size': 665345,
'md5sum': 'md5:a347f2754ff2fd4b6209e7566637efad',
'type': 'application/fits'}
def get_obs_metadata(file_id):
try:
logging.error('obs metadata file_id {}'.format(file_id))
fname = '{}/{}/json/{}.json'.format(TEST_DATA_DIR, DIR_NAME,
file_id)
with open(fname) as f:
y = json.loads(f.read())
em.om.add(y, file_id)
except Exception as e:
logging.error(e)
import traceback
tb = traceback.format_exc()
logging.error(tb)
def get_pi_metadata(program_id):
try:
logging.error('program id is {}'.format(program_id))
fname = '{}/{}/program/{}.xml'.format(TEST_DATA_DIR, DIR_NAME,
program_id)
with open(fname) as f:
y = f.read()
from bs4 import BeautifulSoup
soup = BeautifulSoup(y, 'lxml')
tds = soup.find_all('td')
if len(tds) > 0:
title = tds[1].contents[0].replace('\n', ' ')
pi_name = tds[3].contents[0]
metadata = {'title': title,
'pi_name': pi_name}
return metadata
return None
except Exception as e:
logging.error(e)
import traceback
tb = traceback.format_exc()
logging.error(tb)
def mock_get_votable(url):
try:
x = url.split('/')
filter_name = x[-1].replace('&VERB=0', '')
votable = parse_single_table(
'{}/votable/{}.xml'.format(TEST_DATA_DIR, filter_name))
return votable, None
except Exception as e:
logging.error('get_vo_table failure for url {}'.format(url))
logging.error(e)
return None, None
data_client_mock.return_value.get_file_info.side_effect = \
get_file_info
gemini_client_mock.side_effect = get_obs_metadata
gemini_pi_mock.side_effect = get_pi_metadata
svofps_mock.side_effect = mock_get_votable
if os.path.exists(actual_fqn):
os.remove(actual_fqn)
logging.error('{}'.format(os.path.join(DIR_NAME, input_file)))
sys.argv = \
('{} --quiet --no_validate --local {} '
'--plugin {} --module {} --in {} --out {} --lineage {}'.
format(APPLICATION, local, plugin, plugin,
input_file, actual_fqn, lineage)).split()
print(sys.argv)
main_app2()
expected_fqn = '{}/{}/{}.xml'.format(TEST_DATA_DIR, DIR_NAME, obs_id)
expected = mc.read_obs_from_file(expected_fqn)
actual = mc.read_obs_from_file(actual_fqn)
result = get_differences(expected, actual, 'Observation')
if result:
msg = 'Differences found obs id {} n{}'.format(
expected.observation_id, '\n'.join([r for r in result]))
raise AssertionError(msg)
