def build(self, entry):
"""An entry may be a URL, a file name or an obs id."""
self._logger.debug(f'Build StorageName instance for {entry}.')
if self._config.features.use_urls:
result = sn.VlassName(url=entry, entry=entry)
elif self._config.features.use_file_names:
result = sn.VlassName(file_name=entry, entry=entry)
else:
result = sn.VlassName(obs_id=entry, entry=entry)
return result
def test_aug_visit_works(query_endpoint_mock, get_mock):
get_mock.return_value.__enter__.return_value.raw = test_scrape.WL_INDEX
query_endpoint_mock.side_effect = test_scrape._query_endpoint
test_config = mc.Config()
test_config.get_executors()
test_state = mc.State(test_config.state_fqn)
work.init_web_log(test_state, test_config)
test_name = sn.VlassName(
file_name='VLASS1.2.ql.T07t13.J081828-133000.10.2048.v1.I.iter1.'
'image.pbcor.tt0.subim.fits',
entry='VLASS1.2.ql.T07t13.J081828-133000.10.2048.v1.I.iter1.'
'image.pbcor.tt0.subim.fits')
test_file = os.path.join(TEST_DATA_DIR, f'{test_name.obs_id}.xml')
test_obs = mc.read_obs_from_file(test_file)
assert test_obs is not None, 'unexpected None'
data_dir = os.path.join(THIS_DIR, '../../data')
kwargs = {'working_directory': data_dir, 'cadc_client': Mock()}
test_result = time_bounds_augmentation.visit(test_obs, **kwargs)
assert test_obs is not None, 'unexpected modification'
assert test_result is not None, 'should have a result status'
assert len(test_result) == 1, 'modified artifacts count'
assert test_result['artifacts'] == 2, 'artifact count'
plane = test_obs.planes[test_name.product_id]
chunk = plane.artifacts[test_name.file_uri].parts['0'].chunks[0]
assert chunk is not None
assert chunk.time is not None, 'no time information'
assert chunk.time.axis is not None, 'no axis information'
assert chunk.time.axis.bounds is not None, 'no bounds information'
assert len(chunk.time.axis.bounds.samples) == 1, \
'wrong amount of bounds info'
assert chunk.time.exposure == 234.0, 'wrong exposure value'
def _run_single():
"""expects a single file name on the command line"""
config = mc.Config()
config.get_executors()
if config.features.use_file_names:
vlass_name = sn.VlassName(file_name=sys.argv[1], entry=sys.argv[1])
elif config.features.use_urls:
vlass_name = sn.VlassName(url=sys.argv[1], entry=sys.argv[1])
else:
vlass_name = sn.VlassName(obs_id=sys.argv[1], entry=sys.argv[1])
return rc.run_single(config=config,
storage_name=vlass_name,
command_name=sn.APPLICATION,
meta_visitors=META_VISITORS,
data_visitors=DATA_VISITORS,
store_transfer=tc.HttpTransfer())
def test_visit(header_mock, test_files):
obs_id = test_files[0]
header_mock.side_effect = ac.make_headers_from_file
expected_fqn = f'{TEST_DATA_DIR}/{obs_id}.xml'
expected = mc.read_obs_from_file(expected_fqn)
in_fqn = expected_fqn.replace('.xml', '.in.xml')
actual_fqn = expected_fqn.replace('.xml', '.actual.xml')
observation = None
if os.path.exists(in_fqn):
observation = mc.read_obs_from_file(in_fqn)
for f_name in test_files[1:]:
temp_fqn = f'{TEST_DATA_DIR}/{f_name}'
vlass_name = storage_name.VlassName(entry=temp_fqn)
metadata_reader = rdc.FileMetadataReader()
metadata_reader.set(vlass_name)
file_type = 'application/fits'
metadata_reader.file_info[vlass_name.file_uri].file_type = file_type
kwargs = {
'storage_name': vlass_name,
'metadata_reader': metadata_reader,
}
observation = fits2caom2_augmentation.visit(observation, **kwargs)
try:
compare_result = get_differences(expected, observation)
except Exception as e:
mc.write_obs_to_file(observation, actual_fqn)
raise e
if compare_result is not None:
mc.write_obs_to_file(observation, actual_fqn)
compare_text = '\n'.join([r for r in compare_result])
msg = (f'Differences found in observation {expected.observation_id}\n'
f'{compare_text}')
raise AssertionError(msg)
def __init__(self, observation, **kwargs):
super(VlassPreview, self).__init__(
sn.COLLECTION, ReleaseType.META, **kwargs)
self._storage_name = sn.VlassName(file_name=self._science_file)
self._science_fqn = os.path.join(self._working_dir,
self._science_file)
self._preview_fqn = os.path.join(
self._working_dir, self._storage_name.prev)
self._thumb_fqn = os.path.join(
self._working_dir, self._storage_name.thumb)
self._logger = logging.getLogger(__name__)
def test_storage_name():
test_bit = 'VLASS1.2.ql.T23t09.J083851+483000.10.2048.v1.I.iter1.image.' \
'pbcor.tt0'
test_url = 'https://archive-new.nrao.edu/vlass/quicklook/VLASS1.2/' \
'T23t09/VLASS1.2.ql.T23t09.J083851+483000.10.2048.v1/' \
'{}.subim.fits'.format(test_bit)
ts1 = sn.VlassName(url=test_url)
ts2 = sn.VlassName(file_name='{}.subim.fits'.format(test_bit))
for ts in [ts1, ts2]:
assert ts.obs_id == 'VLASS1.2.T23t09.J083851+483000', 'wrong obs id'
assert ts.fname_on_disk == '{}.subim.fits'.format(test_bit), \
'wrong fname on disk'
assert ts.file_name == '{}.subim.fits'.format(test_bit), 'wrong fname'
assert ts.file_id == '{}.subim'.format(test_bit), 'wrong fid'
assert ts.file_uri == \
'ad:VLASS/{}.subim.fits'.format(test_bit), 'wrong uri'
assert ts.model_file_name == \
'VLASS1.2.T23t09.J083851+483000.fits.xml', 'wrong model name'
assert ts.log_file == 'VLASS1.2.T23t09.J083851+483000.log', \
'wrong log file'
assert sn.VlassName.remove_extensions(ts.file_name) == \
'{}.subim'.format(test_bit), 'wrong extensions'
assert ts.epoch == 'VLASS1.2', 'wrong epoch'
assert ts.tile_url == 'https://archive-new.nrao.edu/vlass/quicklook/' \
'VLASS1.2/T23t09/', 'wrong tile url'
assert ts.rejected_url == 'https://archive-new.nrao.edu/vlass/' \
'quicklook/VLASS1.2/QA_REJECTED/', \
'wrong rejected url'
assert ts.image_pointing_url == 'https://archive-new.nrao.edu/vlass/' \
'quicklook/VLASS1.2/T23t09/' \
'VLASS1.2.ql.T23t09.J083851+483000.' \
'10.2048.v1/', \
'wrong image pointing url'
assert ts.prev == f'{test_bit}.subim_prev.jpg', 'wrong preview'
assert ts.thumb == f'{test_bit}.subim_prev_256.jpg', 'wrong thumbnail'
assert ts.prev_uri == f'ad:{sn.COLLECTION}/{test_bit}.subim_prev.jpg', \
'wrong preview uri'
assert ts.thumb_uri == \
f'ad:{sn.COLLECTION}/{test_bit}.subim_prev_256.jpg', \
'wrong thumbnail uri'
def test_preview_augmentation(access_mock):
access_mock.return_value = 'https://localhost'
test_fqn = f'{TEST_DATA_DIR}/preview_augmentation_start.xml'
test_science_f_name = (
'VLASS1.1.ql.T01t01.J000228-363000.10.2048.v1.I.iter1.image.pbcor.'
'tt0.subim.fits')
test_storage_name = sn.VlassName(test_science_f_name)
test_obs = mc.read_obs_from_file(test_fqn)
test_config = mc.Config()
test_rejected = mc.Rejected(f'{TEST_DATA_DIR}/rejected.yml')
test_metrics = mc.Metrics(test_config)
test_observable = mc.Observable(test_rejected, test_metrics)
kwargs = {
'stream': None,
'observable': test_observable,
'storage_name': test_storage_name,
'working_directory': '/test_files',
}
test_subject = preview_augmentation.VlassPreview(**kwargs)
assert test_subject is not None, 'need a test subject'
assert len(test_obs.planes) == 1, 'wrong number of planes'
assert (len(test_obs.planes[test_storage_name.product_id].artifacts) == 4
), 'wrong starting # of artifacts'
test_obs = test_subject.visit(test_obs)
assert test_obs is not None, 'expect a result'
assert (len(test_obs.planes[test_storage_name.product_id].artifacts) == 6
), 'wrong ending # of artifacts'
test_report = test_subject.report
assert test_report is not None
assert 'artifacts' in test_report
assert test_report.get('artifacts') == 2, 'wrong report count'
# does artifact re-naming work?
test_url = (f'https://archive-new.nrao.edu/vlass/quicklook/VLASS1.1/'
f'T01t01/VLASS1.1.ql.T01t01.J000228-363000.10.2048.v1/'
f'{test_science_f_name}')
kwargs = {'url': test_url}
test_obs = cleanup_augmentation.visit(test_obs, **kwargs)
test_artifacts = test_obs.planes[test_storage_name.product_id].artifacts
assert len(test_artifacts) == 4, 'wrong ending conditions'
assert test_storage_name.prev_uri in test_artifacts, 'missing preview'
assert test_storage_name.thumb_uri in test_artifacts, 'missing thumbnail'
def test_aug_visit_position_bounds():
test_file_id = ('VLASS1.2.ql.T24t07.J065836+563000.10.2048.v1.I.'
'iter1.image.pbcor.tt0.subim')
test_input_file = f'/test_files/{test_file_id}.fits'
if not os.path.exists(test_input_file):
shutil.copy(
f'/usr/src/app/vlass2caom2/int_test/test_files/'
f'{test_file_id}.fits',
test_input_file,
)
test_file = os.path.join(TEST_DATA_DIR, 'fpf_start_obs.xml')
test_obs = mc.read_obs_from_file(test_file)
test_storage_name = sn.VlassName(test_input_file)
kwargs = {
'working_directory': '/test_files',
'storage_name': test_storage_name,
'log_file_directory': os.path.join(TEST_DATA_DIR, 'logs'),
}
test_obs = position_bounds_augmentation.visit(test_obs, **kwargs)
assert test_obs is not None, 'should have a returned observation'
return_file = os.path.join(THIS_DIR, f'{test_file_id}__footprint.txt')
assert not os.path.exists(return_file), 'bad cleanup'
if os.path.exists(test_input_file):
os.unlink(test_input_file)
def test_storage_name():
sn.set_use_storage_inventory(True)
test_bit = (
'VLASS1.2.ql.T23t09.J083851+483000.10.2048.v1.I.iter1.image.pbcor.tt0'
)
test_url = (
f'https://archive-new.nrao.edu/vlass/quicklook/VLASS1.2v2/T23t09/'
f'VLASS1.2.ql.T23t09.J083851+483000.10.2048.v1/{test_bit}.subim.fits'
)
ts1 = sn.VlassName(test_url)
ts2 = sn.VlassName(f'{test_bit}.subim.fits')
for ts in [ts1, ts2]:
assert ts.obs_id == 'VLASS1.2.T23t09.J083851+483000', 'wrong obs id'
assert (
ts.product_id == 'VLASS1.2.T23t09.J083851+483000.quicklook'
), 'wrong product id'
assert ts.file_name == f'{test_bit}.subim.fits', 'wrong fname'
assert ts.file_id == f'{test_bit}.subim', 'wrong fid'
assert (
ts.file_uri == f'{sn.SCHEME}:VLASS/{test_bit}.subim.fits'
), 'wrong uri'
assert (
ts.model_file_name == 'VLASS1.2.T23t09.J083851+483000.xml'
), 'wrong model name'
assert (
ts.log_file == 'VLASS1.2.T23t09.J083851+483000.log'
), 'wrong log file'
assert (
sn.VlassName.remove_extensions(ts.file_name) == f'{test_bit}.subim'
), 'wrong extensions'
assert ts.epoch == 'VLASS1.2', 'wrong epoch'
assert (
ts.tile_url == 'https://archive-new.nrao.edu/vlass/quicklook/'
'VLASS1.2/T23t09/'
), 'wrong tile url'
assert (
ts.rejected_url == 'https://archive-new.nrao.edu/vlass/'
'quicklook/VLASS1.2/QA_REJECTED/'
), 'wrong rejected url'
assert (
ts.image_pointing_url == 'https://archive-new.nrao.edu/vlass/'
'quicklook/VLASS1.2/T23t09/VLASS1.2.ql.'
'T23t09.J083851+483000.10.2048.v1/'
), 'wrong image pointing url'
assert ts.prev == f'{test_bit}.subim_prev.jpg', 'wrong preview'
assert ts.thumb == f'{test_bit}.subim_prev_256.jpg', 'wrong thumbnail'
assert (
ts.prev_uri == f'{sn.CADC_SCHEME}:{sn.COLLECTION}/'
f'{test_bit}.subim_prev.jpg'
), 'wrong preview uri'
assert (
ts.thumb_uri == f'{sn.CADC_SCHEME}:{sn.COLLECTION}/'
f'{test_bit}.subim_prev_256.jpg'
), 'wrong thumbnail uri'
sn.set_use_storage_inventory(False)
ts1 = sn.VlassName(test_url)
ts2 = sn.VlassName(f'{test_bit}.subim.fits')
for ts in [ts1, ts2]:
assert (
ts.file_uri == f'{sn.AD_SCHEME}:VLASS/{test_bit}.subim.fits'
), 'wrong uri'
assert (
ts.prev_uri == f'{sn.AD_SCHEME}:{sn.COLLECTION}/'
f'{test_bit}.subim_prev.jpg'
), 'wrong preview uri'
assert (
ts.thumb_uri == f'{sn.AD_SCHEME}:{sn.COLLECTION}/'
f'{test_bit}.subim_prev_256.jpg'
), 'wrong thumbnail uri'