def change_to_simple(observation):
"""For the case where a DerivedObservation needs to become a
SimpleObservation."""
temp = SimpleObservation(
observation.collection, observation.observation_id,
Algorithm('exposure'), observation.sequence_number, observation.intent,
observation.type, observation.proposal, observation.telescope,
observation.instrument, observation.target, observation.meta_release,
observation.meta_read_groups, observation.planes,
observation.environment, observation.target_position)
temp.last_modified = datetime.utcnow()
temp._id = observation._id
return temp
def test_run_todo_file_data_source_v(
repo_read_mock, set_clients_mock, test_config
):
set_clients_mock.side_effect = _clients_mock
test_config.features.supports_latest_client = True
test_cert_file = os.path.join(TEST_DIR, 'test_proxy.pem')
test_config.proxy_fqn = test_cert_file
repo_read_mock.return_value = SimpleObservation(
collection=test_config.collection,
observation_id='def',
algorithm=Algorithm(str('test')),
)
if os.path.exists(test_config.success_fqn):
os.unlink(test_config.success_fqn)
test_config.work_fqn = f'{TEST_DIR}/todo.txt'
test_config.task_types = [mc.TaskType.VISIT]
test_config.log_to_file = True
test_chooser = ec.OrganizeChooser()
test_result = rc.run_by_todo(
config=test_config, chooser=test_chooser, command_name=TEST_COMMAND
)
assert test_result is not None, 'expect a result'
assert test_result == 0, 'expect success'
assert os.path.exists(test_config.success_fqn), 'expect success file'
with open(test_config.success_fqn) as f:
content = f.read()
# the obs id and file name
assert 'def def.fits' in content, 'wrong success message'
assert repo_read_mock.called, 'expect e call'
repo_read_mock.assert_called_with(), 'wrong e args'
def test_invoke_gem2caom2():
test_obs = SimpleObservation(collection='test', observation_id='1')
def main_app2_mock():
args = fits2caom2.get_gen_proc_arg_parser().parse_args()
if (args is None or (args.no_validate is False)
or (args.observation[0] != 'GEMINI')
or (args.observation[1] != 'GS-2010A-Q-36-5-246') or
(args.external_url[0] !=
'https://archive.gemini.edu/fullheader/rgS20100212S0301.fits')
or (args.plugin != '/app/gem2caom2/gem2caom2/main_app.py')
or (args.module != args.plugin)
or (args.lineage !=
['rgS20100212S0301/gemini:GEM/rgS20100212S0301.fits'])):
raise RuntimeError(args)
if em.gofr is None:
em.gofr = gem2caom2.GemObsFileRelationship('/app/data/from_paul.txt')
main_app_orig = gem2caom2.main_app2
gem2caom2.main_app2 = Mock(side_effect=main_app2_mock)
read_obs_orig = caom2pipe.manage_composable.read_obs_from_file
caom2pipe.manage_composable.read_obs_from_file = Mock(
return_value=test_obs)
try:
result = c._invoke_gem2caom2('GS-2010A-Q-36-5-246-RG')
gem2caom2.main_app2.assert_called_with(), 'command line failure'
assert not caom2pipe.manage_composable.read_obs_from_file.called, \
'read obs should not be called'
assert result is None, 'should be executing failure case'
finally:
gem2caom2.main_app2 = main_app_orig
caom2pipe.manage_composable.read_obs_from_file = read_obs_orig
def _cmd_direct_mock():
from caom2 import SimpleObservation, Algorithm
obs = SimpleObservation(observation_id='VLASS1.2.T07t13.J083838-153000',
collection=COLLECTION,
algorithm=Algorithm(name='testing'))
mc.write_obs_to_file(
obs, '/usr/src/app/logs/VLASS1.2.T07t13.J083838-153000.fits.xml')
def test_run_todo_file_data_source(clients_mock, test_config):
clients_mock.return_value.data_client.get_file_info.return_value = None
clients_mock.return_value.metadata_client.read.return_value = (
SimpleObservation(
collection=test_config.collection,
observation_id='def',
algorithm=Algorithm(str('test')),
))
if os.path.exists(test_config.success_fqn):
os.unlink(test_config.success_fqn)
test_config.work_fqn = f'{TEST_DIR}/todo.txt'
test_config.task_types = [mc.TaskType.VISIT]
test_config.log_to_file = True
test_chooser = ec.OrganizeChooser()
test_result = rc.run_by_todo(config=test_config,
chooser=test_chooser,
command_name=TEST_COMMAND)
assert test_result is not None, 'expect a result'
assert test_result == 0, 'expect success'
assert os.path.exists(test_config.success_fqn), 'expect success file'
with open(test_config.success_fqn) as f:
content = f.read()
# the obs id and file name
assert 'def def.fits' in content, 'wrong success message'
def test_todo_local_common(
caom_mock,
access_mock,
fits2caom2_mock,
test_input_name,
):
if test_input_name not in ['OMM_TODO_LOCAL', 'NEOSSAT_TODO_LOCAL']:
return
test_input = INPUTS.get(test_input_name)
_cleanup()
_setup(test_input, local=True)
access_mock.return_value = 'https://localhost'
getcwd_orig = os.getcwd
os.getcwd = Mock(return_value=TEST_EXEC_DIR)
# import the module for execution
sys.path.append(test_input.test_path)
test_module = import_module('composable')
caom_mock.return_value.data_client.info.side_effect = [
None,
FileInfo(
id=test_input.test_uri,
md5sum='3d29f0edd984065a044d1376a11c6f08',
),
]
caom_mock.return_value.metadata_client.read.side_effect = [
None,
SimpleObservation(
'obs_id',
test_input.collection,
Algorithm(name='exposure'),
),
]
def _info(uri):
assert (uri == test_input.test_uri), 'wrong info uri'
return FileInfo(
id=uri,
md5sum='abc',
size=42,
)
fits2caom2_mock.return_value.info.side_effect = _info
try:
test_result = test_module._run()
assert test_result is not None, f'expect a result {test_input_name}'
assert test_result == 0, f'wrong test result {test_input_name}'
assert (caom_mock.return_value.data_client.put.called
), f'{test_input_name} put not called'
caom_mock.return_value.data_client.put.assert_called_with(
'/usr/src/app/integration_test/mock_test/data/test_files',
test_input.test_uri,
None,
), f'{test_input_name} wrong put args'
finally:
os.getcwd = getcwd_orig
def _write_obs_mock():
args = get_gen_proc_arg_parser().parse_args()
obs = SimpleObservation(
collection=args.observation[0],
observation_id=args.observation[1],
algorithm=Algorithm(name='exposure'),
)
mc.write_obs_to_file(obs, args.out_obs_xml)
def _cmd_direct_mock():
from caom2 import SimpleObservation, Algorithm
obs = SimpleObservation(
observation_id='VLASS1.2.T07t13.J083838-153000',
collection=COLLECTION,
algorithm=Algorithm(name='testing'),
)
return obs
def to_caom2():
plugin = ('/usr/local/lib/python3.9/site-packages/test_execute_composable/'
'test_execute_composable.py')
assert sys.argv is not None, 'expect sys.argv to be set'
local_meta_create_answer = [
'test_execute_composable',
'--verbose',
'--observation',
'OMM',
'test_obs_id',
'--local',
f'{tc.TEST_DATA_DIR}/test_file.fits.gz',
'--out',
f'{tc.THIS_DIR}/test_obs_id/test_obs_id.xml',
'--plugin',
f'{plugin}',
'--module',
f'{plugin}',
'--lineage',
'test_obs_id/cadc:TEST/test_obs_id.fits.gz',
]
scrape_answer = [
'test_execute_composable',
'--verbose',
'--not_connected',
'--observation',
'OMM',
'test_obs_id',
'--local',
f'{tc.TEST_DATA_DIR}/test_file.fits.gz',
'--out',
f'{tc.TEST_DATA_DIR}/test_obs_id/test_obs_id.xml',
'--plugin',
f'{plugin}',
'--module',
f'{plugin}',
'--lineage',
'test_obs_id/cadc:TEST/test_obs_id.fits.gz',
]
# TaskType.SCRAPE (Scrape)
if sys.argv != scrape_answer:
# TaskType.INGEST (LocalMetaCreate)
if sys.argv != local_meta_create_answer:
assert False, (f'wrong sys.argv values \n{sys.argv} '
f'\n{local_meta_create_answer}')
fqn_index = sys.argv.index('--out') + 1
fqn = sys.argv[fqn_index]
mc.write_obs_to_file(
SimpleObservation(
collection='test_collection',
observation_id='test_obs_id',
algorithm=Algorithm(str('exposure')),
),
fqn,
)
def _mock_write():
fqn = f'{tc.THIS_DIR}/NEOS_SCI_2015347000000_clean/' \
f'NEOS_SCI_2015347000000_clean.fits.xml'
mc.write_obs_to_file(
SimpleObservation(
collection='test_collection',
observation_id='ghi',
algorithm=Algorithm(str('test'))
),
fqn,
)
def test_validate_observation():
obs = SimpleObservation('test_collection', 'test_obs_id',
Algorithm('test_name'))
validate(obs)
obs = DerivedObservation('test_collection', 'test_obs_id',
Algorithm('test_name'),
proposal=Proposal('test_proposal'),
telescope=Telescope('test_telescope'),
instrument=Instrument('test_instrument'),
target=Target('test_targets'))
obs.algorithm.keywords = 'foo'
obs.proposal.keywords = set('foo=42')
obs.telescope.keywords = set('foo:42')
obs.instrument.keywords.add("tick'marks")
obs.target.keywords = set('has multiple spaces')
test_plane = Plane('test_plane')
test_plane.provenance = Provenance('test_provenance')
test_plane.provenance.keywords.add('pipe|denied')
obs.planes['test_plane'] = test_plane
with pytest.raises(AssertionError):
validate(obs)
def test_read_write_obs_with_file():
if os.path.exists(TEST_OBS_FILE):
os.unlink(TEST_OBS_FILE)
mc.write_obs_to_file(
SimpleObservation(
collection='test_collection',
observation_id='test_obs_id',
algorithm=Algorithm(str('exposure')),
),
TEST_OBS_FILE,
)
test_subject = mc.read_obs_from_file(TEST_OBS_FILE)
assert test_subject is not None, 'expect a result'
assert isinstance(test_subject, SimpleObservation), 'wrong read'
def test_augment_observation():
test_obs_blueprint = ObsBlueprint(position_axes=(1, 2))
test_obs_blueprint.set('Observation.target.name', 'CGPS Mosaic MA1')
test_obs_blueprint.set('Observation.target.standard', False)
test_obs_blueprint.set('Observation.telescope.name', 'DRAO-ST')
test_obs_blueprint.set('Observation.instrument.name', 'DRAO-ST')
test_obs_blueprint.set('Observation.telescope.geoLocationX',
'-2100330.87517')
test_obs_blueprint.set('Observation.telescope.geoLocationY',
'-3694247.82445')
test_obs_blueprint.set('Observation.telescope.geoLocationZ',
'4741018.33097')
test_obs_blueprint.set('Plane.dataProductType', 'cube')
test_obs_blueprint.set('Artifact.productType', 'info')
test_obs_blueprint.set('Artifact.releaseType', 'data')
test_obs_blueprint.set('Plane.calibrationLevel', '2')
test_fitsparser = FitsParser(sample_file_4axes_obs, test_obs_blueprint)
test_fitsparser.blueprint = test_obs_blueprint
test_obs = SimpleObservation('collection', 'MA1_DRAO-ST',
Algorithm('exposure'))
test_fitsparser.augment_observation(test_obs,
sample_file_4axes_uri,
product_id='HI-line')
assert test_obs is not None
assert test_obs.planes is not None
assert len(test_obs.planes) == 1
test_plane = test_obs.planes['HI-line']
assert test_plane.artifacts is not None
assert len(test_plane.artifacts) == 1
test_artifact = test_plane.artifacts[sample_file_4axes_uri]
assert test_artifact is not None
test_part = test_artifact.parts['0']
# remove the chunk bit, as it's part of other tests -
# results in <caom2:chunks/> xml output
test_part.chunks.pop()
output = BytesIO()
ow = ObservationWriter(False, False, "caom2",
obs_reader_writer.CAOM20_NAMESPACE)
ow.write(test_obs, output)
result = output.getvalue().decode('UTF-8')
output.close()
expected = _get_obs(EXPECTED_OBS_XML)
actual = _get_obs(result)
diff_result = get_differences(expected, actual, 'Observation')
assert diff_result is None
def visit(self):
for uri, file_info in self._metadata_reader.file_info.items():
headers = self._metadata_reader.headers.get(uri)
telescope_data = Telescope(uri, headers)
blueprint = ObsBlueprint(instantiated_class=telescope_data)
telescope_data.accumulate_bp(blueprint)
if len(headers) == 0:
parser = GenericParser(blueprint, uri)
else:
parser = FitsParser(headers, blueprint, uri)
if self._dump_config:
print(f'Blueprint for {uri}: {blueprint}')
if self._observation is None:
if blueprint._get('DerivedObservation.members') is None:
self._logger.debug('Build a SimpleObservation')
self._observation = SimpleObservation(
collection=self._storage_name.collection,
observation_id=self._storage_name.obs_id,
algorithm=Algorithm('exposure'),
)
else:
self._logger.debug('Build a DerivedObservation')
self._observation = DerivedObservation(
collection=self._storage_name.collection,
observation_id=self._storage_name.obs_id,
algorithm=Algorithm('composite'),
)
parser.augment_observation(
observation=self._observation,
artifact_uri=uri,
product_id=self._storage_name.product_id,
)
self._observation = telescope_data.update(self._observation,
self._storage_name,
file_info)
return self._observation
def test_run_todo_file_data_source(
caps_mock,
ad_mock,
data_client_mock,
set_clients_mock,
test_config
):
set_clients_mock.side_effect = _clients_mock
caps_mock.return_value = 'https://sc2.canfar.net/sc2repo'
response = Mock()
response.status_code = 200
response.iter_content.return_value = [b'fileName\n']
ad_mock.return_value.__enter__.return_value = response
data_client_mock.return_value = SimpleObservation(
collection=test_config.collection, observation_id='def',
algorithm=Algorithm(str('test'))
)
if os.path.exists(test_config.success_fqn):
os.unlink(test_config.success_fqn)
test_config.work_fqn = f'{TEST_DIR}/todo.txt'
test_config.task_types = [mc.TaskType.VISIT]
test_config.log_to_file = True
test_chooser = ec.OrganizeChooser()
test_result = rc.run_by_todo(
config=test_config, chooser=test_chooser, command_name=TEST_COMMAND
)
assert test_result is not None, 'expect a result'
assert test_result == 0, 'expect success'
assert os.path.exists(test_config.success_fqn), 'expect success file'
with open(test_config.success_fqn) as f:
content = f.read()
# the obs id and file name
assert 'def def.fits' in content, 'wrong success message'
def _mock_repo_read(collection, obs_id):
return SimpleObservation(collection=collection, observation_id=obs_id)
def mock_read(collection, obs_id):
return SimpleObservation(
collection=collection,
observation_id=obs_id,
algorithm=Algorithm('exposure'),
)
def stuff(args):
obs = None
index = 0
for f_name in args.local:
product_id = args.lineage[index].split('/')[0]
t_header = Table.read(f_name, format='hdf5', path='header')
# logging.error(t_header.colnames)
# ['VERSION_MAJOR', 'VERSION_MINOR', 'TIME_IN_SEC',
# 'TIME_IN_MICROSEC', 'RUN_ID', 'ORIGIN', 'OBSMODE', 'FIELD', 'RA',
# 'DEC', 'EXPTIME', 'NUM_IMAGER']
# logging.error(t_header)
# t_header['RUN_ID'].data[0].decode() - return this string
# 20190805T024026
# logging.error(t_header['RUN_ID'].data[0].decode())
release_date = datetime.strptime(
t_header['RUN_ID'].data[0].decode(), '%Y%m%dT%H%M%S')
t_image = Table.read(f_name, format='hdf5', path='image')
# logging.error(t_image.colnames)
# ['col0', 'col1', 'col2']
# logging.error(t_image)
# logging.error(t_image['col0'].data[0])
# logging.error(t_image['col0'].data[143999])
t_catalog = Table.read(f_name, format='hdf5', path='catalog')
# logging.error(t_catalog.colnames)
# ['CAT_ID', 'GAIA_ID', '2MASS_ID', 'RA', 'DEC', 'TAOS_MAG',
# 'GAIA_MAG', '2MASS_JMAG']
# logging.error(t_catalog)
t_imager = Table.read(f_name, format='hdf5', path='imager')
# logging.error(t_imager.colnames)
# ['TEL_ID', 'CAM_ID', 'IMGR_ID', 'XLOC', 'YLOC']
# logging.error(t_imager)
t_moment = Table.read(f_name, format='hdf5', path='moment')
# logging.error(t_moment.colnames)
# ['col0', 'col1', 'col2']
# logging.error(t_moment)
t_window = Table.read(f_name, format='hdf5', path='window')
# logging.error(t_window.colnames)
# ['X0', 'X1', 'Y0', 'Y1', 'XC', 'YC']
# logging.error(t_window)
t_wcs= Table.read(f_name, format='hdf5', path='/wcs/cdmatrix')
# ['CRVAL1','CRVAL2','CRPIX1','CRPIX2','CD1_1','CD1_2','CD2_1','CD2_2']
# logging.error(t_wcs)
taos = Telescope(name='TAOS',
geo_location_x=-2354953.99637757,
geo_location_y=-4940160.3636381,
geo_location_z=3270123.70695983)
target = Target(name=str(t_header['FIELD'].data[0]),
target_type=TargetType.FIELD,
standard=None,
redshift=None,
keywords=None,
moving=None)
proposal = Proposal(id=COLLECTION,
pi_name=None,
project=COLLECTION,
title=None)
obs = SimpleObservation(collection=COLLECTION,
observation_id=args.observation[1],
sequence_number=None,
intent=ObservationIntentType.SCIENCE,
type='FIELD',
proposal=proposal,
telescope=taos,
instrument=None,
target=target,
meta_release=release_date)
provenance = Provenance(name=COLLECTION,
version='{}.{}'.format(
t_header['VERSION_MAJOR'].data[0],
t_header['VERSION_MINOR'].data[0]),
project=COLLECTION,
producer=COLLECTION,
run_id=t_header['RUN_ID'].data[0].decode(),
reference='https://taos2.asiaa.sinica.edu.tw/',
last_executed=release_date)
plane = Plane(product_id=product_id,
data_release=release_date,
meta_release=release_date,
provenance=provenance,
data_product_type=DataProductType.IMAGE,
calibration_level=CalibrationLevel.RAW_STANDARD)
artifact = mc.get_artifact_metadata(
f_name, ProductType.SCIENCE, ReleaseType.DATA,
mc.build_uri(COLLECTION, os.path.basename(f_name)))
# parts are always named '0'
part = Part('0')
# do each of the three telescopes
for telescope in [0, 1, 2]:
position = build_position(t_wcs,
t_window,
telescope)
time = build_time(t_header['TIME_IN_SEC'].data[0],
t_header['TIME_IN_MICROSEC'].data[0])
energy = build_energy()
chunk = Chunk(naxis=4,
position_axis_1=1,
position_axis_2=2,
energy_axis=3,
time_axis=4,
position=position,
energy=energy,
time=time)
part.chunks.append(chunk)
artifact.parts.add(part)
plane.artifacts.add(artifact)
obs.planes.add(plane)
index += 1
return obs
def _mock_repo_read_not_none(arg1, arg2):
return SimpleObservation(
observation_id='TEST_OBS_ID',
collection='TEST',
algorithm=Algorithm(name='exposure'),
instrument=Instrument(name=metadata.Inst.MEGAPRIME.value))
def _mock_repo_read(ignore_client, collection, obs_id, ignore_metrics):
return SimpleObservation(obs_id, collection, Algorithm(name='exposure'))
def test_gem_todo(
data_mock,
caom_mock,
json_mock,
filter_mock,
http_get_mock,
tap_mock,
external_header_mock,
test_input_name,
):
if 'GEM_TODO' != test_input_name:
return
test_input = INPUTS.get(test_input_name)
_cleanup()
getcwd_orig = os.getcwd
os.getcwd = Mock(return_value=TEST_EXEC_DIR)
_setup(test_input)
todo_fqn = TEST_EXEC_DIR / 'todo.txt'
with open(todo_fqn, 'w') as f1:
f1.write('S20191214S0301.fits\n')
def _json_mock(url, ignore_session):
response = Mock()
response.close = Mock()
fqn = test_input.input_dir / 'input.json'
with open(fqn) as f:
response.text = f.read()
def x():
return json.loads(response.text)
response.json = x
return response
json_mock.side_effect = _json_mock
def _filter_mock():
from astropy.table import parse_single_table
fqn = test_input.input_dir / 'filter.xml'
content = parse_single_table(fqn)
return content, None
filter_mock.side_effect = _filter_mock
caom_mock.return_value.metadata_client.read.side_effect = [
None,
SimpleObservation(
'obs_id',
'GEMINI',
Algorithm(name='exposure'),
),
]
def _tap_query(
ignore_query,
output_file,
data_only=True,
response_format='csv',
):
output_file.write(
'observationID,instrument_name\n'
'GS-CAL20191214-1-029,F2\n', )
# caom_mock.return_value.query_client.query.side_effect = _tap_query
tap_mock.return_value.query.side_effect = _tap_query
def _get_mock(ignore, uri):
if uri == 'gemini:GEMINI/S20191214S0301.jpg':
raise exceptions.UnexpectedException('')
caom_mock.return_value.data_client.get.side_effect = _get_mock
def _http_get_mock(ignore_url, fqn):
if (ignore_url != 'https://archive.gemini.edu/file/S20191214S0301.fits'
and ignore_url !=
'https://archive.gemini.edu/preview/S20191214S0301.fits'):
assert False, f'bad http get mock url {ignore_url}'
for ext in ['fits', 'jpg']:
test_src_fqn = test_input.input_dir / f'S20191214S0301.{ext}'
shutil.copy(test_src_fqn, fqn)
http_get_mock.side_effect = _http_get_mock
def _external_header(ignore_url):
assert (ignore_url ==
'https://archive.gemini.edu/fullheader/S20191214S0301.fits'
), 'wrong file header url'
fqn = test_input.input_dir / 'S20191214S0301.fits'
return caom2utils.data_util.get_local_file_headers(fqn.as_posix())
external_header_mock.side_effect = _external_header
def _info(uri):
return FileInfo(
id=uri,
md5sum='abc',
size=42,
)
data_mock.return_value.info.side_effect = _info
# import the module for execution
sys.path.append(test_input.test_path)
test_module = import_module('composable')
try:
test_result = test_module._run()
assert test_result is not None, f'expect a result {test_input_name}'
assert test_result == 0, f'wrong test result {test_input_name}'
assert (caom_mock.return_value.data_client.put.called
), f'{test_input_name} put not called'
assert (caom_mock.return_value.data_client.put.call_count == 3
), 'wrong put call count'
put_calls = [
call(
'/usr/src/app/integration_test/mock_test/data/execution/'
'GS-CAL20191214-1-029',
'gemini:GEMINI/S20191214S0301.jpg',
),
call(
'/usr/src/app/integration_test/mock_test/data/execution/'
'GS-CAL20191214-1-029',
'cadc:GEMINI/S20191214S0301_th.jpg',
),
call(
'/usr/src/app/integration_test/mock_test/data/execution/'
'GS-CAL20191214-1-029',
'gemini:GEMINI/S20191214S0301.fits',
),
]
caom_mock.return_value.data_client.put.assert_has_calls(
put_calls), f'{test_input_name} wrong put args'
assert http_get_mock.called, 'expect http get call'
assert http_get_mock.call_count == 2, 'wrong http get call count'
http_get_calls = [
call(
'https://archive.gemini.edu/preview/S20191214S0301.fits',
'/usr/src/app/integration_test/mock_test/data/execution/'
'GS-CAL20191214-1-029/S20191214S0301.jpg',
),
call(
'https://archive.gemini.edu/file/S20191214S0301.fits',
'/usr/src/app/integration_test/mock_test/data/execution/'
'GS-CAL20191214-1-029/S20191214S0301.fits',
),
]
http_get_mock.assert_has_calls(http_get_calls), 'wrong http get args'
except Exception as e:
logging.error(traceback.format_exc())
raise e
finally:
os.getcwd = getcwd_orig
del sys.modules['composable']
def test_todo_local_move(
caom_mock,
access_mock,
fits2caom2_mock,
test_input_name,
):
if 'MOVE' not in test_input_name:
return
test_input = INPUTS.get(test_input_name)
_cleanup()
new_dir = TEST_DATA_DIR / 'new'
fail_dir = TEST_DATA_DIR / 'failure'
success_dir = TEST_DATA_DIR / 'success'
for entry in [new_dir, fail_dir, success_dir]:
for listing in entry.iterdir():
listing.unlink()
shutil.copy(test_input.test_file, new_dir / '2460606o.fits.gz')
access_mock.return_value = 'https://localhost'
# make sure the working directory TEST_EXEC_DIR has the correct things
# in it
config_file_target = TEST_EXEC_DIR / 'config.yml'
shutil.copy(test_input.config_file, config_file_target)
cache_file_target = TEST_EXEC_DIR / 'cache.yml'
shutil.copy(test_input.cache_file, cache_file_target)
with open(TEST_EXEC_DIR / 'cadcproxy.pem', 'w') as f:
f.write('test content')
getcwd_orig = os.getcwd
os.getcwd = Mock(return_value=TEST_EXEC_DIR)
# import the module for execution
sys.path.append(test_input.test_path)
test_module = import_module('composable')
caom_mock.return_value.data_client.info.side_effect = [
None,
FileInfo(
id='ad:CFHT/2460606o.fits.gz',
md5sum='3d29f0edd984065a044d1376a11c6f08',
),
]
caom_mock.return_value.metadata_client.read.side_effect = [
None,
SimpleObservation(
'obs_id',
'CFHT',
Algorithm(name='exposure'),
instrument=Instrument(name='ESPaDOnS'),
),
]
def _info(uri):
assert uri == 'ad:CFHT/2460606o.fits.gz', 'wrong info uri'
return FileInfo(
id=uri,
md5sum='abc',
size=42,
)
fits2caom2_mock.return_value.info.side_effect = _info
try:
test_result = test_module._run_by_builder()
assert test_result is not None, f'expect a result {test_input_name}'
assert test_result == 0, f'wrong test result {test_input_name}'
assert (caom_mock.return_value.data_client.put.called
), f'{test_input_name} put not called'
caom_mock.return_value.data_client.put.assert_called_with(
'/usr/src/app/integration_test/mock_test/data/test_files/new',
'ad:CFHT/2460606o.fits.gz',
'raw',
), f'{test_input_name} wrong put args'
count = 0
for entry in new_dir.iterdir():
count += 1
assert count == 0, 'wrong new dir content'
for entry in success_dir.iterdir():
count += 1
assert count == 1, 'wrong success dir content'
count = 0
for entry in fail_dir.iterdir():
logging.error(f'fail entry {entry}')
count += 1
assert count == 0, f'wrong fail dir content {count}'
finally:
os.getcwd = getcwd_orig
del sys.modules['composable']
def test_todo_vos(
vo_mock,
caom_mock,
access_mock,
transfer_mock,
fits2caom2_mock,
test_input_name,
):
if 'VOS' not in test_input_name:
return
test_input = INPUTS.get(test_input_name)
_cleanup()
access_mock.return_value = 'https://localhost'
# make sure the working directory TEST_EXEC_DIR has the correct things
# in it
config_file_target = TEST_EXEC_DIR / 'config.yml'
shutil.copy(test_input.config_file, config_file_target)
with open(TEST_EXEC_DIR / 'cadcproxy.pem', 'w') as f:
f.write('test content')
# import the module for execution
sys.path.append(test_input.test_path)
test_module = import_module('composable')
reload(test_module)
logging.error(dir(test_module))
def _vo_listdir(entry):
assert entry == 'vos:goliaths/test', 'wrong parameter'
return ['a2021_08_17_19_30_01.fits.gz']
vo_mock.return_value.listdir.side_effect = _vo_listdir
vo_mock.return_value.isdir.return_value = False
def _vo_get_node(uri, limit=None, force=False):
assert (uri == 'vos:goliaths/test/a2021_08_17_19_30_01.fits.gz'
), f'wrong get node uri {uri}'
test_start_time = (datetime.now(tz=dateutil.tz.UTC) -
timedelta(minutes=5))
node = type('', (), {})()
node.props = {
'length': 42,
'MD5': '1234',
'lastmod': test_start_time.isoformat(),
}
return node
vo_mock.return_value.get_node.side_effect = _vo_get_node
caom_mock.return_value.data_client.info.side_effect = [
None,
FileInfo(id='ad:DAO/a2021_08_17_19_30_01.fits.gz'),
]
caom_mock.return_value.metadata_client.read.side_effect = [
None,
SimpleObservation(
'obs_id',
'DAO',
Algorithm(name='exposure'),
),
]
def _transfer_get(src, dst):
logging.error(src)
logging.error(dst)
assert (src == 'vos:goliaths/test/a2021_08_17_19_30_01.fits.gz'
), 'wrong source'
assert (
dst == '/usr/src/app/integration_test/mock_test/data/execution/'
'a2021_08_17_19_30_01/a2021_08_17_19_30_01.fits.gz'), 'wrong dst'
with open(dst, 'w') as f2:
f2.write('test content')
transfer_mock.return_value.get.side_effect = _transfer_get
def _get_head(ignore):
x = """SIMPLE = T / Written by IDL: Fri Oct 6 01:48:35 2017
BITPIX = -32 / Bits per pixel
NAXIS = 2 / Number of dimensions
NAXIS1 = 2048 /
NAXIS2 = 2048 /
EXPTIME = 1.23
NCOMBINE= 1
DATATYPE= 'REDUC ' /Data type, SCIENCE/CALIB/REJECT/FOCUS/TEST
END
"""
delim = '\nEND'
extensions = \
[e + delim for e in x.split(delim) if e.strip()]
headers = [fits.Header.fromstring(e, sep='\n') for e in extensions]
return headers
fits2caom2_mock.return_value.get_head.side_effect = _get_head
def _info(uri):
return FileInfo(
id=uri,
md5sum='abc',
size=42,
)
fits2caom2_mock.return_value.info.side_effect = _info
getcwd_orig = os.getcwd
os.getcwd = Mock(return_value=TEST_EXEC_DIR)
try:
test_result = test_module._run_vo()
assert test_result is not None, f'expect a result {test_input_name}'
assert test_result == 0, f'wrong test result {test_input_name}'
assert (caom_mock.return_value.data_client.put.called
), f'{test_input_name} put not called'
caom_mock.return_value.data_client.put.assert_called_with(
'/usr/src/app/integration_test/mock_test/data/execution/'
'a2021_08_17_19_30_01',
'ad:DAO/a2021_08_17_19_30_01.fits.gz',
'raw',
), f'{test_input_name} wrong put args'
finally:
os.getcwd = getcwd_orig
del sys.modules['composable']
def test_neoss_state(
data_mock,
csa_mock,
caom_mock,
transfer_mock,
local_header_mock,
test_input_name,
):
if 'NEOSS' not in test_input_name:
return
test_input = INPUTS.get(test_input_name)
# make sure the working directory TEXT_EXEC_DIR has nothing in it
_cleanup()
# make sure the working directory TEST_EXEC_DIR has the correct things
# in it
config_file_target = TEST_EXEC_DIR / 'config.yml'
shutil.copy(test_input.config_file, config_file_target)
state_file_target = TEST_EXEC_DIR / 'state.yml'
shutil.copy(test_input.state_file, state_file_target)
with open(TEST_EXEC_DIR / 'cadcproxy.pem', 'w') as f:
f.write('test content')
# make the state file won't take decades to execute
test_start_time = datetime.now(tz=dateutil.tz.UTC) - timedelta(minutes=5)
state = mc.State(state_file_target.as_posix())
state.save_state(test_input.bookmark, test_start_time)
def _csa_mock(start_date, ign1, ign2, ign3, ign4, ign5):
return {
'/users/OpenData_DonneesOuvertes/pub/NEOSSAT/ASTRO/2019/256/'
'NEOS_SCI_2019213215700.fits':
[False, start_date + timedelta(minutes=5).total_seconds()],
}
csa_mock.side_effect = _csa_mock
def _transfer_get(src, dst):
assert (src ==
'/users/OpenData_DonneesOuvertes/pub/NEOSSAT/ASTRO/2019/256/'
'NEOS_SCI_2019213215700.fits'), 'wrong source'
assert (
dst == '/usr/src/app/integration_test/mock_test/data/execution/'
'2019213215700/NEOS_SCI_2019213215700.fits'), 'wrong dst'
with open(dst, 'w') as f2:
f2.write('test content')
transfer_mock.return_value.get.side_effect = _transfer_get
caom_mock.return_value.metadata_client.read.side_effect = [
None,
SimpleObservation(
'obs_id',
'NEOSSAT',
Algorithm(name='exposure'),
),
]
def _local_header(ignore):
x = """SIMPLE = T / Written by IDL: Fri Oct 6 01:48:35 2017
BITPIX = -32 / Bits per pixel
NAXIS = 2 / Number of dimensions
NAXIS1 = 14 /
NAXIS2 = 24 /
RA = '22:53:27.5'
DEC = '-30:04:37.6'
MODE = '14 - FINE_SETTLE'
OBJECT = '2020-P4-C'
EXPOSURE= 128.0311
DATATYPE= 'REDUC ' /Data type, SCIENCE/CALIB/REJECT/FOCUS/TEST
END
"""
delim = '\nEND'
extensions = \
[e + delim for e in x.split(delim) if e.strip()]
headers = [fits.Header.fromstring(e, sep='\n') for e in extensions]
return headers
local_header_mock.side_effect = _local_header
def _info(uri):
return FileInfo(
id=uri,
md5sum='abc',
size=42,
)
data_mock.return_value.info.side_effect = _info
# import the module for execution
sys.path.append(test_input.test_path)
test_module = import_module('composable')
getcwd_orig = os.getcwd
os.getcwd = Mock(return_value=TEST_EXEC_DIR)
try:
test_result = test_module._run_state()
assert test_result is not None, f'expect a result {test_input_name}'
assert test_result == 0, f'wrong test result {test_input_name}'
# was state updated?
post_state = mc.State(state_file_target.as_posix())
assert (post_state.get_bookmark(test_input.bookmark) >
test_start_time), f'state not updated {test_input_name}'
assert (caom_mock.return_value.data_client.put.called
), f'{test_input_name} put not called'
caom_mock.return_value.data_client.put.assert_called_with(
'/usr/src/app/integration_test/mock_test/data/execution/'
'2019213215700',
'cadc:NEOSSAT/NEOS_SCI_2019213215700.fits',
None,
), f'{test_input_name} wrong put args'
except Exception as e:
logging.error(traceback.format_exc())
raise e
finally:
os.getcwd = getcwd_orig
del sys.modules['composable']
def test_state(
data_mock,
web_log_mock,
nrao_mock,
caom_mock,
transfer_mock,
local_header_mock,
qa_mock,
test_input_name,
):
if 'TODO' in test_input_name:
return
test_input = INPUTS.get(test_input_name)
# make sure the working directory TEXT_EXEC_DIR has nothing in it
for child in TEST_EXEC_DIR.iterdir():
if child == TEST_EXEC_DIR:
continue
if child.is_dir():
for child_2 in child.iterdir():
child_2.unlink()
child.rmdir()
else:
child.unlink()
# make sure the working directory TEST_EXEC_DIR has the correct things
# in it
config_file_target = TEST_EXEC_DIR / 'config.yml'
shutil.copy(test_input.config_file, config_file_target)
state_file_target = TEST_EXEC_DIR / 'state.yml'
shutil.copy(test_input.state_file, state_file_target)
with open(TEST_EXEC_DIR / 'cadcproxy.pem', 'w') as f:
f.write('test content')
# make the state file won't take decades to execute
test_start_time = datetime.now(tz=dateutil.tz.UTC) - timedelta(minutes=5)
state = mc.State(state_file_target.as_posix())
state.save_state(test_input.bookmark, test_start_time)
# import the module for execution
sys.path.append(test_input.test_path)
test_module = import_module('composable')
nrao_mock.side_effect = _nrao_mock
def _web_log_init(ignore):
global web_log_content
web_log_content = {
'VLASS1.1_T07t13.J083838-153000_P68878v1_2020_08_29T21_'
'48_48.092':
'2020-09-09 07:53',
}
web_log_mock.side_effect = _web_log_init
def _transfer_get(src, dst):
assert (src ==
'https://archive-new.nrao.edu/vlass/quicklook/VLASS1.1/T07t13/'
'VLASS1.1.ql.T07t13.J083838-153000.10.2048.v1.I.iter1.image.'
'pbcor.tt0.rms.subim.fits'), 'wrong source'
assert (
dst == '/usr/src/app/integration_test/mock_test/data/execution/'
'VLASS1.1.T07t13.J083838-153000/'
'VLASS1.1.ql.T07t13.J083838-153000.10.2048.v1.I.iter1.image.'
'pbcor.tt0.rms.subim.fits'), 'wrong dst'
with open(dst, 'w') as f2:
f2.write('test content')
transfer_mock.return_value.get.side_effect = _transfer_get
caom_mock.return_value.metadata_client.read.side_effect = [
None,
SimpleObservation(
'obs_id',
'VLASS',
Algorithm(name='exposure'),
),
]
def _local_header(ignore):
x = """SIMPLE = T / Written by IDL: Fri Oct 6 01:48:35 2017
BITPIX = -32 / Bits per pixel
NAXIS = 2 / Number of dimensions
NAXIS1 = 2048 /
NAXIS2 = 2048 /
TYPE = 'image'
BMAJ = 1.09
BMIN = 0.19
DATATYPE= 'REDUC ' /Data type, SCIENCE/CALIB/REJECT/FOCUS/TEST
END
"""
delim = '\nEND'
extensions = \
[e + delim for e in x.split(delim) if e.strip()]
headers = [fits.Header.fromstring(e, sep='\n') for e in extensions]
return headers
local_header_mock.side_effect = _local_header
def _info(uri):
return FileInfo(
id=uri,
md5sum='abc',
size=42,
)
data_mock.return_value.info.side_effect = _info
qa_mock.return_value = False
getcwd_orig = os.getcwd
os.getcwd = Mock(return_value=TEST_EXEC_DIR)
logging.getLogger('StorageClientWrapper').setLevel(logging.DEBUG)
try:
test_result = test_module._run_state()
assert test_result is not None, f'expect a result {test_input_name}'
assert test_result == 0, f'wrong test result {test_input_name}'
# was state updated?
post_state = mc.State(state_file_target.as_posix())
assert (post_state.get_bookmark(test_input.bookmark) >
test_start_time), f'state not updated {test_input_name}'
assert (caom_mock.return_value.data_client.put.called
), f'{test_input_name} put not called'
caom_mock.return_value.data_client.put.assert_called_with(
'/usr/src/app/integration_test/mock_test/data/execution/'
'VLASS1.1.T07t13.J083838-153000',
'nrao:VLASS/VLASS1.1.ql.T07t13.J083838-153000.10.2048.v1.I.'
'iter1.image.pbcor.tt0.rms.subim.fits',
None,
), f'{test_input_name} wrong put args'
finally:
os.getcwd = getcwd_orig
del sys.modules['composable']
def test_omm_retry(
caom_mock,
access_mock,
fits2caom2_mock,
test_input_name,
):
if 'OMM_RETRY' != test_input_name:
return
access_mock.return_value = 'https://localhost'
test_input = INPUTS.get(test_input_name)
_cleanup()
getcwd_orig = os.getcwd
os.getcwd = Mock(return_value=TEST_EXEC_DIR)
_setup(test_input)
shutil.copy(f'{TEST_DIR}/vlass/footprintfinder.py', TEST_EXEC_DIR)
retry_fqn = Path(f'{TEST_EXEC_DIR.as_posix()}_0')
todo_fqn = TEST_EXEC_DIR / 'todo.txt'
with open(todo_fqn, 'w') as f1:
f1.write(f'{test_input.test_file.name}\n')
caom_mock.return_value.data_client.info.side_effect = [
None,
FileInfo(
id=test_input.test_uri,
md5sum='3d29f0edd984065a044d1376a11c6f08',
),
FileInfo(
id=test_input.test_uri,
md5sum='3d29f0edd984065a044d1376a11c6f08',
),
FileInfo(
id=test_input.test_uri,
md5sum='3d29f0edd984065a044d1376a11c6f08',
),
]
test_obs = mc.read_obs_from_file(test_input.obs_xml.as_posix())
caom_mock.return_value.metadata_client.read.side_effect = [
None,
SimpleObservation(
'obs_id',
test_input.collection,
Algorithm(name='exposure'),
),
test_obs,
test_obs,
]
caom_mock.return_value.metadata_client.update.side_effect = [
mc.CadcException('pretend there is a timeout'),
None,
None,
None,
]
test_exec_fqn = Path(TEST_EXEC_DIR / 'C170324_0054')
def _get_mock(ignore, uri):
assert uri == test_input.test_uri, 'wrong uri'
assert ignore == test_exec_fqn.as_posix(), 'wrong working directory'
shutil.copy(test_input.test_file, test_exec_fqn)
caom_mock.return_value.data_client.get.side_effect = _get_mock
def _info(uri):
return FileInfo(
id=uri,
md5sum='abc',
size=42,
)
fits2caom2_mock.return_value.info.side_effect = _info
def _get_head(uri):
assert uri == test_input.test_uri, 'wrong get_head parameter'
return caom2utils.data_util.get_local_file_headers(
test_input.test_file.as_posix())
fits2caom2_mock.return_value.get_head.side_effect = _get_head
# import the module for execution
sys.path.append(test_input.test_path)
test_module = import_module('composable')
try:
test_result = test_module._run()
assert test_result is not None, f'expect a result {test_input_name}'
# expect a non-zero return, because evidence of the first failure
# is preserved even through the retry
assert test_result == -1, f'wrong test result {test_input_name}'
assert (caom_mock.return_value.data_client.get.call_count == 2
), 'wrong get call count'
assert retry_fqn.exists(), 'expect directory to exist'
assert retry_fqn.is_dir(), 'expect it to be a directory'
assert (caom_mock.return_value.data_client.put.call_count == 2
), 'wrong put call count'
put_calls = [
call(
test_exec_fqn.as_posix(),
'cadc:OMM/C170324_0054_SCI_prev_256.jpg',
None,
),
call(
test_exec_fqn.as_posix(),
'cadc:OMM/C170324_0054_SCI_prev.jpg',
None,
),
]
caom_mock.return_value.data_client.put.assert_has_calls(
put_calls), 'wrong put calls'
except Exception as e:
logging.error(traceback.format_exc())
raise e
finally:
os.getcwd = getcwd_orig
if retry_fqn.exists():
logging.error(f'Cleaning up {retry_fqn.as_posix()}')
for child in retry_fqn.iterdir():
child.unlink()
retry_fqn.rmdir()
def _mock_read(ignore_fqn):
return SimpleObservation(
collection='test_collection',
observation_id='ghi',
algorithm=Algorithm(str('test')),
)
def _read_mock(ignore_fqn):
return SimpleObservation(
collection='TEST',
observation_id='TEST_OBS_ID',
algorithm=Algorithm('exposure'),
)
def test_gem_state(
data_mock,
caom_mock,
local_header_mock,
json_mock,
filter_mock,
http_get_mock,
endpoint_mock,
tap_mock,
external_header_mock,
test_input_name,
):
if 'GEM_STATE' not in test_input_name:
return
test_input = INPUTS.get(test_input_name)
_cleanup()
getcwd_orig = os.getcwd
os.getcwd = Mock(return_value=TEST_EXEC_DIR)
test_start_time, state_file_target = _setup(test_input)
def _json_mock(url, ignore_session):
response = Mock()
response.close = Mock()
fqn = test_input.input_dir / 'input.json'
with open(fqn) as f:
response.text = f.read()
def x():
return json.loads(response.text)
response.json = x
return response
json_mock.side_effect = _json_mock
def _endpoint_mock(ignore):
assert (ignore.startswith(
'https://archive.gemini.edu/jsonsummary/canonical/NotFail/'
'notengineering/entrytimedaterange')
), 'wrong url for incremental querying'
return _json_mock(ignore, None)
endpoint_mock.side_effect = _endpoint_mock
def _filter_mock():
from astropy.table import parse_single_table
fqn = test_input.input_dir / 'filter.xml'
content = parse_single_table(fqn)
return content, None
filter_mock.side_effect = _filter_mock
caom_mock.return_value.metadata_client.read.side_effect = [
None,
SimpleObservation(
'obs_id',
'GEMINI',
Algorithm(name='exposure'),
),
]
def _tap_query(
ignore_query,
output_file,
data_only=True,
response_format='csv',
):
output_file.write(
'observationID,instrument_name\n'
'GS-CAL20191214-1-029,F2\n', )
# caom_mock.return_value.query_client.query.side_effect = _tap_query
tap_mock.return_value.query.side_effect = _tap_query
def _local_header(ignore):
x = """SIMPLE = T / Written by IDL: Fri Oct 6 01:48:35 2017
BITPIX = -32 / Bits per pixel
NAXIS = 2 / Number of dimensions
NAXIS1 = 14 /
NAXIS2 = 24 /
INSTRUME= 'F2'
DATALAB = 'GS-CAL20191214-1-029
END
"""
delim = '\nEND'
extensions = \
[e + delim for e in x.split(delim) if e.strip()]
headers = [fits.Header.fromstring(e, sep='\n') for e in extensions]
return headers
local_header_mock.side_effect = _local_header
external_header_mock.side_effect = _local_header
def _info(uri):
return FileInfo(
id=uri,
md5sum='abc',
size=42,
)
data_mock.return_value.info.side_effect = _info
# import the module for execution
sys.path.append(test_input.test_path)
test_module = import_module('composable')
try:
test_result = test_module._run_state()
assert test_result is not None, f'expect a result {test_input_name}'
assert test_result == 0, f'wrong test result {test_input_name}'
# was state updated?
post_state = mc.State(state_file_target.as_posix())
assert (post_state.get_bookmark(test_input.bookmark) >
test_start_time), f'state not updated {test_input_name}'
assert (caom_mock.return_value.data_client.put.called
), f'{test_input_name} put not called'
caom_mock.return_value.data_client.put.assert_called_with(
'/usr/src/app/integration_test/mock_test/data/execution/'
'GS-CAL20191214-1-029',
'gemini:GEMINI/S20191214S0301.fits',
), f'{test_input_name} wrong put args'
assert http_get_mock.called, 'expect http get call'
http_get_mock.assert_called_with(
'https://archive.gemini.edu/file/S20191214S0301.fits',
'/usr/src/app/integration_test/mock_test/data/execution/'
'GS-CAL20191214-1-029/S20191214S0301.fits',
), 'wrong http get args'
except Exception as e:
logging.error(traceback.format_exc())
raise e
finally:
os.getcwd = getcwd_orig
del sys.modules['composable']
def test_gem_todo_local(
caom_mock,
access_mock,
fits2caom2_mock,
tap_mock,
test_input_name,
):
if 'GEM_TODO_LOCAL' not in test_input_name:
return
test_input = INPUTS.get(test_input_name)
_cleanup()
shutil.copy(test_input.test_file, TEST_DATA_DIR / 'S20191214S0301.fits')
access_mock.return_value = 'https://localhost'
_setup(test_input)
getcwd_orig = os.getcwd
os.getcwd = Mock(return_value=TEST_EXEC_DIR)
# import the module for execution
sys.path.append(test_input.test_path)
test_module = import_module('composable')
caom_mock.return_value.data_client.info.side_effect = [
None,
FileInfo(
id='gemini:GEMINI/S20191214S0301.fits',
md5sum='3d29f0edd984065a044d1376a11c6f08',
),
]
caom_mock.return_value.metadata_client.read.side_effect = [
None,
SimpleObservation(
'obs_id',
'GEMINI',
Algorithm(name='exposure'),
),
]
def _info(uri):
assert (uri == 'gemini:GEMINI/S20191214S0301.fits'), 'wrong info uri'
return FileInfo(
id=uri,
md5sum='abc',
size=42,
)
fits2caom2_mock.return_value.info.side_effect = _info
def _tap_query(
ignore_query,
output_file,
data_only=True,
response_format='csv',
):
output_file.write(
'observationID,instrument_name\n'
'GS-CAL20191214-1-029,F2\n', )
# caom_mock.return_value.query_client.query.side_effect = _tap_query
tap_mock.return_value.query.side_effect = _tap_query
try:
test_result = test_module._run()
assert test_result is not None, f'expect a result {test_input_name}'
assert test_result == 0, f'wrong test result {test_input_name}'
# Gemini local should not be checking archive.gemini.edu for a
# newer version of the file
assert not caom_mock.return_value.data_client.put.called, 'no put'
finally:
os.getcwd = getcwd_orig
del sys.modules['composable']
