def test_mixed_case_file_names():
mixed_case_f_names_order_1 = os.path.join(gem_mocks.TEST_DATA_DIR,
'mixed_case_1.txt')
mixed_case_f_names_order_2 = os.path.join(gem_mocks.TEST_DATA_DIR,
'mixed_case_2.txt')
test_obs_id = 'GN-CAL20100415-6-086-BIAS'
test_file_id = 'N20100415S0452_bias'
for f_name in [mixed_case_f_names_order_1, mixed_case_f_names_order_2]:
import shutil
shutil.copy(f_name, '/app/data/from_paul.txt')
test_subject = GemObsFileRelationship()
result_obs_id = test_subject.get_obs_id(test_file_id)
assert result_obs_id is not None, 'expected result {}'.format(f_name)
assert result_obs_id == test_obs_id, 'wrong result {}'.format(f_name)
test_timestamp = test_subject.get_timestamp(test_file_id)
assert test_timestamp is not None, 'expected result {}'.format(f_name)
assert test_timestamp == 1498316473.885391, 'wrong timestamp'
result_file_names = test_subject.get_file_names(test_obs_id)
assert result_file_names is not None, 'expected result {}'.format(
f_name)
assert len(result_file_names) == 1, 'wrong size result {}'.format(
f_name)
assert result_file_names[0] == '{}.fits'.format(test_file_id), \
'wrong result {} {}'.format(f_name, result_file_names)
def test_subset_all():
gofr = GemObsFileRelationship(TEST_FILE)
temp = gofr.subset()
assert temp is not None, 'should have content'
assert temp[0].startswith(
'GEMINI GN-CAL20170616-11-022 2017-06-19T03:21:29.345'), \
'wrong content'
assert len(list(temp)) == 513, 'wrong count'
result = gofr.get_file_names('GN-2015B-Q-1-12-1003')
assert result == \
['N20150807G0044m.fits', 'N20150807G0044i.fits',
'N20150807G0044.fits'], \
'entry missing {}'.format(result)
def test_repair_provenance():
if em.gofr is None:
em.gofr = GemObsFileRelationship(TEST_FILE)
for ii in test_subjects:
ignore, test_fid = _repair_provenance_value(ii[1], 'test obs')
assert test_fid is not None, 'failed lookup {}'.format(ii)
assert test_fid == ii[0], 'error {}'.format(ii[1])
def test_subset_only_end():
end = datetime.strptime('2018-12-16T18:12:26.16614', ISO_DATE)
gofr = GemObsFileRelationship(TEST_FILE)
temp = gofr.subset(end=end)
assert temp is not None, 'should have content'
assert temp[0].startswith(
'GEMINI GN-CAL20170616-11-022 2017-06-19T03:21:29.345'), \
'wrong content'
assert len(list(temp)) == 416, 'wrong count'
temp = gofr.subset(end=end, maxrec=3)
assert temp is not None, 'should have content'
assert temp[0].startswith(
'GEMINI GN-CAL20170616-11-022 2017-06-19T03:21:29.345'), \
'wrong content'
assert len(list(temp)) == 3, 'wrong maxrec count'
def test_subset_only_start():
start = datetime.strptime('2018-12-16T03:47:03.939488', ISO_DATE)
gofr = GemObsFileRelationship(TEST_FILE)
temp = gofr.subset(start=start)
assert temp is not None, 'should have content'
assert temp[0].startswith(
'GEMINI GN-2018B-FT-113-24-015 2018-12-17T18:08:29.362'), \
'wrong content'
assert len(list(temp)) == 97, 'wrong count'
temp = gofr.subset(start=start, maxrec=3)
assert temp is not None, 'should have content'
assert temp[0].startswith(
'GEMINI GN-2018B-FT-113-24-015 2018-12-17T18:08:29.362'), \
'wrong content'
assert len(list(temp)) == 3, 'wrong maxrec count'
def list_observations(start=None, end=None, maxrec=None):
"""
List observations
:param start: start date (UTC)
:param end: end date (UTC)
:param maxrec: maximum number of rows to return
:return: Comma separated list, each row consisting of ObservationID,
last mod date.
NOTE: For stream the results use a generator, e.g:
for i in range(3):
yield "{}\n".format(datetime.datetime.now().isoformat())
time.sleep(1)
The results of this query come from a file supplied by Gemini
approximately once every six months. That file is located on this
container. It's read in and re-formatted into the information required
to support this endpoint query.
"""
if em.gofr is None:
em.gofr = GemObsFileRelationship('/app/data/from_paul.txt')
temp = em.gofr.subset(start, end, maxrec)
for ii in temp:
yield '{}\n'.format(ii)
def test_subset_start_end():
start = datetime.strptime('2017-06-20T12:36:35.681662', ISO_DATE)
end = datetime.strptime('2017-12-17T20:13:56.572387', ISO_DATE)
test_subject = GemObsFileRelationship(TEST_FILE)
temp = test_subject.subset(start=start, end=end)
assert temp is not None, 'should have content'
assert temp[0].startswith(
'GEMINI GN-CAL20150925-2-007 2017-06-20T14:50:59.795'), \
'wrong content'
assert len(list(temp)) == 314, 'wrong count'
temp = test_subject.subset(start=start, end=end, maxrec=3)
assert temp is not None, 'should have content'
assert temp[0].startswith(
'GEMINI GN-CAL20150925-2-007 2017-06-20T14:50:59.795'), \
'wrong content'
assert len(list(temp)) == 3, 'wrong maxrec count'
def test_make_gem2caom2_args():
gofr = GemObsFileRelationship('/app/data/from_paul.txt')
for ii in x:
test_result = gofr.get_args(ii)
assert test_result is not None, 'no result'
assert len(test_result) == len(x[ii]), \
'wrong length for {}'.format(ii)
for jj in test_result:
found = False
for kk in x[ii]:
if jj.obs_id == kk.obs_id:
found = True
assert jj.lineage == kk.lineage, \
'{} lineage {} expected {}'.format(
jj.obs_id, jj.lineage, kk.lineage)
assert jj.urls == kk.urls, \
'{} urls {} expected {}'.format(jj.obs_id, jj.urls, kk.urls)
break
assert found, 'new obs id {}'.format(jj.obs_id)
def test_repair_data_label():
if em.gofr is None:
em.gofr = GemObsFileRelationship(TEST_FILE)
for ii in test_main_app.LOOKUP:
test_result = em.gofr.repair_data_label(ii)
if ii == 'S20181230S0026':
assert test_result == 'S20181230S0026', \
'repair failed for {} actual {} expected {}'.format(
ii, test_result, test_main_app.LOOKUP[ii][0])
else:
assert test_result == test_main_app.LOOKUP[ii][0], \
'repair failed for {} actual {} expected {}'.format(
ii, test_result, test_main_app.LOOKUP[ii][0])
test_result = em.gofr.repair_data_label('N20181217S0266')
assert test_result is not None, 'no result'
assert test_result == 'GN-2018B-Q-133-20-001', 'wrong result'
def get_observation(obs_id):
"""
Return the observation corresponding to the obs_id
:param obs_id: id of the observation
:return: observation corresponding to the id or None if such
such observation does not exist
"""
if em.gofr is None:
em.gofr = GemObsFileRelationship('/app/data/from_paul.txt')
obs = _invoke_gem2caom2(obs_id)
if obs is None:
logger.error('Could not create observation for {}'.format(obs_id))
return None
return obs
def test_is_processed():
tests = {
'c2016may18_sci128': False,
'abu01aug16_001': False,
'ag2003feb19_6.0001': True,
'TX06A_flt.2511': True,
'GS20141226S0203_BIAS': True,
'N20070819S0339_dark': True,
'N20110927S0170_fringe': True,
'N20120320S0328_stack_fringe': True,
'N20130404S0512_flat': True,
'N20140313S0072_flat': True,
'N20141109S0266_bias': True,
'N20150804S0348_dark': True,
'N20160403S0236_flat_pasted': True,
'S20120922S0406': False,
'S20131007S0067_fringe': True,
'S20140124S0039_dark': True,
'S20141129S0331_dark': True,
'S20161227S0051': False,
'fmrgN20020413S0120_add': True,
'gS20181219S0216_flat': True,
'gS20190301S0556_bias': True,
'mfrgS20041117S0073_add': True,
'mfrgS20160310S0154_add': True,
'mrgN20041016S0095': True,
'mrgN20050831S0770_add': True,
'mrgN20160311S0691_add': True,
'mrgS20120922S0406': True,
'mrgS20160901S0122_add': True,
'mrgS20181016S0184_fringe': True,
'rS20121030S0136': True,
'rgS20100212S0301': True,
'rgS20100316S0366': True,
'rgS20130103S0098_FRINGE': True,
'rgS20131109S0166_FRINGE': True,
'rgS20161227S0051_fringe': True,
'p2004may20_0048_FLAT': True,
'p2004may19_0255_COMB': True,
'P2003JAN14_0148_DARK': True,
'P2002FEB03_0045_DARK10SEC': True,
'P2002DEC02_0161_SUB': True,
'P2002DEC02_0075_SUB.0001': True}
for ii in tests:
assert GemObsFileRelationship.is_processed(ii) == tests[ii], \
'failed {}'.format(ii)
def test_get_timestamp():
gofr = GemObsFileRelationship('/app/data/from_paul.txt')
test_result = gofr.get_timestamp('ag2003feb19_6.0001')
assert test_result is not None, 'no result'
assert test_result == 1498571069.924588, 'wrong result'