def test_mixed_case_file_names(): mixed_case_f_names_order_1 = os.path.join(gem_mocks.TEST_DATA_DIR, 'mixed_case_1.txt') mixed_case_f_names_order_2 = os.path.join(gem_mocks.TEST_DATA_DIR, 'mixed_case_2.txt') test_obs_id = 'GN-CAL20100415-6-086-BIAS' test_file_id = 'N20100415S0452_bias' for f_name in [mixed_case_f_names_order_1, mixed_case_f_names_order_2]: import shutil shutil.copy(f_name, '/app/data/from_paul.txt') test_subject = GemObsFileRelationship() result_obs_id = test_subject.get_obs_id(test_file_id) assert result_obs_id is not None, 'expected result {}'.format(f_name) assert result_obs_id == test_obs_id, 'wrong result {}'.format(f_name) test_timestamp = test_subject.get_timestamp(test_file_id) assert test_timestamp is not None, 'expected result {}'.format(f_name) assert test_timestamp == 1498316473.885391, 'wrong timestamp' result_file_names = test_subject.get_file_names(test_obs_id) assert result_file_names is not None, 'expected result {}'.format( f_name) assert len(result_file_names) == 1, 'wrong size result {}'.format( f_name) assert result_file_names[0] == '{}.fits'.format(test_file_id), \ 'wrong result {} {}'.format(f_name, result_file_names)
def test_subset_all(): gofr = GemObsFileRelationship(TEST_FILE) temp = gofr.subset() assert temp is not None, 'should have content' assert temp[0].startswith( 'GEMINI GN-CAL20170616-11-022 2017-06-19T03:21:29.345'), \ 'wrong content' assert len(list(temp)) == 513, 'wrong count' result = gofr.get_file_names('GN-2015B-Q-1-12-1003') assert result == \ ['N20150807G0044m.fits', 'N20150807G0044i.fits', 'N20150807G0044.fits'], \ 'entry missing {}'.format(result)
def test_repair_provenance(): if em.gofr is None: em.gofr = GemObsFileRelationship(TEST_FILE) for ii in test_subjects: ignore, test_fid = _repair_provenance_value(ii[1], 'test obs') assert test_fid is not None, 'failed lookup {}'.format(ii) assert test_fid == ii[0], 'error {}'.format(ii[1])
def test_subset_only_end(): end = datetime.strptime('2018-12-16T18:12:26.16614', ISO_DATE) gofr = GemObsFileRelationship(TEST_FILE) temp = gofr.subset(end=end) assert temp is not None, 'should have content' assert temp[0].startswith( 'GEMINI GN-CAL20170616-11-022 2017-06-19T03:21:29.345'), \ 'wrong content' assert len(list(temp)) == 416, 'wrong count' temp = gofr.subset(end=end, maxrec=3) assert temp is not None, 'should have content' assert temp[0].startswith( 'GEMINI GN-CAL20170616-11-022 2017-06-19T03:21:29.345'), \ 'wrong content' assert len(list(temp)) == 3, 'wrong maxrec count'
def test_subset_only_start(): start = datetime.strptime('2018-12-16T03:47:03.939488', ISO_DATE) gofr = GemObsFileRelationship(TEST_FILE) temp = gofr.subset(start=start) assert temp is not None, 'should have content' assert temp[0].startswith( 'GEMINI GN-2018B-FT-113-24-015 2018-12-17T18:08:29.362'), \ 'wrong content' assert len(list(temp)) == 97, 'wrong count' temp = gofr.subset(start=start, maxrec=3) assert temp is not None, 'should have content' assert temp[0].startswith( 'GEMINI GN-2018B-FT-113-24-015 2018-12-17T18:08:29.362'), \ 'wrong content' assert len(list(temp)) == 3, 'wrong maxrec count'
def list_observations(start=None, end=None, maxrec=None): """ List observations :param start: start date (UTC) :param end: end date (UTC) :param maxrec: maximum number of rows to return :return: Comma separated list, each row consisting of ObservationID, last mod date. NOTE: For stream the results use a generator, e.g: for i in range(3): yield "{}\n".format(datetime.datetime.now().isoformat()) time.sleep(1) The results of this query come from a file supplied by Gemini approximately once every six months. That file is located on this container. It's read in and re-formatted into the information required to support this endpoint query. """ if em.gofr is None: em.gofr = GemObsFileRelationship('/app/data/from_paul.txt') temp = em.gofr.subset(start, end, maxrec) for ii in temp: yield '{}\n'.format(ii)
def test_subset_start_end(): start = datetime.strptime('2017-06-20T12:36:35.681662', ISO_DATE) end = datetime.strptime('2017-12-17T20:13:56.572387', ISO_DATE) test_subject = GemObsFileRelationship(TEST_FILE) temp = test_subject.subset(start=start, end=end) assert temp is not None, 'should have content' assert temp[0].startswith( 'GEMINI GN-CAL20150925-2-007 2017-06-20T14:50:59.795'), \ 'wrong content' assert len(list(temp)) == 314, 'wrong count' temp = test_subject.subset(start=start, end=end, maxrec=3) assert temp is not None, 'should have content' assert temp[0].startswith( 'GEMINI GN-CAL20150925-2-007 2017-06-20T14:50:59.795'), \ 'wrong content' assert len(list(temp)) == 3, 'wrong maxrec count'
def test_make_gem2caom2_args(): gofr = GemObsFileRelationship('/app/data/from_paul.txt') for ii in x: test_result = gofr.get_args(ii) assert test_result is not None, 'no result' assert len(test_result) == len(x[ii]), \ 'wrong length for {}'.format(ii) for jj in test_result: found = False for kk in x[ii]: if jj.obs_id == kk.obs_id: found = True assert jj.lineage == kk.lineage, \ '{} lineage {} expected {}'.format( jj.obs_id, jj.lineage, kk.lineage) assert jj.urls == kk.urls, \ '{} urls {} expected {}'.format(jj.obs_id, jj.urls, kk.urls) break assert found, 'new obs id {}'.format(jj.obs_id)
def test_repair_data_label(): if em.gofr is None: em.gofr = GemObsFileRelationship(TEST_FILE) for ii in test_main_app.LOOKUP: test_result = em.gofr.repair_data_label(ii) if ii == 'S20181230S0026': assert test_result == 'S20181230S0026', \ 'repair failed for {} actual {} expected {}'.format( ii, test_result, test_main_app.LOOKUP[ii][0]) else: assert test_result == test_main_app.LOOKUP[ii][0], \ 'repair failed for {} actual {} expected {}'.format( ii, test_result, test_main_app.LOOKUP[ii][0]) test_result = em.gofr.repair_data_label('N20181217S0266') assert test_result is not None, 'no result' assert test_result == 'GN-2018B-Q-133-20-001', 'wrong result'
def get_observation(obs_id): """ Return the observation corresponding to the obs_id :param obs_id: id of the observation :return: observation corresponding to the id or None if such such observation does not exist """ if em.gofr is None: em.gofr = GemObsFileRelationship('/app/data/from_paul.txt') obs = _invoke_gem2caom2(obs_id) if obs is None: logger.error('Could not create observation for {}'.format(obs_id)) return None return obs
def test_is_processed(): tests = { 'c2016may18_sci128': False, 'abu01aug16_001': False, 'ag2003feb19_6.0001': True, 'TX06A_flt.2511': True, 'GS20141226S0203_BIAS': True, 'N20070819S0339_dark': True, 'N20110927S0170_fringe': True, 'N20120320S0328_stack_fringe': True, 'N20130404S0512_flat': True, 'N20140313S0072_flat': True, 'N20141109S0266_bias': True, 'N20150804S0348_dark': True, 'N20160403S0236_flat_pasted': True, 'S20120922S0406': False, 'S20131007S0067_fringe': True, 'S20140124S0039_dark': True, 'S20141129S0331_dark': True, 'S20161227S0051': False, 'fmrgN20020413S0120_add': True, 'gS20181219S0216_flat': True, 'gS20190301S0556_bias': True, 'mfrgS20041117S0073_add': True, 'mfrgS20160310S0154_add': True, 'mrgN20041016S0095': True, 'mrgN20050831S0770_add': True, 'mrgN20160311S0691_add': True, 'mrgS20120922S0406': True, 'mrgS20160901S0122_add': True, 'mrgS20181016S0184_fringe': True, 'rS20121030S0136': True, 'rgS20100212S0301': True, 'rgS20100316S0366': True, 'rgS20130103S0098_FRINGE': True, 'rgS20131109S0166_FRINGE': True, 'rgS20161227S0051_fringe': True, 'p2004may20_0048_FLAT': True, 'p2004may19_0255_COMB': True, 'P2003JAN14_0148_DARK': True, 'P2002FEB03_0045_DARK10SEC': True, 'P2002DEC02_0161_SUB': True, 'P2002DEC02_0075_SUB.0001': True} for ii in tests: assert GemObsFileRelationship.is_processed(ii) == tests[ii], \ 'failed {}'.format(ii)
def test_get_timestamp(): gofr = GemObsFileRelationship('/app/data/from_paul.txt') test_result = gofr.get_timestamp('ag2003feb19_6.0001') assert test_result is not None, 'no result' assert test_result == 1498571069.924588, 'wrong result'