def test_incremental_source_reproduce(query_mock):
# https://archive.gemini.edu/jsonsummary/canonical/NotFail/notengineering/
# entrytimedaterange=
# 2022-03-14T17:30:05.000006%202022-03-14T17:31:05.000006/
# ?orderby=entrytime
# get results
query_mock.side_effect = gem_mocks.mock_query_endpoint_reproduce
test_subject = data_source.IncrementalSource(Mock())
assert test_subject is not None, 'expect construction success'
prev_exec_time = datetime(year=2022,
month=1,
day=1,
hour=20,
minute=3,
second=0).timestamp()
exec_time = datetime(year=2022,
month=4,
day=1,
hour=22,
minute=13,
second=0).timestamp()
test_result = test_subject.get_time_box_work(prev_exec_time, exec_time)
assert test_result is not None, 'expect a result'
assert len(test_result) == 2, 'wrong number of results'
def test_incremental_source(query_mock):
# https://archive.gemini.edu/jsonsummary/canonical/entrytimedaterange=
# 2021-01-01T20:03:00.000000%202021-01-01T22:13:00.000000/
# ?orderby=entrytime
# get results
query_mock.side_effect = gem_mocks.mock_query_endpoint_2
test_subject = data_source.IncrementalSource()
assert test_subject is not None, 'expect construction success'
prev_exec_time = datetime(year=2021, month=1, day=1,
hour=20, minute=3, second=0).timestamp()
exec_time = datetime(year=2021, month=1, day=1,
hour=22, minute=13, second=0).timestamp()
test_result = test_subject.get_time_box_work(prev_exec_time, exec_time)
assert test_result is not None, 'expect a result'
assert len(test_result) == 2, 'wrong number of results'
test_entry = test_result.popleft()
assert test_entry.entry_name == 'N20210101S0043.fits', 'wrong first file'
assert test_entry.entry_ts == 1609535565.237183, 'wrong first timestamp'
test_entry = test_result.popleft()
assert test_entry.entry_name == 'N20210101S0042.fits', 'wrong 2nd file'
assert test_entry.entry_ts == 1609535567.250666, 'wrong 2nd timestamp'
# get nothing
prev_exec_time = datetime(year=2019, month=1, day=1,
hour=20, minute=3, second=0).timestamp()
exec_time = datetime(year=2019, month=2, day=1,
hour=22, minute=13, second=0).timestamp()
test_result = test_subject.get_time_box_work(prev_exec_time, exec_time)
assert test_result is not None, 'expect a result'
assert len(test_result) == 0, 'wrong number of empty result list'
def _run_state():
"""Run incremental processing for observations that are posted on the site
archive.gemini.edu. TODO in the future this will depend on the incremental
query endpoint.
:return 0 if successful, -1 if there's any sort of failure. Return status
is used by airflow for task instance management and reporting.
"""
(
clients,
config,
metadata_reader,
meta_visitors,
name_builder,
) = _common_init()
state = mc.State(config.state_fqn)
end_timestamp_s = state.bookmarks.get(data_source.GEM_BOOKMARK).get(
'end_timestamp', datetime.now())
end_timestamp_dt = mc.make_time_tz(end_timestamp_s)
logging.info(f'{main_app.APPLICATION} will end at {end_timestamp_s}')
incremental_source = data_source.IncrementalSource(metadata_reader)
result = rc.run_by_state(
config=config,
name_builder=name_builder,
bookmark_name=data_source.GEM_BOOKMARK,
meta_visitors=meta_visitors,
data_visitors=DATA_VISITORS,
end_time=end_timestamp_dt,
source=incremental_source,
clients=clients,
metadata_reader=metadata_reader,
)
if incremental_source.max_records_encountered:
logging.warning('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
logging.warning('Encountered maximum records!!')
logging.warning('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
result |= -1
return result
def _run_by_incremental():
"""Run incremental processing for observations that are posted on the site
archive.gemini.edu. TODO in the future this will depend on the incremental
query endpoint.
:return 0 if successful, -1 if there's any sort of failure. Return status
is used by airflow for task instance management and reporting.
"""
config = mc.Config()
config.get_executors()
state = mc.State(config.state_fqn)
end_timestamp_s = state.bookmarks.get(data_source.GEM_BOOKMARK).get(
'end_timestamp', datetime.now())
end_timestamp_dt = mc.make_time_tz(end_timestamp_s)
logging.info(f'{main_app.APPLICATION} will end at {end_timestamp_s}')
external_metadata.init_global(config=config)
name_builder = nbc.FileNameBuilder(gem_name.GemName)
incremental_source = data_source.IncrementalSource()
meta_visitors = _define_meta_visitors(config)
result = rc.run_by_state(
config=config,
name_builder=name_builder,
command_name=main_app.APPLICATION,
bookmark_name=data_source.GEM_BOOKMARK,
meta_visitors=meta_visitors,
data_visitors=DATA_VISITORS,
end_time=end_timestamp_dt,
source=incremental_source,
chooser=None,
)
if incremental_source.max_records_encountered:
logging.warning('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
logging.warning('Encountered maximum records!!')
logging.warning('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
result |= -1
return result