def _run_by_public():
"""Run the processing for observations that are public, but there are
no artifacts representing the previews in CAOM, or a FITS file in ad.
Called as gem_run_public. The time-boxing is based on timestamps from a
state.yml file. Call once/day, since data release timestamps have times
of 00:00:00.000.
:return 0 if successful, -1 if there's any sort of failure. Return status
is used by airflow for task instance management and reporting.
"""
config = mc.Config()
config.get_executors()
external_metadata.init_global(config=config)
name_builder = nbc.FileNameBuilder(gem_name.GemName)
incremental_source = data_source.PublicIncremental(config)
meta_visitors = _define_meta_visitors(config)
return rc.run_by_state(config=config,
name_builder=name_builder,
command_name=main_app.APPLICATION,
bookmark_name=data_source.GEM_BOOKMARK,
meta_visitors=meta_visitors,
data_visitors=DATA_VISITORS,
end_time=None,
source=incremental_source,
chooser=None)
def _run_by_state():
"""Uses a state file with a timestamp to control which quicklook
files will be retrieved from VLASS.
Ingestion is based on URLs, because a URL that contains the phrase
'QA_REJECTED' is the only way to tell if the attribute 'requirements'
should be set to 'fail', or not.
"""
config = mc.Config()
config.get_executors()
state = mc.State(config.state_fqn)
# a way to get a datetime from a string, or maybe a datetime, depending
# on the execution environment
start_time = mc.increment_time(state.get_bookmark(VLASS_BOOKMARK), 0)
todo_list, max_date = scrape.build_file_url_list(start_time)
if len(todo_list) > 0:
state = mc.State(config.state_fqn)
work.init_web_log(state, config)
# still make all subsequent calls if len == 0, for consistent reporting
source = data_source.NraoPage(todo_list)
name_builder = builder.VlassInstanceBuilder(config)
return rc.run_by_state(config=config,
command_name=sn.APPLICATION,
bookmark_name=VLASS_BOOKMARK,
meta_visitors=META_VISITORS,
data_visitors=DATA_VISITORS,
name_builder=name_builder,
source=source,
end_time=max_date,
store_transfer=tc.HttpTransfer())
def _run_by_public():
"""Run the processing for observations that are public, but there are
no artifacts representing the previews in CAOM, or a FITS file in ad.
Called as gem_run_public. The time-boxing is based on timestamps from a
state.yml file. Call once/day, since data release timestamps have times
of 00:00:00.000.
:return 0 if successful, -1 if there's any sort of failure. Return status
is used by airflow for task instance management and reporting.
"""
(
clients,
config,
metadata_reader,
meta_visitors,
name_builder,
) = _common_init()
incremental_source = data_source.PublicIncremental(config,
clients.query_client)
return rc.run_by_state(
config=config,
name_builder=name_builder,
bookmark_name=data_source.GEM_BOOKMARK,
meta_visitors=meta_visitors,
data_visitors=DATA_VISITORS,
end_time=None,
source=incremental_source,
clients=clients,
metadata_reader=metadata_reader,
)
def _run_state():
"""Uses a state file with a timestamp to control which files will be
retrieved from the CSA ftp host.
Ingestion is based on fully-qualified file names from the CSA ftp host,
because those are difficult to reproduce otherwise.
"""
builder = nbc.FileNameBuilder(NEOSSatName)
config = mc.Config()
config.get_executors()
state = mc.State(config.state_fqn)
start_time = state.get_bookmark(NEOS_BOOKMARK)
temp = mc.increment_time(start_time, 0).timestamp()
todo_list, max_timestamp = scrape.build_todo(
temp, config.working_directory, config.state_fqn)
max_date = datetime.fromtimestamp(max_timestamp)
incremental_source = data_source.IncrementalSource(todo_list)
transferrer = tc.FtpTransfer(config.data_source)
return rc.run_by_state(config=config, name_builder=builder,
command_name=APPLICATION,
bookmark_name=NEOS_BOOKMARK,
meta_visitors=META_VISITORS,
data_visitors=DATA_VISITORS,
end_time=max_date, chooser=None,
source=incremental_source,
store_transfer=transferrer)
def _run_state():
"""Uses a state file with a timestamp to control which quicklook
files will be retrieved from VLASS.
Ingestion is based on URLs, because a URL that contains the phrase
'QA_REJECTED' is the only way to tell if the attribute 'requirements'
should be set to 'fail', or not.
"""
config = mc.Config()
config.get_executors()
state = mc.State(config.state_fqn)
# a way to get a datetime from a string, or maybe a datetime, depending
# on the execution environment
start_time = mc.increment_time(state.get_bookmark(VLASS_BOOKMARK), 0)
todo_list, max_date = scrape.build_file_url_list(start_time)
source = data_source.NraoPage(todo_list)
name_builder = nbc.EntryBuilder(storage_name.VlassName)
storage_name.set_use_storage_inventory(
config.features.supports_latest_client)
return rc.run_by_state(
config=config,
bookmark_name=VLASS_BOOKMARK,
meta_visitors=META_VISITORS,
data_visitors=DATA_VISITORS,
name_builder=name_builder,
source=source,
end_time=max_date,
store_transfer=tc.HttpTransfer(),
)
def test_run_state_retry(get_work_mock, tap_mock, do_one_mock, test_config):
_write_state(rc.get_utc_now_tz())
retry_success_fqn, retry_failure_fqn, retry_retry_fqn = \
_clean_up_log_files(test_config)
global call_count
call_count = 0
# get_work_mock.return_value.get_time_box_work.side_effect = _mock_get_work
get_work_mock.side_effect = _mock_get_work
do_one_mock.side_effect = _mock_do_one
test_config.log_to_file = True
test_config.retry_failures = True
test_config.state_fqn = STATE_FILE
test_config.interval = 10
test_config.logging_level = 'DEBUG'
test_result = rc.run_by_state(
config=test_config,
command_name=TEST_COMMAND,
bookmark_name=TEST_BOOKMARK,
)
assert test_result is not None, 'expect a result'
assert test_result == -1, 'expect failure'
_check_log_files(
test_config,
retry_success_fqn,
retry_failure_fqn,
retry_retry_fqn,
)
assert do_one_mock.called, 'expect do_one call'
assert do_one_mock.call_count == 2, 'wrong number of calls'
assert tap_mock.called, 'init should be called'
def _run_state():
config = mc.Config()
config.get_executors()
builder = cfht_builder.CFHTBuilder(config)
return rc.run_by_state(config=config,
name_builder=builder,
command_name=main_app.APPLICATION,
bookmark_name=CFHT_BOOKMARK,
meta_visitors=META_VISITORS,
data_visitors=DATA_VISITORS)
def _run_state():
"""Uses a state file with a timestamp to control which entries will be
processed.
"""
config = mc.Config()
config.get_executors()
return rc.run_by_state(name_builder=nbc.FileNameBuilder(VliteName),
command_name=APPLICATION,
meta_visitors=META_VISITORS,
data_visitors=DATA_VISITORS)
def _run_state():
"""Uses a state file with a timestamp to control which entries will be
processed.
"""
return rc.run_by_state(config=None,
name_builder=None,
command_name=APPLICATION,
bookmark_name=None,
meta_visitors=META_VISITORS,
data_visitors=DATA_VISITORS,
end_time=None,
source=None,
chooser=None)
def _run_state():
"""Uses a state file with a timestamp to control which entries will be
processed.
"""
config = mc.Config()
config.get_executors()
source = dsc.QueryTimeBoxDataSource(config, preview_suffix='png')
name_builder = nbc.FileNameBuilder(dao_name.DAOName)
return rc.run_by_state(name_builder=name_builder,
command_name=APPLICATION,
bookmark_name=DAO_BOOKMARK,
meta_visitors=META_VISITORS,
data_visitors=DATA_VISITORS,
source=source)
def _run_incremental():
"""Uses a state file with a timestamp to control which entries will be
processed.
"""
StorageName.collection = COLLECTION
return rc.run_by_state(
config=None,
name_builder=None,
bookmark_name=BLANK_BOOKMARK,
meta_visitors=META_VISITORS,
data_visitors=DATA_VISITORS,
end_time=None,
source=None,
chooser=None,
)
def _run_state():
"""Uses a state file with a timestamp to control which entries will be
processed.
"""
config, clients, name_builder, metadata_reader = _common()
files_source = None
if config.use_local_files:
if config.cleanup_files_when_storing:
files_source = data_source.DAOLocalFilesDataSource(
config, clients.data_client, metadata_reader
)
else:
files_source = dsc.ListDirTimeBoxDataSource(config)
return rc.run_by_state(
name_builder=name_builder,
bookmark_name=DAO_BOOKMARK,
meta_visitors=META_VISITORS,
data_visitors=DATA_VISITORS,
source=files_source,
clients=clients,
metadata_reader=metadata_reader,
)
def _run_state():
"""Run incremental processing for observations that are posted on the site
archive.gemini.edu. TODO in the future this will depend on the incremental
query endpoint.
:return 0 if successful, -1 if there's any sort of failure. Return status
is used by airflow for task instance management and reporting.
"""
(
clients,
config,
metadata_reader,
meta_visitors,
name_builder,
) = _common_init()
state = mc.State(config.state_fqn)
end_timestamp_s = state.bookmarks.get(data_source.GEM_BOOKMARK).get(
'end_timestamp', datetime.now())
end_timestamp_dt = mc.make_time_tz(end_timestamp_s)
logging.info(f'{main_app.APPLICATION} will end at {end_timestamp_s}')
incremental_source = data_source.IncrementalSource(metadata_reader)
result = rc.run_by_state(
config=config,
name_builder=name_builder,
bookmark_name=data_source.GEM_BOOKMARK,
meta_visitors=meta_visitors,
data_visitors=DATA_VISITORS,
end_time=end_timestamp_dt,
source=incremental_source,
clients=clients,
metadata_reader=metadata_reader,
)
if incremental_source.max_records_encountered:
logging.warning('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
logging.warning('Encountered maximum records!!')
logging.warning('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
result |= -1
return result
def _run_by_incremental():
"""Run incremental processing for observations that are posted on the site
archive.gemini.edu. TODO in the future this will depend on the incremental
query endpoint.
:return 0 if successful, -1 if there's any sort of failure. Return status
is used by airflow for task instance management and reporting.
"""
config = mc.Config()
config.get_executors()
state = mc.State(config.state_fqn)
end_timestamp_s = state.bookmarks.get(data_source.GEM_BOOKMARK).get(
'end_timestamp', datetime.now())
end_timestamp_dt = mc.make_time_tz(end_timestamp_s)
logging.info(f'{main_app.APPLICATION} will end at {end_timestamp_s}')
external_metadata.init_global(config=config)
name_builder = nbc.FileNameBuilder(gem_name.GemName)
incremental_source = data_source.IncrementalSource()
meta_visitors = _define_meta_visitors(config)
result = rc.run_by_state(
config=config,
name_builder=name_builder,
command_name=main_app.APPLICATION,
bookmark_name=data_source.GEM_BOOKMARK,
meta_visitors=meta_visitors,
data_visitors=DATA_VISITORS,
end_time=end_timestamp_dt,
source=incremental_source,
chooser=None,
)
if incremental_source.max_records_encountered:
logging.warning('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
logging.warning('Encountered maximum records!!')
logging.warning('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
result |= -1
return result
def test_run_state_v(client_mock):
client_mock.metadata_client.read.side_effect = tc.mock_read
client_mock.data_client.info.return_value = FileInfo(
id='cadc:TEST/anything.fits',
size=42,
md5sum='9473fdd0d880a43c21b7778d34872157',
)
test_wd = '/usr/src/app/caom2pipe/int_test'
caom2pipe_bookmark = 'caom2_timestamp'
test_config = mc.Config()
test_config.working_directory = test_wd
test_config.collection = 'TEST'
test_config.interval = 10
test_config.log_file_directory = f'{test_wd}/logs'
test_config.failure_fqn = (
f'{test_config.log_file_directory}/failure_log.txt')
test_config.log_to_file = True
test_config.logging_level = 'INFO'
test_config.progress_file_name = 'progress.txt'
test_config.proxy_file_name = f'{test_wd}/cadcproxy.pem'
test_config.rejected_file_name = 'rejected.yml'
test_config.rejected_directory = f'{test_wd}/rejected'
test_config._report_fqn = (
f'{test_config.log_file_directory}/app_report.txt')
test_config.resource_id = 'ivo://cadc.nrc.ca/sc2repo'
test_config.retry_file_name = 'retries.txt'
test_config.retry_fqn = (
f'{test_config.log_file_directory}/{test_config.retry_file_name}')
test_config.state_file_name = 'state.yml'
test_config.success_fqn = (
f'{test_config.log_file_directory}/success_log.txt')
test_config.tap_id = 'ivo://cadc.nrc.ca/sc2tap'
test_config.task_types = [
mc.TaskType.STORE,
mc.TaskType.INGEST,
mc.TaskType.MODIFY,
]
test_config.features.use_file_names = True
test_config.features.use_urls = False
test_config.features.supports_latest_client = True
test_config.use_local_files = False
test_config.storage_inventory_resource_id = 'ivo://cadc.nrc.ca/test'
if not os.path.exists(test_wd):
os.mkdir(test_wd)
test_start_time, test_end_time = _get_times(test_config,
caom2pipe_bookmark)
with open(test_config.proxy_fqn, 'w') as f:
f.write('test content\n')
test_data_source = TestListDirTimeBoxDataSource()
test_builder = nbc.GuessingBuilder(tc.TestStorageName)
transferrer = TestTransfer()
try:
test_result = rc.run_by_state(
bookmark_name=caom2pipe_bookmark,
command_name='collection2caom2',
config=test_config,
end_time=test_end_time,
name_builder=test_builder,
source=test_data_source,
modify_transfer=transferrer,
store_transfer=transferrer,
clients=client_mock,
)
assert test_result is not None, 'expect a result'
assert test_result == 0, 'expect success'
assert client_mock.data_client.put.called, 'expect put call'
client_mock.data_client.put.assert_called_with(
'/usr/src/app/caom2pipe/int_test/test_obs_id',
'cadc:TEST/test_file.fits.gz',
None,
), 'wrong call args'
# state file checking
test_state = mc.State(test_config.state_fqn)
assert test_state is not None, 'expect state content'
test_checkpoint = test_state.get_bookmark(caom2pipe_bookmark)
assert test_checkpoint == test_end_time, 'wrong bookmark'
# success file testing
assert os.path.exists(test_config.log_file_directory), 'log directory'
assert os.path.exists(test_config.success_fqn), 'success fqn'
assert os.path.exists(test_config.progress_fqn), 'progress fqn'
log_file = f'{test_config.log_file_directory}/test_obs_id.log'
actual = glob.glob(f'{test_config.log_file_directory}/**')
assert os.path.exists(log_file), f'specific log file {actual}'
xml_file = f'{test_config.log_file_directory}/test_obs_id.xml'
assert os.path.exists(xml_file), f'xml file {actual}'
# reporting testing
report_file = f'{test_config.log_file_directory}/app_report.txt'
assert os.path.exists(report_file), f'report file {actual}'
pass_through_test = False
with open(report_file, 'r') as f:
for line in f:
pass_through_test = True
if 'Number' in line:
bits = line.split(':')
found = False
if 'Inputs' in bits[0]:
assert bits[1].strip() == '1', 'wrong inputs'
found = True
elif 'Successes' in bits[0]:
assert bits[1].strip() == '1', 'wrong successes'
found = True
elif 'Timeouts' in bits[0]:
assert bits[1].strip() == '0', 'wrong timeouts'
found = True
elif 'Retries' in bits[0]:
assert bits[1].strip() == '0', 'wrong retries'
found = True
elif 'Errors' in bits[0]:
assert bits[1].strip() == '0', 'wrong errors'
found = True
elif 'Rejections' in bits[0]:
assert bits[1].strip() == '0', 'wrong rejections'
found = True
assert found, f'{line}'
assert pass_through_test, 'found a report file and checked it'
finally:
f_list = glob.glob(f'{test_wd}/**', recursive=True)
for entry in f_list:
try:
if os.path.isdir(entry):
os.rmdir(entry)
else:
os.unlink(entry)
except OSError as e:
logging.error(f'failed to delete {e}')
def test_run_state(
fits2caom2_mock,
tap_query_mock,
tap_mock,
set_clients_mock,
repo_get_mock,
test_config
):
# tap mock is used by the data_source_composable class
set_clients_mock.side_effect = _clients_mock
fits2caom2_mock.side_effect = _mock_write
repo_get_mock.side_effect = Mock(return_value=None)
tap_query_mock.side_effect = _mock_get_work
test_end_time = datetime.fromtimestamp(1579740838, tz=timezone.utc)
start_time = test_end_time - timedelta(seconds=900)
_write_state(start_time)
test_config.task_types = [mc.TaskType.INGEST]
test_config.state_fqn = STATE_FILE
test_config.interval = 10
if os.path.exists(test_config.progress_fqn):
os.unlink(test_config.progress_fqn)
if os.path.exists(test_config.success_fqn):
os.unlink(test_config.success_fqn)
test_chooser = ec.OrganizeChooser()
test_result = rc.run_by_state(
config=test_config,
chooser=test_chooser,
command_name=TEST_COMMAND,
bookmark_name=TEST_BOOKMARK,
end_time=test_end_time,
)
assert test_result is not None, 'expect a result'
assert test_result == 0, 'expect success'
assert fits2caom2_mock.called, 'expect fits2caom2 call'
fits2caom2_mock.assert_called_once_with()
test_state = mc.State(STATE_FILE)
test_bookmark = test_state.get_bookmark(TEST_BOOKMARK)
assert test_bookmark == test_end_time, 'wrong time'
assert os.path.exists(test_config.progress_fqn), 'expect progress file'
assert (
not os.path.exists(test_config.success_fqn)
), 'log_to_file set to false, no success file'
# test that runner does nothing when times haven't changed
start_time = test_end_time
_write_state(start_time)
fits2caom2_mock.reset_mock()
test_result = rc.run_by_state(
config=test_config,
chooser=test_chooser,
command_name=TEST_COMMAND,
bookmark_name=TEST_BOOKMARK,
end_time=test_end_time,
)
assert test_result is not None, 'expect a result'
assert test_result == 0, 'expect success'
assert not fits2caom2_mock.called, 'expect no fits2caom2 call'
def test_run_state_v(client_mock, repo_mock):
repo_mock.return_value.read.side_effect = tc.mock_read
client_mock.get_node.side_effect = tc.mock_get_node
# the test file is length 0
client_mock.return_value.copy.return_value = 0
test_wd = '/usr/src/app/caom2pipe/int_test'
caom2pipe_bookmark = 'caom2_timestamp'
test_config = mc.Config()
test_config.working_directory = test_wd
test_config.collection = 'TEST'
test_config.interval = 10
test_config.log_file_directory = f'{test_wd}/logs'
test_config.failure_fqn = \
f'{test_config.log_file_directory}/failure_log.txt'
test_config.log_to_file = True
test_config.logging_level = 'DEBUG'
test_config.progress_file_name = 'progress.txt'
test_config.proxy_file_name = f'{test_wd}/cadcproxy.pem'
test_config.rejected_file_name = 'rejected.yml'
test_config.rejected_directory = f'{test_wd}/rejected'
test_config._report_fqn = f'{test_config.log_file_directory}/app_report.txt'
test_config.resource_id = 'ivo://cadc.nrc.ca/sc2repo'
test_config.retry_file_name = 'retries.txt'
test_config.retry_fqn = \
f'{test_config.log_file_directory}/{test_config.retry_file_name}'
test_config.state_file_name = 'state.yml'
test_config.success_fqn = \
f'{test_config.log_file_directory}/success_log.txt'
test_config.tap_id = 'ivo://cadc.nrc.ca/sc2tap'
test_config.task_types = [
mc.TaskType.STORE, mc.TaskType.INGEST, mc.TaskType.MODIFY
]
test_config.features.use_file_names = True
test_config.features.use_urls = False
test_config.features.supports_latest_client = True
test_config.use_local_files = False
if not os.path.exists(test_wd):
os.mkdir(test_wd)
# if this test is failing, did the docker container get
# restarted recently?
# first create /caom2pipe_test/1000003f.fits.fz,
# then check that the test_start_time and test_end_time values
# correspond somewhat to the timestamp on that file
#
# this timestamp is 15 minutes earlier than the timestamp of the
# file in /caom2pipe_test
#
test_start_time = '2021-05-08 02:25:09'
with open(test_config.state_fqn, 'w') as f:
f.write('bookmarks:\n')
f.write(f' {caom2pipe_bookmark}:\n')
f.write(f' last_record: {test_start_time}\n')
test_end_time = datetime(2021,
5,
8,
2,
41,
27,
965132,
tzinfo=timezone.utc)
with open(test_config.proxy_fqn, 'w') as f:
f.write('test content\n')
test_data_source = TestListDirTimeBoxDataSource()
test_builder = nbc.FileNameBuilder(tc.TestStorageName)
transferrer = TestTransfer()
try:
test_result = rc.run_by_state(
bookmark_name=caom2pipe_bookmark,
command_name='collection2caom2',
config=test_config,
end_time=test_end_time,
name_builder=test_builder,
source=test_data_source,
modify_transfer=None,
store_transfer=transferrer,
)
assert test_result is not None, 'expect a result'
assert test_result == 0, 'expect success'
assert client_mock.return_value.copy.called, 'expect put call'
args, kwargs = client_mock.return_value.copy.call_args
assert args[0] == 'ad:TEST/test_obs_id.fits.gz', 'wrong args[0]'
assert (args[1] == '/usr/src/app/caom2pipe/int_test/test_obs_id/'
'test_obs_id.fits'), 'wrong args[1]'
# state file checking
test_state = mc.State(test_config.state_fqn)
assert test_state is not None, 'expect state content'
test_checkpoint = test_state.get_bookmark(caom2pipe_bookmark)
assert test_checkpoint == test_end_time, 'wrong bookmark'
# success file testing
assert os.path.exists(test_config.log_file_directory), 'log directory'
assert os.path.exists(test_config.success_fqn), 'success fqn'
assert os.path.exists(test_config.progress_fqn), 'progress fqn'
log_file = f'{test_config.log_file_directory}/test_obs_id.log'
actual = glob.glob(f'{test_config.log_file_directory}/**')
assert os.path.exists(log_file), f'specific log file {actual}'
xml_file = f'{test_config.log_file_directory}/test_obs_id.fits.xml'
assert os.path.exists(xml_file), f'xml file {actual}'
# reporting testing
report_file = f'{test_config.log_file_directory}/app_report.txt'
assert os.path.exists(report_file), f'report file {actual}'
pass_through_test = False
with open(report_file, 'r') as f:
for line in f:
pass_through_test = True
if 'Number' in line:
bits = line.split(':')
found = False
if 'Inputs' in bits[0]:
assert bits[1].strip() == '1', 'wrong inputs'
found = True
elif 'Successes' in bits[0]:
assert bits[1].strip() == '1', 'wrong successes'
found = True
elif 'Timeouts' in bits[0]:
assert bits[1].strip() == '0', 'wrong timeouts'
found = True
elif 'Retries' in bits[0]:
assert bits[1].strip() == '0', 'wrong retries'
found = True
elif 'Errors' in bits[0]:
assert bits[1].strip() == '0', 'wrong errors'
found = True
elif 'Rejections' in bits[0]:
assert bits[1].strip() == '0', 'wrong rejections'
found = True
assert found, f'{line}'
assert pass_through_test, 'found a report file and checked it'
finally:
f_list = glob.glob(f'{test_wd}/**', recursive=True)
for entry in f_list:
try:
if os.path.isdir(entry):
os.rmdir(entry)
else:
os.unlink(entry)
except OSError as e:
logging.error(f'failed to delete {e}')
def test_run_state_log_to_file_true(
fits2caom2_mock,
tap_mock,
clients_mock,
tap_mock2,
test_config,
):
# tap_mock2 is needed by the data_source_composable specialization
# this test is about making sure the summary .txt files are copied
# as expected when there is more than one time-box
pattern = None
try:
clients_mock.return_value.metadata_client.read.side_effect = Mock(
return_value=SimpleObservation(
collection=test_config.collection,
observation_id='def',
algorithm=Algorithm(str('test')),
))
fits2caom2_mock.side_effect = _mock_write
tap_mock.side_effect = _mock_get_work
test_end_time = datetime.fromtimestamp(1579740838)
start_time = test_end_time - timedelta(seconds=900)
_write_state(start_time)
test_config.task_types = [mc.TaskType.INGEST]
test_config.log_to_file = True
test_config.state_fqn = STATE_FILE
test_config.interval = 10
pattern = f'{test_config.success_fqn.split(".")[0]}*'
if os.path.exists(test_config.progress_fqn):
os.unlink(test_config.progress_fqn)
# preconditions for the success file: - one file named pattern.txt
#
original_success_files = glob.glob(pattern)
for entry in original_success_files:
os.unlink(entry)
if not os.path.exists(test_config.success_fqn):
with open(test_config.success_fqn, 'w') as f:
f.write('test content\n')
test_chooser = ec.OrganizeChooser()
test_result = rc.run_by_state(
config=test_config,
chooser=test_chooser,
command_name='collection2caom2',
bookmark_name=TEST_BOOKMARK,
end_time=test_end_time,
)
assert test_result is not None, 'expect a result'
assert test_result == 0, 'expect success'
assert os.path.exists(test_config.progress_fqn), 'expect progress file'
file_count = glob.glob(pattern)
assert len(file_count) == 2, 'wrong number of success files'
finally:
if pattern is not None:
original_success_files = glob.glob(pattern)
for entry in original_success_files:
os.unlink(entry)
def test_run_state(
fits2caom2_mock,
fits2caom2_in_out_mock,
tap_query_mock,
tap_mock,
clients_mock,
test_config,
):
# tap_mock is used by the data_source_composable class
fits2caom2_mock.side_effect = _mock_write
clients_mock.return_value.metadata_client.read.side_effect = Mock(
return_value=None)
tap_query_mock.side_effect = _mock_get_work
test_end_time = datetime.fromtimestamp(1579740838, tz=timezone.utc)
start_time = test_end_time - timedelta(seconds=900)
_write_state(start_time)
test_config.task_types = [mc.TaskType.INGEST]
test_config.state_fqn = STATE_FILE
test_config.interval = 10
individual_log_file = (
f'{test_config.log_file_directory}/NEOS_SCI_2015347000000_clean.log')
if os.path.exists(test_config.progress_fqn):
os.unlink(test_config.progress_fqn)
if os.path.exists(test_config.success_fqn):
os.unlink(test_config.success_fqn)
if os.path.exists(individual_log_file):
os.unlink(individual_log_file)
test_chooser = ec.OrganizeChooser()
# use_local_files set so run_by_state chooses QueryTimeBoxDataSourceTS
test_config.use_local_files = False
test_result = rc.run_by_state(
config=test_config,
chooser=test_chooser,
command_name=TEST_COMMAND,
bookmark_name=TEST_BOOKMARK,
end_time=test_end_time,
)
assert test_result is not None, 'expect a result'
assert test_result == 0, 'expect success'
if fits2caom2_mock.called:
fits2caom2_mock.assert_called_once_with()
elif fits2caom2_in_out_mock.called:
fits2caom2_in_out_mock.assert_called_once_with(ANY)
test_state = mc.State(STATE_FILE)
test_bookmark = test_state.get_bookmark(TEST_BOOKMARK)
assert test_bookmark == test_end_time, 'wrong time'
assert os.path.exists(test_config.progress_fqn), 'expect progress file'
assert os.path.exists(
test_config.success_fqn), 'log_to_file set to false, no success file'
assert not os.path.exists(
individual_log_file), f'log_to_file is False, no entry log'
# test that runner does nothing when times haven't changed
start_time = test_end_time
_write_state(start_time)
fits2caom2_mock.reset_mock()
fits2caom2_in_out_mock.reset_mock()
test_result = rc.run_by_state(
config=test_config,
chooser=test_chooser,
command_name=TEST_COMMAND,
bookmark_name=TEST_BOOKMARK,
end_time=test_end_time,
)
assert test_result is not None, 'expect a result'
assert test_result == 0, 'expect success'
assert not fits2caom2_mock.called, 'expect no fits2caom2 call'
assert (
not fits2caom2_in_out_mock.called), 'expect no update fits2caom2 call'