def _run_by_public(): """Run the processing for observations that are public, but there are no artifacts representing the previews in CAOM, or a FITS file in ad. Called as gem_run_public. The time-boxing is based on timestamps from a state.yml file. Call once/day, since data release timestamps have times of 00:00:00.000. :return 0 if successful, -1 if there's any sort of failure. Return status is used by airflow for task instance management and reporting. """ config = mc.Config() config.get_executors() external_metadata.init_global(config=config) name_builder = nbc.FileNameBuilder(gem_name.GemName) incremental_source = data_source.PublicIncremental(config) meta_visitors = _define_meta_visitors(config) return rc.run_by_state(config=config, name_builder=name_builder, command_name=main_app.APPLICATION, bookmark_name=data_source.GEM_BOOKMARK, meta_visitors=meta_visitors, data_visitors=DATA_VISITORS, end_time=None, source=incremental_source, chooser=None)
def _run_by_state(): """Uses a state file with a timestamp to control which quicklook files will be retrieved from VLASS. Ingestion is based on URLs, because a URL that contains the phrase 'QA_REJECTED' is the only way to tell if the attribute 'requirements' should be set to 'fail', or not. """ config = mc.Config() config.get_executors() state = mc.State(config.state_fqn) # a way to get a datetime from a string, or maybe a datetime, depending # on the execution environment start_time = mc.increment_time(state.get_bookmark(VLASS_BOOKMARK), 0) todo_list, max_date = scrape.build_file_url_list(start_time) if len(todo_list) > 0: state = mc.State(config.state_fqn) work.init_web_log(state, config) # still make all subsequent calls if len == 0, for consistent reporting source = data_source.NraoPage(todo_list) name_builder = builder.VlassInstanceBuilder(config) return rc.run_by_state(config=config, command_name=sn.APPLICATION, bookmark_name=VLASS_BOOKMARK, meta_visitors=META_VISITORS, data_visitors=DATA_VISITORS, name_builder=name_builder, source=source, end_time=max_date, store_transfer=tc.HttpTransfer())
def _run_by_public(): """Run the processing for observations that are public, but there are no artifacts representing the previews in CAOM, or a FITS file in ad. Called as gem_run_public. The time-boxing is based on timestamps from a state.yml file. Call once/day, since data release timestamps have times of 00:00:00.000. :return 0 if successful, -1 if there's any sort of failure. Return status is used by airflow for task instance management and reporting. """ ( clients, config, metadata_reader, meta_visitors, name_builder, ) = _common_init() incremental_source = data_source.PublicIncremental(config, clients.query_client) return rc.run_by_state( config=config, name_builder=name_builder, bookmark_name=data_source.GEM_BOOKMARK, meta_visitors=meta_visitors, data_visitors=DATA_VISITORS, end_time=None, source=incremental_source, clients=clients, metadata_reader=metadata_reader, )
def _run_state(): """Uses a state file with a timestamp to control which files will be retrieved from the CSA ftp host. Ingestion is based on fully-qualified file names from the CSA ftp host, because those are difficult to reproduce otherwise. """ builder = nbc.FileNameBuilder(NEOSSatName) config = mc.Config() config.get_executors() state = mc.State(config.state_fqn) start_time = state.get_bookmark(NEOS_BOOKMARK) temp = mc.increment_time(start_time, 0).timestamp() todo_list, max_timestamp = scrape.build_todo( temp, config.working_directory, config.state_fqn) max_date = datetime.fromtimestamp(max_timestamp) incremental_source = data_source.IncrementalSource(todo_list) transferrer = tc.FtpTransfer(config.data_source) return rc.run_by_state(config=config, name_builder=builder, command_name=APPLICATION, bookmark_name=NEOS_BOOKMARK, meta_visitors=META_VISITORS, data_visitors=DATA_VISITORS, end_time=max_date, chooser=None, source=incremental_source, store_transfer=transferrer)
def _run_state(): """Uses a state file with a timestamp to control which quicklook files will be retrieved from VLASS. Ingestion is based on URLs, because a URL that contains the phrase 'QA_REJECTED' is the only way to tell if the attribute 'requirements' should be set to 'fail', or not. """ config = mc.Config() config.get_executors() state = mc.State(config.state_fqn) # a way to get a datetime from a string, or maybe a datetime, depending # on the execution environment start_time = mc.increment_time(state.get_bookmark(VLASS_BOOKMARK), 0) todo_list, max_date = scrape.build_file_url_list(start_time) source = data_source.NraoPage(todo_list) name_builder = nbc.EntryBuilder(storage_name.VlassName) storage_name.set_use_storage_inventory( config.features.supports_latest_client) return rc.run_by_state( config=config, bookmark_name=VLASS_BOOKMARK, meta_visitors=META_VISITORS, data_visitors=DATA_VISITORS, name_builder=name_builder, source=source, end_time=max_date, store_transfer=tc.HttpTransfer(), )
def test_run_state_retry(get_work_mock, tap_mock, do_one_mock, test_config): _write_state(rc.get_utc_now_tz()) retry_success_fqn, retry_failure_fqn, retry_retry_fqn = \ _clean_up_log_files(test_config) global call_count call_count = 0 # get_work_mock.return_value.get_time_box_work.side_effect = _mock_get_work get_work_mock.side_effect = _mock_get_work do_one_mock.side_effect = _mock_do_one test_config.log_to_file = True test_config.retry_failures = True test_config.state_fqn = STATE_FILE test_config.interval = 10 test_config.logging_level = 'DEBUG' test_result = rc.run_by_state( config=test_config, command_name=TEST_COMMAND, bookmark_name=TEST_BOOKMARK, ) assert test_result is not None, 'expect a result' assert test_result == -1, 'expect failure' _check_log_files( test_config, retry_success_fqn, retry_failure_fqn, retry_retry_fqn, ) assert do_one_mock.called, 'expect do_one call' assert do_one_mock.call_count == 2, 'wrong number of calls' assert tap_mock.called, 'init should be called'
def _run_state(): config = mc.Config() config.get_executors() builder = cfht_builder.CFHTBuilder(config) return rc.run_by_state(config=config, name_builder=builder, command_name=main_app.APPLICATION, bookmark_name=CFHT_BOOKMARK, meta_visitors=META_VISITORS, data_visitors=DATA_VISITORS)
def _run_state(): """Uses a state file with a timestamp to control which entries will be processed. """ config = mc.Config() config.get_executors() return rc.run_by_state(name_builder=nbc.FileNameBuilder(VliteName), command_name=APPLICATION, meta_visitors=META_VISITORS, data_visitors=DATA_VISITORS)
def _run_state(): """Uses a state file with a timestamp to control which entries will be processed. """ return rc.run_by_state(config=None, name_builder=None, command_name=APPLICATION, bookmark_name=None, meta_visitors=META_VISITORS, data_visitors=DATA_VISITORS, end_time=None, source=None, chooser=None)
def _run_state(): """Uses a state file with a timestamp to control which entries will be processed. """ config = mc.Config() config.get_executors() source = dsc.QueryTimeBoxDataSource(config, preview_suffix='png') name_builder = nbc.FileNameBuilder(dao_name.DAOName) return rc.run_by_state(name_builder=name_builder, command_name=APPLICATION, bookmark_name=DAO_BOOKMARK, meta_visitors=META_VISITORS, data_visitors=DATA_VISITORS, source=source)
def _run_incremental(): """Uses a state file with a timestamp to control which entries will be processed. """ StorageName.collection = COLLECTION return rc.run_by_state( config=None, name_builder=None, bookmark_name=BLANK_BOOKMARK, meta_visitors=META_VISITORS, data_visitors=DATA_VISITORS, end_time=None, source=None, chooser=None, )
def _run_state(): """Uses a state file with a timestamp to control which entries will be processed. """ config, clients, name_builder, metadata_reader = _common() files_source = None if config.use_local_files: if config.cleanup_files_when_storing: files_source = data_source.DAOLocalFilesDataSource( config, clients.data_client, metadata_reader ) else: files_source = dsc.ListDirTimeBoxDataSource(config) return rc.run_by_state( name_builder=name_builder, bookmark_name=DAO_BOOKMARK, meta_visitors=META_VISITORS, data_visitors=DATA_VISITORS, source=files_source, clients=clients, metadata_reader=metadata_reader, )
def _run_state(): """Run incremental processing for observations that are posted on the site archive.gemini.edu. TODO in the future this will depend on the incremental query endpoint. :return 0 if successful, -1 if there's any sort of failure. Return status is used by airflow for task instance management and reporting. """ ( clients, config, metadata_reader, meta_visitors, name_builder, ) = _common_init() state = mc.State(config.state_fqn) end_timestamp_s = state.bookmarks.get(data_source.GEM_BOOKMARK).get( 'end_timestamp', datetime.now()) end_timestamp_dt = mc.make_time_tz(end_timestamp_s) logging.info(f'{main_app.APPLICATION} will end at {end_timestamp_s}') incremental_source = data_source.IncrementalSource(metadata_reader) result = rc.run_by_state( config=config, name_builder=name_builder, bookmark_name=data_source.GEM_BOOKMARK, meta_visitors=meta_visitors, data_visitors=DATA_VISITORS, end_time=end_timestamp_dt, source=incremental_source, clients=clients, metadata_reader=metadata_reader, ) if incremental_source.max_records_encountered: logging.warning('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!') logging.warning('Encountered maximum records!!') logging.warning('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!') result |= -1 return result
def _run_by_incremental(): """Run incremental processing for observations that are posted on the site archive.gemini.edu. TODO in the future this will depend on the incremental query endpoint. :return 0 if successful, -1 if there's any sort of failure. Return status is used by airflow for task instance management and reporting. """ config = mc.Config() config.get_executors() state = mc.State(config.state_fqn) end_timestamp_s = state.bookmarks.get(data_source.GEM_BOOKMARK).get( 'end_timestamp', datetime.now()) end_timestamp_dt = mc.make_time_tz(end_timestamp_s) logging.info(f'{main_app.APPLICATION} will end at {end_timestamp_s}') external_metadata.init_global(config=config) name_builder = nbc.FileNameBuilder(gem_name.GemName) incremental_source = data_source.IncrementalSource() meta_visitors = _define_meta_visitors(config) result = rc.run_by_state( config=config, name_builder=name_builder, command_name=main_app.APPLICATION, bookmark_name=data_source.GEM_BOOKMARK, meta_visitors=meta_visitors, data_visitors=DATA_VISITORS, end_time=end_timestamp_dt, source=incremental_source, chooser=None, ) if incremental_source.max_records_encountered: logging.warning('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!') logging.warning('Encountered maximum records!!') logging.warning('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!') result |= -1 return result
def test_run_state_v(client_mock): client_mock.metadata_client.read.side_effect = tc.mock_read client_mock.data_client.info.return_value = FileInfo( id='cadc:TEST/anything.fits', size=42, md5sum='9473fdd0d880a43c21b7778d34872157', ) test_wd = '/usr/src/app/caom2pipe/int_test' caom2pipe_bookmark = 'caom2_timestamp' test_config = mc.Config() test_config.working_directory = test_wd test_config.collection = 'TEST' test_config.interval = 10 test_config.log_file_directory = f'{test_wd}/logs' test_config.failure_fqn = ( f'{test_config.log_file_directory}/failure_log.txt') test_config.log_to_file = True test_config.logging_level = 'INFO' test_config.progress_file_name = 'progress.txt' test_config.proxy_file_name = f'{test_wd}/cadcproxy.pem' test_config.rejected_file_name = 'rejected.yml' test_config.rejected_directory = f'{test_wd}/rejected' test_config._report_fqn = ( f'{test_config.log_file_directory}/app_report.txt') test_config.resource_id = 'ivo://cadc.nrc.ca/sc2repo' test_config.retry_file_name = 'retries.txt' test_config.retry_fqn = ( f'{test_config.log_file_directory}/{test_config.retry_file_name}') test_config.state_file_name = 'state.yml' test_config.success_fqn = ( f'{test_config.log_file_directory}/success_log.txt') test_config.tap_id = 'ivo://cadc.nrc.ca/sc2tap' test_config.task_types = [ mc.TaskType.STORE, mc.TaskType.INGEST, mc.TaskType.MODIFY, ] test_config.features.use_file_names = True test_config.features.use_urls = False test_config.features.supports_latest_client = True test_config.use_local_files = False test_config.storage_inventory_resource_id = 'ivo://cadc.nrc.ca/test' if not os.path.exists(test_wd): os.mkdir(test_wd) test_start_time, test_end_time = _get_times(test_config, caom2pipe_bookmark) with open(test_config.proxy_fqn, 'w') as f: f.write('test content\n') test_data_source = TestListDirTimeBoxDataSource() test_builder = nbc.GuessingBuilder(tc.TestStorageName) transferrer = TestTransfer() try: test_result = rc.run_by_state( bookmark_name=caom2pipe_bookmark, command_name='collection2caom2', config=test_config, end_time=test_end_time, name_builder=test_builder, source=test_data_source, modify_transfer=transferrer, store_transfer=transferrer, clients=client_mock, ) assert test_result is not None, 'expect a result' assert test_result == 0, 'expect success' assert client_mock.data_client.put.called, 'expect put call' client_mock.data_client.put.assert_called_with( '/usr/src/app/caom2pipe/int_test/test_obs_id', 'cadc:TEST/test_file.fits.gz', None, ), 'wrong call args' # state file checking test_state = mc.State(test_config.state_fqn) assert test_state is not None, 'expect state content' test_checkpoint = test_state.get_bookmark(caom2pipe_bookmark) assert test_checkpoint == test_end_time, 'wrong bookmark' # success file testing assert os.path.exists(test_config.log_file_directory), 'log directory' assert os.path.exists(test_config.success_fqn), 'success fqn' assert os.path.exists(test_config.progress_fqn), 'progress fqn' log_file = f'{test_config.log_file_directory}/test_obs_id.log' actual = glob.glob(f'{test_config.log_file_directory}/**') assert os.path.exists(log_file), f'specific log file {actual}' xml_file = f'{test_config.log_file_directory}/test_obs_id.xml' assert os.path.exists(xml_file), f'xml file {actual}' # reporting testing report_file = f'{test_config.log_file_directory}/app_report.txt' assert os.path.exists(report_file), f'report file {actual}' pass_through_test = False with open(report_file, 'r') as f: for line in f: pass_through_test = True if 'Number' in line: bits = line.split(':') found = False if 'Inputs' in bits[0]: assert bits[1].strip() == '1', 'wrong inputs' found = True elif 'Successes' in bits[0]: assert bits[1].strip() == '1', 'wrong successes' found = True elif 'Timeouts' in bits[0]: assert bits[1].strip() == '0', 'wrong timeouts' found = True elif 'Retries' in bits[0]: assert bits[1].strip() == '0', 'wrong retries' found = True elif 'Errors' in bits[0]: assert bits[1].strip() == '0', 'wrong errors' found = True elif 'Rejections' in bits[0]: assert bits[1].strip() == '0', 'wrong rejections' found = True assert found, f'{line}' assert pass_through_test, 'found a report file and checked it' finally: f_list = glob.glob(f'{test_wd}/**', recursive=True) for entry in f_list: try: if os.path.isdir(entry): os.rmdir(entry) else: os.unlink(entry) except OSError as e: logging.error(f'failed to delete {e}')
def test_run_state( fits2caom2_mock, tap_query_mock, tap_mock, set_clients_mock, repo_get_mock, test_config ): # tap mock is used by the data_source_composable class set_clients_mock.side_effect = _clients_mock fits2caom2_mock.side_effect = _mock_write repo_get_mock.side_effect = Mock(return_value=None) tap_query_mock.side_effect = _mock_get_work test_end_time = datetime.fromtimestamp(1579740838, tz=timezone.utc) start_time = test_end_time - timedelta(seconds=900) _write_state(start_time) test_config.task_types = [mc.TaskType.INGEST] test_config.state_fqn = STATE_FILE test_config.interval = 10 if os.path.exists(test_config.progress_fqn): os.unlink(test_config.progress_fqn) if os.path.exists(test_config.success_fqn): os.unlink(test_config.success_fqn) test_chooser = ec.OrganizeChooser() test_result = rc.run_by_state( config=test_config, chooser=test_chooser, command_name=TEST_COMMAND, bookmark_name=TEST_BOOKMARK, end_time=test_end_time, ) assert test_result is not None, 'expect a result' assert test_result == 0, 'expect success' assert fits2caom2_mock.called, 'expect fits2caom2 call' fits2caom2_mock.assert_called_once_with() test_state = mc.State(STATE_FILE) test_bookmark = test_state.get_bookmark(TEST_BOOKMARK) assert test_bookmark == test_end_time, 'wrong time' assert os.path.exists(test_config.progress_fqn), 'expect progress file' assert ( not os.path.exists(test_config.success_fqn) ), 'log_to_file set to false, no success file' # test that runner does nothing when times haven't changed start_time = test_end_time _write_state(start_time) fits2caom2_mock.reset_mock() test_result = rc.run_by_state( config=test_config, chooser=test_chooser, command_name=TEST_COMMAND, bookmark_name=TEST_BOOKMARK, end_time=test_end_time, ) assert test_result is not None, 'expect a result' assert test_result == 0, 'expect success' assert not fits2caom2_mock.called, 'expect no fits2caom2 call'
def test_run_state_v(client_mock, repo_mock): repo_mock.return_value.read.side_effect = tc.mock_read client_mock.get_node.side_effect = tc.mock_get_node # the test file is length 0 client_mock.return_value.copy.return_value = 0 test_wd = '/usr/src/app/caom2pipe/int_test' caom2pipe_bookmark = 'caom2_timestamp' test_config = mc.Config() test_config.working_directory = test_wd test_config.collection = 'TEST' test_config.interval = 10 test_config.log_file_directory = f'{test_wd}/logs' test_config.failure_fqn = \ f'{test_config.log_file_directory}/failure_log.txt' test_config.log_to_file = True test_config.logging_level = 'DEBUG' test_config.progress_file_name = 'progress.txt' test_config.proxy_file_name = f'{test_wd}/cadcproxy.pem' test_config.rejected_file_name = 'rejected.yml' test_config.rejected_directory = f'{test_wd}/rejected' test_config._report_fqn = f'{test_config.log_file_directory}/app_report.txt' test_config.resource_id = 'ivo://cadc.nrc.ca/sc2repo' test_config.retry_file_name = 'retries.txt' test_config.retry_fqn = \ f'{test_config.log_file_directory}/{test_config.retry_file_name}' test_config.state_file_name = 'state.yml' test_config.success_fqn = \ f'{test_config.log_file_directory}/success_log.txt' test_config.tap_id = 'ivo://cadc.nrc.ca/sc2tap' test_config.task_types = [ mc.TaskType.STORE, mc.TaskType.INGEST, mc.TaskType.MODIFY ] test_config.features.use_file_names = True test_config.features.use_urls = False test_config.features.supports_latest_client = True test_config.use_local_files = False if not os.path.exists(test_wd): os.mkdir(test_wd) # if this test is failing, did the docker container get # restarted recently? # first create /caom2pipe_test/1000003f.fits.fz, # then check that the test_start_time and test_end_time values # correspond somewhat to the timestamp on that file # # this timestamp is 15 minutes earlier than the timestamp of the # file in /caom2pipe_test # test_start_time = '2021-05-08 02:25:09' with open(test_config.state_fqn, 'w') as f: f.write('bookmarks:\n') f.write(f' {caom2pipe_bookmark}:\n') f.write(f' last_record: {test_start_time}\n') test_end_time = datetime(2021, 5, 8, 2, 41, 27, 965132, tzinfo=timezone.utc) with open(test_config.proxy_fqn, 'w') as f: f.write('test content\n') test_data_source = TestListDirTimeBoxDataSource() test_builder = nbc.FileNameBuilder(tc.TestStorageName) transferrer = TestTransfer() try: test_result = rc.run_by_state( bookmark_name=caom2pipe_bookmark, command_name='collection2caom2', config=test_config, end_time=test_end_time, name_builder=test_builder, source=test_data_source, modify_transfer=None, store_transfer=transferrer, ) assert test_result is not None, 'expect a result' assert test_result == 0, 'expect success' assert client_mock.return_value.copy.called, 'expect put call' args, kwargs = client_mock.return_value.copy.call_args assert args[0] == 'ad:TEST/test_obs_id.fits.gz', 'wrong args[0]' assert (args[1] == '/usr/src/app/caom2pipe/int_test/test_obs_id/' 'test_obs_id.fits'), 'wrong args[1]' # state file checking test_state = mc.State(test_config.state_fqn) assert test_state is not None, 'expect state content' test_checkpoint = test_state.get_bookmark(caom2pipe_bookmark) assert test_checkpoint == test_end_time, 'wrong bookmark' # success file testing assert os.path.exists(test_config.log_file_directory), 'log directory' assert os.path.exists(test_config.success_fqn), 'success fqn' assert os.path.exists(test_config.progress_fqn), 'progress fqn' log_file = f'{test_config.log_file_directory}/test_obs_id.log' actual = glob.glob(f'{test_config.log_file_directory}/**') assert os.path.exists(log_file), f'specific log file {actual}' xml_file = f'{test_config.log_file_directory}/test_obs_id.fits.xml' assert os.path.exists(xml_file), f'xml file {actual}' # reporting testing report_file = f'{test_config.log_file_directory}/app_report.txt' assert os.path.exists(report_file), f'report file {actual}' pass_through_test = False with open(report_file, 'r') as f: for line in f: pass_through_test = True if 'Number' in line: bits = line.split(':') found = False if 'Inputs' in bits[0]: assert bits[1].strip() == '1', 'wrong inputs' found = True elif 'Successes' in bits[0]: assert bits[1].strip() == '1', 'wrong successes' found = True elif 'Timeouts' in bits[0]: assert bits[1].strip() == '0', 'wrong timeouts' found = True elif 'Retries' in bits[0]: assert bits[1].strip() == '0', 'wrong retries' found = True elif 'Errors' in bits[0]: assert bits[1].strip() == '0', 'wrong errors' found = True elif 'Rejections' in bits[0]: assert bits[1].strip() == '0', 'wrong rejections' found = True assert found, f'{line}' assert pass_through_test, 'found a report file and checked it' finally: f_list = glob.glob(f'{test_wd}/**', recursive=True) for entry in f_list: try: if os.path.isdir(entry): os.rmdir(entry) else: os.unlink(entry) except OSError as e: logging.error(f'failed to delete {e}')
def test_run_state_log_to_file_true( fits2caom2_mock, tap_mock, clients_mock, tap_mock2, test_config, ): # tap_mock2 is needed by the data_source_composable specialization # this test is about making sure the summary .txt files are copied # as expected when there is more than one time-box pattern = None try: clients_mock.return_value.metadata_client.read.side_effect = Mock( return_value=SimpleObservation( collection=test_config.collection, observation_id='def', algorithm=Algorithm(str('test')), )) fits2caom2_mock.side_effect = _mock_write tap_mock.side_effect = _mock_get_work test_end_time = datetime.fromtimestamp(1579740838) start_time = test_end_time - timedelta(seconds=900) _write_state(start_time) test_config.task_types = [mc.TaskType.INGEST] test_config.log_to_file = True test_config.state_fqn = STATE_FILE test_config.interval = 10 pattern = f'{test_config.success_fqn.split(".")[0]}*' if os.path.exists(test_config.progress_fqn): os.unlink(test_config.progress_fqn) # preconditions for the success file: - one file named pattern.txt # original_success_files = glob.glob(pattern) for entry in original_success_files: os.unlink(entry) if not os.path.exists(test_config.success_fqn): with open(test_config.success_fqn, 'w') as f: f.write('test content\n') test_chooser = ec.OrganizeChooser() test_result = rc.run_by_state( config=test_config, chooser=test_chooser, command_name='collection2caom2', bookmark_name=TEST_BOOKMARK, end_time=test_end_time, ) assert test_result is not None, 'expect a result' assert test_result == 0, 'expect success' assert os.path.exists(test_config.progress_fqn), 'expect progress file' file_count = glob.glob(pattern) assert len(file_count) == 2, 'wrong number of success files' finally: if pattern is not None: original_success_files = glob.glob(pattern) for entry in original_success_files: os.unlink(entry)
def test_run_state( fits2caom2_mock, fits2caom2_in_out_mock, tap_query_mock, tap_mock, clients_mock, test_config, ): # tap_mock is used by the data_source_composable class fits2caom2_mock.side_effect = _mock_write clients_mock.return_value.metadata_client.read.side_effect = Mock( return_value=None) tap_query_mock.side_effect = _mock_get_work test_end_time = datetime.fromtimestamp(1579740838, tz=timezone.utc) start_time = test_end_time - timedelta(seconds=900) _write_state(start_time) test_config.task_types = [mc.TaskType.INGEST] test_config.state_fqn = STATE_FILE test_config.interval = 10 individual_log_file = ( f'{test_config.log_file_directory}/NEOS_SCI_2015347000000_clean.log') if os.path.exists(test_config.progress_fqn): os.unlink(test_config.progress_fqn) if os.path.exists(test_config.success_fqn): os.unlink(test_config.success_fqn) if os.path.exists(individual_log_file): os.unlink(individual_log_file) test_chooser = ec.OrganizeChooser() # use_local_files set so run_by_state chooses QueryTimeBoxDataSourceTS test_config.use_local_files = False test_result = rc.run_by_state( config=test_config, chooser=test_chooser, command_name=TEST_COMMAND, bookmark_name=TEST_BOOKMARK, end_time=test_end_time, ) assert test_result is not None, 'expect a result' assert test_result == 0, 'expect success' if fits2caom2_mock.called: fits2caom2_mock.assert_called_once_with() elif fits2caom2_in_out_mock.called: fits2caom2_in_out_mock.assert_called_once_with(ANY) test_state = mc.State(STATE_FILE) test_bookmark = test_state.get_bookmark(TEST_BOOKMARK) assert test_bookmark == test_end_time, 'wrong time' assert os.path.exists(test_config.progress_fqn), 'expect progress file' assert os.path.exists( test_config.success_fqn), 'log_to_file set to false, no success file' assert not os.path.exists( individual_log_file), f'log_to_file is False, no entry log' # test that runner does nothing when times haven't changed start_time = test_end_time _write_state(start_time) fits2caom2_mock.reset_mock() fits2caom2_in_out_mock.reset_mock() test_result = rc.run_by_state( config=test_config, chooser=test_chooser, command_name=TEST_COMMAND, bookmark_name=TEST_BOOKMARK, end_time=test_end_time, ) assert test_result is not None, 'expect a result' assert test_result == 0, 'expect success' assert not fits2caom2_mock.called, 'expect no fits2caom2 call' assert ( not fits2caom2_in_out_mock.called), 'expect no update fits2caom2 call'