def test_run_todo_file_data_source_v(
repo_read_mock, set_clients_mock, test_config
):
set_clients_mock.side_effect = _clients_mock
test_config.features.supports_latest_client = True
test_cert_file = os.path.join(TEST_DIR, 'test_proxy.pem')
test_config.proxy_fqn = test_cert_file
repo_read_mock.return_value = SimpleObservation(
collection=test_config.collection,
observation_id='def',
algorithm=Algorithm(str('test')),
)
if os.path.exists(test_config.success_fqn):
os.unlink(test_config.success_fqn)
test_config.work_fqn = f'{TEST_DIR}/todo.txt'
test_config.task_types = [mc.TaskType.VISIT]
test_config.log_to_file = True
test_chooser = ec.OrganizeChooser()
test_result = rc.run_by_todo(
config=test_config, chooser=test_chooser, command_name=TEST_COMMAND
)
assert test_result is not None, 'expect a result'
assert test_result == 0, 'expect success'
assert os.path.exists(test_config.success_fqn), 'expect success file'
with open(test_config.success_fqn) as f:
content = f.read()
# the obs id and file name
assert 'def def.fits' in content, 'wrong success message'
assert repo_read_mock.called, 'expect e call'
repo_read_mock.assert_called_with(), 'wrong e args'
def test_run_todo_file_data_source(clients_mock, test_config):
clients_mock.return_value.data_client.get_file_info.return_value = None
clients_mock.return_value.metadata_client.read.return_value = (
SimpleObservation(
collection=test_config.collection,
observation_id='def',
algorithm=Algorithm(str('test')),
))
if os.path.exists(test_config.success_fqn):
os.unlink(test_config.success_fqn)
test_config.work_fqn = f'{TEST_DIR}/todo.txt'
test_config.task_types = [mc.TaskType.VISIT]
test_config.log_to_file = True
test_chooser = ec.OrganizeChooser()
test_result = rc.run_by_todo(config=test_config,
chooser=test_chooser,
command_name=TEST_COMMAND)
assert test_result is not None, 'expect a result'
assert test_result == 0, 'expect success'
assert os.path.exists(test_config.success_fqn), 'expect success file'
with open(test_config.success_fqn) as f:
content = f.read()
# the obs id and file name
assert 'def def.fits' in content, 'wrong success message'
def test_run_todo_list_dir_data_source(
write_obs_mock,
read_obs_mock,
fits2caom2_in_out_mock,
fits2caom2_mock,
test_config,
):
read_obs_mock.side_effect = _mock_read
test_config.working_directory = tc.TEST_DATA_DIR
test_config.use_local_files = True
test_config.task_types = [mc.TaskType.SCRAPE]
test_config.data_sources = [tc.TEST_FILES_DIR]
test_config.data_source_extensions = ['.fits']
test_chooser = ec.OrganizeChooser()
test_result = rc.run_by_todo(config=test_config,
chooser=test_chooser,
command_name=TEST_COMMAND)
assert test_result is not None, 'expect a result'
assert test_result == 0, 'expect success'
if fits2caom2_mock.called:
fits2caom2_mock.assert_called_with(connected=False)
else:
assert fits2caom2_in_out_mock.called, 'expect fits2caom2 in/out call'
assert write_obs_mock.called, 'expect write call'
def test_run_state_log_to_file_true(
fits2caom2_mock,
tap_mock,
read_obs_mock,
set_clients_mock,
tap_mock2,
test_config,
):
# tap_mock2 is needed by the data_source_composable specialization
set_clients_mock.side_effect = _clients_mock
# this test is about making sure the summary .txt files are copied
# as expected when there is more than one time-box
pattern = None
try:
read_obs_mock.side_effect = _mock_read_2
fits2caom2_mock.side_effect = _mock_write
tap_mock.side_effect = _mock_get_work
test_end_time = datetime.fromtimestamp(1579740838)
start_time = test_end_time - timedelta(seconds=900)
_write_state(start_time)
test_config.task_types = [mc.TaskType.INGEST]
test_config.log_to_file = True
test_config.state_fqn = STATE_FILE
test_config.interval = 10
pattern = f'{test_config.success_fqn.split(".")[0]}*'
if os.path.exists(test_config.progress_fqn):
os.unlink(test_config.progress_fqn)
# preconditions for the success file: - one file named pattern.txt
#
original_success_files = glob.glob(pattern)
for entry in original_success_files:
os.unlink(entry)
if not os.path.exists(test_config.success_fqn):
with open(test_config.success_fqn, 'w') as f:
f.write('test content\n')
test_chooser = ec.OrganizeChooser()
test_result = rc.run_by_state_ad(
config=test_config,
chooser=test_chooser,
command_name='collection2caom2',
bookmark_name=TEST_BOOKMARK,
end_time=test_end_time,
)
assert test_result is not None, 'expect a result'
assert test_result == 0, 'expect success'
assert os.path.exists(test_config.progress_fqn), 'expect progress file'
file_count = glob.glob(pattern)
assert len(file_count) == 2, 'wrong number of success files'
finally:
if pattern is not None:
original_success_files = glob.glob(pattern)
for entry in original_success_files:
os.unlink(entry)
def test_run_todo_list_dir_data_source_invalid_fname_v(
set_clients_mock, test_config
):
set_clients_mock.side_effect = _clients_mock
test_config.working_directory = TEST_DIR
test_config.use_local_files = True
test_config.task_types = [mc.TaskType.INGEST]
test_config.log_to_file = False
test_config.features.supports_latest_client = True
if os.path.exists(test_config.failure_fqn):
os.unlink(test_config.failure_fqn)
if os.path.exists(test_config.retry_fqn):
os.unlink(test_config.retry_fqn)
if not os.path.exists(f'{TEST_DIR}/abc.fits.gz'):
with open(f'{TEST_DIR}/abc.fits.gz', 'w') as f:
f.write('abc')
class TestStorageName(mc.StorageName):
def __init__(self, entry):
self._obs_id = entry
def is_valid(self):
return False
class TestStorageNameInstanceBuilder(b.StorageNameInstanceBuilder):
def __init__(self):
pass
def build(self, entry):
return TestStorageName(entry)
test_builder = TestStorageNameInstanceBuilder()
test_chooser = ec.OrganizeChooser()
test_result = rc.run_by_todo(
config=test_config,
chooser=test_chooser,
name_builder=test_builder,
command_name=TEST_COMMAND,
)
assert test_result is not None, 'expect a result'
assert test_result == -1, 'expect failure, because of file naming'
assert (
not os.path.exists(test_config.failure_fqn)
), 'no logging, no failure file'
assert (
not os.path.exists(test_config.retry_fqn)
), 'no logging, no retry file'
test_config.log_to_file = True
test_result = rc.run_by_todo(
config=test_config,
chooser=test_chooser,
command_name=TEST_COMMAND,
)
assert test_result is not None, 'expect a result'
assert test_result == -1, 'expect failure, because of file naming'
assert os.path.exists(test_config.failure_fqn), 'expect failure file'
assert os.path.exists(test_config.retry_fqn), 'expect retry file'
def test_run_todo_list_dir_data_source(
read_obs_mock, fits2caom2_in_out_mock, fits2caom2_mock, test_config
):
read_obs_mock.side_effect = _mock_read
test_config.working_directory = tc.TEST_FILES_DIR
test_config.use_local_files = True
test_config.task_types = [mc.TaskType.SCRAPE]
test_chooser = ec.OrganizeChooser()
test_result = rc.run_by_todo(
config=test_config, chooser=test_chooser, command_name=TEST_COMMAND
)
assert test_result is not None, 'expect a result'
assert test_result == 0, 'expect success'
if fits2caom2_mock.called:
assert not fits2caom2_in_out_mock.called, 'expect no in/out call'
else:
assert fits2caom2_in_out_mock.called, 'expect fits2caom2 in/out call'
def test_run_todo_list_dir_data_source_exception(do_one_mock, clients_mock,
test_config):
test_config.working_directory = TEST_DIR
test_config.use_local_files = True
test_config.task_types = [mc.TaskType.SCRAPE]
test_config.log_to_file = True
for entry in [False, True]:
test_config.features.supports_latest_client = entry
do_one_mock.side_effect = mc.CadcException
if os.path.exists(test_config.failure_fqn):
os.unlink(test_config.failure_fqn)
if os.path.exists(test_config.retry_fqn):
os.unlink(test_config.retry_fqn)
test_chooser = ec.OrganizeChooser()
test_data_source = dsc.ListDirDataSource(test_config, test_chooser)
test_result = rc.run_by_todo(
config=test_config,
chooser=test_chooser,
command_name=TEST_COMMAND,
source=test_data_source,
)
assert test_result is not None, 'expect a result'
assert test_result == -1, 'expect failure'
assert do_one_mock.called, 'expect do_one call'
assert os.path.exists(test_config.failure_fqn), 'expect failure file'
assert os.path.exists(test_config.retry_fqn), 'expect retry file'
with open(test_config.failure_fqn) as f:
content = f.read()
# the obs id and file name
assert 'abc abc.fits' in content, 'wrong failure message'
with open(test_config.retry_fqn) as f:
content = f.read()
# retry file names
assert content == 'abc.fits\n', 'wrong retry content'
assert not (clients_mock.return_value.metadata_client.read.called
), 'scrape, should be no metadata client call'
assert not (clients_mock.return_value.data_client.get_file_info.called
), 'scrape, should be no data client call'
def test_run_todo_file_data_source(
caps_mock,
ad_mock,
data_client_mock,
set_clients_mock,
test_config
):
set_clients_mock.side_effect = _clients_mock
caps_mock.return_value = 'https://sc2.canfar.net/sc2repo'
response = Mock()
response.status_code = 200
response.iter_content.return_value = [b'fileName\n']
ad_mock.return_value.__enter__.return_value = response
data_client_mock.return_value = SimpleObservation(
collection=test_config.collection, observation_id='def',
algorithm=Algorithm(str('test'))
)
if os.path.exists(test_config.success_fqn):
os.unlink(test_config.success_fqn)
test_config.work_fqn = f'{TEST_DIR}/todo.txt'
test_config.task_types = [mc.TaskType.VISIT]
test_config.log_to_file = True
test_chooser = ec.OrganizeChooser()
test_result = rc.run_by_todo(
config=test_config, chooser=test_chooser, command_name=TEST_COMMAND
)
assert test_result is not None, 'expect a result'
assert test_result == 0, 'expect success'
assert os.path.exists(test_config.success_fqn), 'expect success file'
with open(test_config.success_fqn) as f:
content = f.read()
# the obs id and file name
assert 'def def.fits' in content, 'wrong success message'
def test_run_state(
fits2caom2_mock,
tap_query_mock,
tap_mock,
set_clients_mock,
repo_get_mock,
test_config
):
# tap mock is used by the data_source_composable class
set_clients_mock.side_effect = _clients_mock
fits2caom2_mock.side_effect = _mock_write
repo_get_mock.side_effect = Mock(return_value=None)
tap_query_mock.side_effect = _mock_get_work
test_end_time = datetime.fromtimestamp(1579740838, tz=timezone.utc)
start_time = test_end_time - timedelta(seconds=900)
_write_state(start_time)
test_config.task_types = [mc.TaskType.INGEST]
test_config.state_fqn = STATE_FILE
test_config.interval = 10
if os.path.exists(test_config.progress_fqn):
os.unlink(test_config.progress_fqn)
if os.path.exists(test_config.success_fqn):
os.unlink(test_config.success_fqn)
test_chooser = ec.OrganizeChooser()
test_result = rc.run_by_state(
config=test_config,
chooser=test_chooser,
command_name=TEST_COMMAND,
bookmark_name=TEST_BOOKMARK,
end_time=test_end_time,
)
assert test_result is not None, 'expect a result'
assert test_result == 0, 'expect success'
assert fits2caom2_mock.called, 'expect fits2caom2 call'
fits2caom2_mock.assert_called_once_with()
test_state = mc.State(STATE_FILE)
test_bookmark = test_state.get_bookmark(TEST_BOOKMARK)
assert test_bookmark == test_end_time, 'wrong time'
assert os.path.exists(test_config.progress_fqn), 'expect progress file'
assert (
not os.path.exists(test_config.success_fqn)
), 'log_to_file set to false, no success file'
# test that runner does nothing when times haven't changed
start_time = test_end_time
_write_state(start_time)
fits2caom2_mock.reset_mock()
test_result = rc.run_by_state(
config=test_config,
chooser=test_chooser,
command_name=TEST_COMMAND,
bookmark_name=TEST_BOOKMARK,
end_time=test_end_time,
)
assert test_result is not None, 'expect a result'
assert test_result == 0, 'expect success'
assert not fits2caom2_mock.called, 'expect no fits2caom2 call'
def test_run_state(
fits2caom2_mock,
fits2caom2_in_out_mock,
tap_query_mock,
tap_mock,
clients_mock,
test_config,
):
# tap_mock is used by the data_source_composable class
fits2caom2_mock.side_effect = _mock_write
clients_mock.return_value.metadata_client.read.side_effect = Mock(
return_value=None)
tap_query_mock.side_effect = _mock_get_work
test_end_time = datetime.fromtimestamp(1579740838, tz=timezone.utc)
start_time = test_end_time - timedelta(seconds=900)
_write_state(start_time)
test_config.task_types = [mc.TaskType.INGEST]
test_config.state_fqn = STATE_FILE
test_config.interval = 10
individual_log_file = (
f'{test_config.log_file_directory}/NEOS_SCI_2015347000000_clean.log')
if os.path.exists(test_config.progress_fqn):
os.unlink(test_config.progress_fqn)
if os.path.exists(test_config.success_fqn):
os.unlink(test_config.success_fqn)
if os.path.exists(individual_log_file):
os.unlink(individual_log_file)
test_chooser = ec.OrganizeChooser()
# use_local_files set so run_by_state chooses QueryTimeBoxDataSourceTS
test_config.use_local_files = False
test_result = rc.run_by_state(
config=test_config,
chooser=test_chooser,
command_name=TEST_COMMAND,
bookmark_name=TEST_BOOKMARK,
end_time=test_end_time,
)
assert test_result is not None, 'expect a result'
assert test_result == 0, 'expect success'
if fits2caom2_mock.called:
fits2caom2_mock.assert_called_once_with()
elif fits2caom2_in_out_mock.called:
fits2caom2_in_out_mock.assert_called_once_with(ANY)
test_state = mc.State(STATE_FILE)
test_bookmark = test_state.get_bookmark(TEST_BOOKMARK)
assert test_bookmark == test_end_time, 'wrong time'
assert os.path.exists(test_config.progress_fqn), 'expect progress file'
assert os.path.exists(
test_config.success_fqn), 'log_to_file set to false, no success file'
assert not os.path.exists(
individual_log_file), f'log_to_file is False, no entry log'
# test that runner does nothing when times haven't changed
start_time = test_end_time
_write_state(start_time)
fits2caom2_mock.reset_mock()
fits2caom2_in_out_mock.reset_mock()
test_result = rc.run_by_state(
config=test_config,
chooser=test_chooser,
command_name=TEST_COMMAND,
bookmark_name=TEST_BOOKMARK,
end_time=test_end_time,
)
assert test_result is not None, 'expect a result'
assert test_result == 0, 'expect success'
assert not fits2caom2_mock.called, 'expect no fits2caom2 call'
assert (
not fits2caom2_in_out_mock.called), 'expect no update fits2caom2 call'
def test_run_todo_list_dir_data_source_invalid_fname_v(clients_mock,
test_config):
test_dir = os.path.join('/test_files', '1')
test_fqn = os.path.join(test_dir, 'abc.fits.gz')
test_config.working_directory = tc.TEST_DATA_DIR
test_config.use_local_files = True
test_config.data_sources = [test_dir]
test_config.data_source_extensions = ['.fits', '.fits.gz']
test_config.task_types = [mc.TaskType.INGEST]
test_config.log_to_file = False
test_config.features.supports_latest_client = True
if os.path.exists(test_config.failure_fqn):
os.unlink(test_config.failure_fqn)
if os.path.exists(test_config.retry_fqn):
os.unlink(test_config.retry_fqn)
if not os.path.exists(test_dir):
os.mkdir(test_dir)
if not os.path.exists(test_fqn):
with open(test_fqn, 'w') as f:
f.write('abc')
class TestStorageName(mc.StorageName):
def __init__(self, entry):
self._obs_id = os.path.basename(entry)
self._source_names = [entry]
def is_valid(self):
return False
class TestStorageNameInstanceBuilder(b.StorageNameInstanceBuilder):
def __init__(self):
pass
def build(self, entry):
return TestStorageName(entry)
try:
test_builder = TestStorageNameInstanceBuilder()
test_chooser = ec.OrganizeChooser()
test_result = rc.run_by_todo(
config=test_config,
chooser=test_chooser,
name_builder=test_builder,
command_name=TEST_COMMAND,
)
assert test_result is not None, 'expect a result'
assert test_result == -1, 'expect failure, because of file naming'
assert os.path.exists(test_config.failure_fqn), 'expect failure file'
assert os.path.exists(test_config.retry_fqn), 'expect retry file'
assert (not clients_mock.metadata_client.read.called
), 'repo client read access happens after is_valid call'
assert not (clients_mock.data_client.get_file.called
), 'bad file naming, should be no client access'
finally:
if os.path.exists(test_fqn):
os.unlink(test_fqn)
if os.path.exists(test_dir):
logging.error(os.listdir(test_dir))
os.rmdir(test_dir)