def test_build_todo_good(query_endpoint_mock):
query_endpoint_mock.side_effect = _query_endpoint
start_time = scrape.make_date_time('24Apr2019 12:34')
test_result_list, test_result_date = scrape.build_good_todo(start_time,
session=Mock())
assert test_result_list is not None, 'expected list result'
assert test_result_date is not None, 'expected date result'
assert len(test_result_list) == 3, 'wrong number of results'
temp = test_result_list.popitem()
assert (temp[1][0] ==
'https://archive-new.nrao.edu/vlass/quicklook/VLASS1.2v2/T07t13/'
'VLASS1.2.ql.T07t13.J083838-153000.10.2048.v1/'), temp[1][0]
assert test_result_date == datetime(2019, 4, 28, 15,
18), 'wrong date result'
def read_file_url_list_from_nrao(nrao_state_fqn):
"""
:param nrao_state_fqn: str cache file name
:return: result dict key is file_name, value is timestamp from NRAO site
of file
validate_dict key is file_name, value is NRAO URL of file
"""
if os.path.exists(nrao_state_fqn):
vlass_list = mc.read_as_yaml(nrao_state_fqn)
else:
start_date = scrape.make_date_time('01Jan1990 00:00')
vlass_list = scrape.build_url_list(start_date)
mc.write_as_yaml(vlass_list, nrao_state_fqn)
result, validate_dict = get_file_url_list_max_versions(vlass_list)
return result, validate_dict
def read_list_from_nrao(nrao_state_fqn):
if os.path.exists(nrao_state_fqn):
vlass_list = mc.read_as_yaml(nrao_state_fqn)
else:
start_date = scrape.make_date_time('01Jan1990 00:00')
vlass_list, vlass_date = scrape.build_file_url_list(start_date)
mc.write_as_yaml(vlass_list, nrao_state_fqn)
result = {}
validate_dict = {}
for key, value in vlass_list.items():
for url in value:
f_name = url.split('/')[-1]
result[f_name] = key
validate_dict[f_name] = url
return result, validate_dict
def _write_state(start_time_str):
test_time = scrape.make_date_time(start_time_str)
test_bookmark = {
'bookmarks': {
'vlass_timestamp': {
'last_record': test_time
}
},
'context': {
'vlass_context': {
'VLASS1.1': '01-Jan-2018 00:00',
'VLASS1.2': '01-Nov-2018 00:00',
'VLASS2.1': '01-Jul-2020 00:00'
}
}
}
mc.write_as_yaml(test_bookmark, STATE_FILE)
THIS_DIR = os.path.dirname(os.path.realpath(__file__))
TEST_DATA_DIR = os.path.join(THIS_DIR, 'data')
ALL_FIELDS = os.path.join(TEST_DATA_DIR, 'all_fields_list.html')
CAOM_QUERY = os.path.join(TEST_DATA_DIR, 'caom_query.csv')
CROSS_EPOCH = os.path.join(TEST_DATA_DIR, 'cross_epoch.html')
SINGLE_TILE = os.path.join(TEST_DATA_DIR, 'single_tile_list.html')
QL_INDEX = os.path.join(TEST_DATA_DIR, 'vlass_quicklook.html')
WL_INDEX = os.path.join(TEST_DATA_DIR, 'weblog_quicklook.html')
PIPELINE_INDEX = os.path.join(TEST_DATA_DIR, 'pipeline_weblog_quicklook.htm')
SINGLE_FIELD_DETAIL = os.path.join(TEST_DATA_DIR, 'single_field_detail.html')
REJECT_INDEX = os.path.join(TEST_DATA_DIR, 'rejected_index.html')
SPECIFIC_REJECTED = os.path.join(TEST_DATA_DIR, 'specific_rejected.html')
SPECIFIC_NO_FILES = os.path.join(TEST_DATA_DIR, 'no_files.html')
TEST_START_TIME_STR = '24Apr2019 12:34'
TEST_START_TIME = scrape.make_date_time(TEST_START_TIME_STR)
STATE_FILE = os.path.join(TEST_DATA_DIR, 'state.yml')
TEST_OBS_ID = 'VLASS1.2.T07t14.J084202-123000'
class MyExitError(Exception):
pass
# Response mock
class Object(object):
pass
def close(self):
pass