def test_build_todo_good(query_endpoint_mock): query_endpoint_mock.side_effect = _query_endpoint start_time = scrape.make_date_time('24Apr2019 12:34') test_result_list, test_result_date = scrape.build_good_todo(start_time, session=Mock()) assert test_result_list is not None, 'expected list result' assert test_result_date is not None, 'expected date result' assert len(test_result_list) == 3, 'wrong number of results' temp = test_result_list.popitem() assert (temp[1][0] == 'https://archive-new.nrao.edu/vlass/quicklook/VLASS1.2v2/T07t13/' 'VLASS1.2.ql.T07t13.J083838-153000.10.2048.v1/'), temp[1][0] assert test_result_date == datetime(2019, 4, 28, 15, 18), 'wrong date result'
def read_file_url_list_from_nrao(nrao_state_fqn): """ :param nrao_state_fqn: str cache file name :return: result dict key is file_name, value is timestamp from NRAO site of file validate_dict key is file_name, value is NRAO URL of file """ if os.path.exists(nrao_state_fqn): vlass_list = mc.read_as_yaml(nrao_state_fqn) else: start_date = scrape.make_date_time('01Jan1990 00:00') vlass_list = scrape.build_url_list(start_date) mc.write_as_yaml(vlass_list, nrao_state_fqn) result, validate_dict = get_file_url_list_max_versions(vlass_list) return result, validate_dict
def read_list_from_nrao(nrao_state_fqn): if os.path.exists(nrao_state_fqn): vlass_list = mc.read_as_yaml(nrao_state_fqn) else: start_date = scrape.make_date_time('01Jan1990 00:00') vlass_list, vlass_date = scrape.build_file_url_list(start_date) mc.write_as_yaml(vlass_list, nrao_state_fqn) result = {} validate_dict = {} for key, value in vlass_list.items(): for url in value: f_name = url.split('/')[-1] result[f_name] = key validate_dict[f_name] = url return result, validate_dict
def _write_state(start_time_str): test_time = scrape.make_date_time(start_time_str) test_bookmark = { 'bookmarks': { 'vlass_timestamp': { 'last_record': test_time } }, 'context': { 'vlass_context': { 'VLASS1.1': '01-Jan-2018 00:00', 'VLASS1.2': '01-Nov-2018 00:00', 'VLASS2.1': '01-Jul-2020 00:00' } } } mc.write_as_yaml(test_bookmark, STATE_FILE)
THIS_DIR = os.path.dirname(os.path.realpath(__file__)) TEST_DATA_DIR = os.path.join(THIS_DIR, 'data') ALL_FIELDS = os.path.join(TEST_DATA_DIR, 'all_fields_list.html') CAOM_QUERY = os.path.join(TEST_DATA_DIR, 'caom_query.csv') CROSS_EPOCH = os.path.join(TEST_DATA_DIR, 'cross_epoch.html') SINGLE_TILE = os.path.join(TEST_DATA_DIR, 'single_tile_list.html') QL_INDEX = os.path.join(TEST_DATA_DIR, 'vlass_quicklook.html') WL_INDEX = os.path.join(TEST_DATA_DIR, 'weblog_quicklook.html') PIPELINE_INDEX = os.path.join(TEST_DATA_DIR, 'pipeline_weblog_quicklook.htm') SINGLE_FIELD_DETAIL = os.path.join(TEST_DATA_DIR, 'single_field_detail.html') REJECT_INDEX = os.path.join(TEST_DATA_DIR, 'rejected_index.html') SPECIFIC_REJECTED = os.path.join(TEST_DATA_DIR, 'specific_rejected.html') SPECIFIC_NO_FILES = os.path.join(TEST_DATA_DIR, 'no_files.html') TEST_START_TIME_STR = '24Apr2019 12:34' TEST_START_TIME = scrape.make_date_time(TEST_START_TIME_STR) STATE_FILE = os.path.join(TEST_DATA_DIR, 'state.yml') TEST_OBS_ID = 'VLASS1.2.T07t14.J084202-123000' class MyExitError(Exception): pass # Response mock class Object(object): pass def close(self): pass