def test_is_time_in_spc_date_after(self): """Ensures correct output from is_time_in_spc_date. In this case, time is after SPC date. """ self.assertFalse( time_conversion.is_time_in_spc_date( TIME_1200UTC_AFTER_DATE_UNIX_SEC, SPC_DATE_STRING))
def test_is_time_in_spc_date_end(self): """Ensures correct output from is_time_in_spc_date. In this case, time is at end of SPC date. """ self.assertTrue( time_conversion.is_time_in_spc_date( TIME_115959UTC_SPC_DATE_UNIX_SEC, SPC_DATE_STRING))
def test_is_time_in_spc_date_before(self): """Ensures correct output from is_time_in_spc_date. In this case, time is before SPC date. """ self.assertFalse( time_conversion.is_time_in_spc_date( TIME_115959UTC_BEFORE_DATE_UNIX_SEC, SPC_DATE_STRING))
def test_is_time_in_spc_date_middle(self): """Ensures correct output from is_time_in_spc_date. In this case, time is in middle of SPC date. """ self.assertTrue( time_conversion.is_time_in_spc_date(TIME_0000UTC_SPC_DATE_UNIX_SEC, SPC_DATE_STRING))
def find_files_for_smart_io(start_time_unix_sec=None, start_spc_date_string=None, end_time_unix_sec=None, end_spc_date_string=None, data_source=None, tracking_scale_metres2=None, top_input_dir_name=None, top_output_dir_name=None): """Finds input, output, and temporary working files for smart IO. N = number of SPC dates in period T_i = number of time steps in the [i]th SPC date :param start_time_unix_sec: Beginning of time period. :param start_spc_date_string: SPC date at beginning of time period (format "yyyymmdd"). :param end_time_unix_sec: End of time period. :param end_spc_date_string: SPC date at end of time period (format "yyyymmdd"). :param data_source: Source for input data (examples: "segmotion", "probSevere"). :param tracking_scale_metres2: Tracking scale. :param top_input_dir_name: Name of top-level directory for input files. :param top_output_dir_name: Name of top-level directory for output files. :return: file_dict: Dictionary with the following keys. file_dict.spc_dates_unix_sec: length-N numpy array of SPC dates. file_dict.temp_file_names: 1-D list of paths to temp files (will be used for intermediate IO). file_dict.input_file_names_by_spc_date: length-N list, where the [i]th element is a 1-D list (length T_i) of paths to input files. file_dict.output_file_names_by_spc_date: Same but for output files. :raises: ValueError: if start_time_unix_sec is not part of the first SPC date (determined by start_spc_date_unix_sec). :raises: ValueError: if end_time_unix_sec is not part of the last SPC date (determined by end_spc_date_unix_sec). """ if not time_conversion.is_time_in_spc_date(start_time_unix_sec, start_spc_date_string): start_time_string = time_conversion.unix_sec_to_string( start_time_unix_sec, TIME_FORMAT_FOR_MESSAGES) raise ValueError('Start time (' + start_time_string + ') is not in first SPC date (' + start_spc_date_string + ').') if not time_conversion.is_time_in_spc_date(end_time_unix_sec, end_spc_date_string): end_time_string = time_conversion.unix_sec_to_string( end_time_unix_sec, TIME_FORMAT_FOR_MESSAGES) raise ValueError('End time (' + end_time_string + ') is not in last SPC date (' + end_spc_date_string + ').') error_checking.assert_is_greater(end_time_unix_sec, start_time_unix_sec) start_spc_date_unix_sec = time_conversion.spc_date_string_to_unix_sec( start_spc_date_string) end_spc_date_unix_sec = time_conversion.spc_date_string_to_unix_sec( end_spc_date_string) num_spc_dates = int(1 + (end_spc_date_unix_sec - start_spc_date_unix_sec) / DAYS_TO_SECONDS) spc_dates_unix_sec = numpy.linspace(start_spc_date_unix_sec, end_spc_date_unix_sec, num=num_spc_dates, dtype=int) temp_file_names = [''] * num_spc_dates input_file_names_by_spc_date = [['']] * num_spc_dates output_file_names_by_spc_date = [['']] * num_spc_dates for i in range(num_spc_dates): spc_dates_unix_sec[i] = time_conversion.time_to_spc_date_unix_sec( spc_dates_unix_sec[i]) temp_file_names[i] = tempfile.NamedTemporaryFile(delete=False).name input_file_names_by_spc_date[i] = ( tracking_io.find_processed_files_one_spc_date( spc_dates_unix_sec[i], data_source=data_source, top_processed_dir_name=top_input_dir_name, tracking_scale_metres2=tracking_scale_metres2, raise_error_if_missing=True)) this_num_files = len(input_file_names_by_spc_date[i]) these_times_unix_sec = numpy.full(this_num_files, -1, dtype=int) output_file_names_by_spc_date[i] = [''] * this_num_files for j in range(this_num_files): these_times_unix_sec[j] = tracking_io.processed_file_name_to_time( input_file_names_by_spc_date[i][j]) output_file_names_by_spc_date[i][j] = ( tracking_io.find_processed_file( unix_time_sec=these_times_unix_sec[j], data_source=data_source, spc_date_unix_sec=spc_dates_unix_sec[i], top_processed_dir_name=top_output_dir_name, tracking_scale_metres2=tracking_scale_metres2, raise_error_if_missing=False)) if i == 0: keep_time_indices = numpy.where( these_times_unix_sec >= start_time_unix_sec)[0] these_times_unix_sec = these_times_unix_sec[keep_time_indices] input_file_names_by_spc_date[i] = [ input_file_names_by_spc_date[i][j] for j in keep_time_indices ] output_file_names_by_spc_date[i] = [ output_file_names_by_spc_date[i][j] for j in keep_time_indices ] if i == num_spc_dates - 1: keep_time_indices = numpy.where( these_times_unix_sec <= end_time_unix_sec)[0] input_file_names_by_spc_date[i] = [ input_file_names_by_spc_date[i][j] for j in keep_time_indices ] output_file_names_by_spc_date[i] = [ output_file_names_by_spc_date[i][j] for j in keep_time_indices ] return { SPC_DATES_KEY: spc_dates_unix_sec, TEMP_FILE_NAMES_KEY: temp_file_names, INPUT_FILE_NAMES_KEY: input_file_names_by_spc_date, OUTPUT_FILE_NAMES_KEY: output_file_names_by_spc_date }