def test_is_time_in_spc_date_after(self):
"""Ensures correct output from is_time_in_spc_date.
In this case, time is after SPC date.
"""
self.assertFalse(
time_conversion.is_time_in_spc_date(
TIME_1200UTC_AFTER_DATE_UNIX_SEC, SPC_DATE_STRING))
def test_is_time_in_spc_date_end(self):
"""Ensures correct output from is_time_in_spc_date.
In this case, time is at end of SPC date.
"""
self.assertTrue(
time_conversion.is_time_in_spc_date(
TIME_115959UTC_SPC_DATE_UNIX_SEC, SPC_DATE_STRING))
def test_is_time_in_spc_date_before(self):
"""Ensures correct output from is_time_in_spc_date.
In this case, time is before SPC date.
"""
self.assertFalse(
time_conversion.is_time_in_spc_date(
TIME_115959UTC_BEFORE_DATE_UNIX_SEC, SPC_DATE_STRING))
def test_is_time_in_spc_date_middle(self):
"""Ensures correct output from is_time_in_spc_date.
In this case, time is in middle of SPC date.
"""
self.assertTrue(
time_conversion.is_time_in_spc_date(TIME_0000UTC_SPC_DATE_UNIX_SEC,
SPC_DATE_STRING))
def find_files_for_smart_io(start_time_unix_sec=None,
start_spc_date_string=None,
end_time_unix_sec=None,
end_spc_date_string=None,
data_source=None,
tracking_scale_metres2=None,
top_input_dir_name=None,
top_output_dir_name=None):
"""Finds input, output, and temporary working files for smart IO.
N = number of SPC dates in period
T_i = number of time steps in the [i]th SPC date
:param start_time_unix_sec: Beginning of time period.
:param start_spc_date_string: SPC date at beginning of time period (format
"yyyymmdd").
:param end_time_unix_sec: End of time period.
:param end_spc_date_string: SPC date at end of time period (format
"yyyymmdd").
:param data_source: Source for input data (examples: "segmotion",
"probSevere").
:param tracking_scale_metres2: Tracking scale.
:param top_input_dir_name: Name of top-level directory for input files.
:param top_output_dir_name: Name of top-level directory for output files.
:return: file_dict: Dictionary with the following keys.
file_dict.spc_dates_unix_sec: length-N numpy array of SPC dates.
file_dict.temp_file_names: 1-D list of paths to temp files (will be used for
intermediate IO).
file_dict.input_file_names_by_spc_date: length-N list, where the [i]th
element is a 1-D list (length T_i) of paths to input files.
file_dict.output_file_names_by_spc_date: Same but for output files.
:raises: ValueError: if start_time_unix_sec is not part of the first SPC
date (determined by start_spc_date_unix_sec).
:raises: ValueError: if end_time_unix_sec is not part of the last SPC date
(determined by end_spc_date_unix_sec).
"""
if not time_conversion.is_time_in_spc_date(start_time_unix_sec,
start_spc_date_string):
start_time_string = time_conversion.unix_sec_to_string(
start_time_unix_sec, TIME_FORMAT_FOR_MESSAGES)
raise ValueError('Start time (' + start_time_string +
') is not in first SPC date (' +
start_spc_date_string + ').')
if not time_conversion.is_time_in_spc_date(end_time_unix_sec,
end_spc_date_string):
end_time_string = time_conversion.unix_sec_to_string(
end_time_unix_sec, TIME_FORMAT_FOR_MESSAGES)
raise ValueError('End time (' + end_time_string +
') is not in last SPC date (' + end_spc_date_string +
').')
error_checking.assert_is_greater(end_time_unix_sec, start_time_unix_sec)
start_spc_date_unix_sec = time_conversion.spc_date_string_to_unix_sec(
start_spc_date_string)
end_spc_date_unix_sec = time_conversion.spc_date_string_to_unix_sec(
end_spc_date_string)
num_spc_dates = int(1 + (end_spc_date_unix_sec - start_spc_date_unix_sec) /
DAYS_TO_SECONDS)
spc_dates_unix_sec = numpy.linspace(start_spc_date_unix_sec,
end_spc_date_unix_sec,
num=num_spc_dates,
dtype=int)
temp_file_names = [''] * num_spc_dates
input_file_names_by_spc_date = [['']] * num_spc_dates
output_file_names_by_spc_date = [['']] * num_spc_dates
for i in range(num_spc_dates):
spc_dates_unix_sec[i] = time_conversion.time_to_spc_date_unix_sec(
spc_dates_unix_sec[i])
temp_file_names[i] = tempfile.NamedTemporaryFile(delete=False).name
input_file_names_by_spc_date[i] = (
tracking_io.find_processed_files_one_spc_date(
spc_dates_unix_sec[i],
data_source=data_source,
top_processed_dir_name=top_input_dir_name,
tracking_scale_metres2=tracking_scale_metres2,
raise_error_if_missing=True))
this_num_files = len(input_file_names_by_spc_date[i])
these_times_unix_sec = numpy.full(this_num_files, -1, dtype=int)
output_file_names_by_spc_date[i] = [''] * this_num_files
for j in range(this_num_files):
these_times_unix_sec[j] = tracking_io.processed_file_name_to_time(
input_file_names_by_spc_date[i][j])
output_file_names_by_spc_date[i][j] = (
tracking_io.find_processed_file(
unix_time_sec=these_times_unix_sec[j],
data_source=data_source,
spc_date_unix_sec=spc_dates_unix_sec[i],
top_processed_dir_name=top_output_dir_name,
tracking_scale_metres2=tracking_scale_metres2,
raise_error_if_missing=False))
if i == 0:
keep_time_indices = numpy.where(
these_times_unix_sec >= start_time_unix_sec)[0]
these_times_unix_sec = these_times_unix_sec[keep_time_indices]
input_file_names_by_spc_date[i] = [
input_file_names_by_spc_date[i][j] for j in keep_time_indices
]
output_file_names_by_spc_date[i] = [
output_file_names_by_spc_date[i][j] for j in keep_time_indices
]
if i == num_spc_dates - 1:
keep_time_indices = numpy.where(
these_times_unix_sec <= end_time_unix_sec)[0]
input_file_names_by_spc_date[i] = [
input_file_names_by_spc_date[i][j] for j in keep_time_indices
]
output_file_names_by_spc_date[i] = [
output_file_names_by_spc_date[i][j] for j in keep_time_indices
]
return {
SPC_DATES_KEY: spc_dates_unix_sec,
TEMP_FILE_NAMES_KEY: temp_file_names,
INPUT_FILE_NAMES_KEY: input_file_names_by_spc_date,
OUTPUT_FILE_NAMES_KEY: output_file_names_by_spc_date
}
