def to_array(self, ts, start_date=None, end_date=None, month_index=None, include_months=None, include_missing=True,
match_other_nonmissing=None, paired_ts=None, return_type=None):
"""
Return an array containing the data values of the time series for the specified
period. If the start date or end date are outside the period of
record for the time series, use the missing data value from the time series
for those values. If the start date or end date are null, the start and end
dates of the time series are used. This is a utility routine mainly used by other versions of this routine.
@return The array of data for the time series. If an error, return null.
@param ts Time series to convert data to array format.
@param start_date Date corresponding to the first date of the returned array.
@param end_date Date corresponding to the last date of the returned array.
@param month_index Month of interest (1=Jan, 12=Dec). If zero, process all months.
@param include_months
@param include_missing indicate whether missing values should be included in the result.
@param match_other_nonmissing
@param paired_ts
@param return_type
"""
if month_index == None:
# Called with no month index
month_indices = None
if month_index != 0:
month_indices = [month_index]
# Recursively call
return self.to_array(ts, start_date=start_date, end_date=end_date, include_months=include_months,
include_missing=include_missing)
# If here do the processing based on input arguments
if paired_ts is not None:
if not TimeInterval.is_regular_interval(ts.get_data_interval_base()):
# throw new IrregularTimeSeriesNotSupportedException(
raise ValueError(
"Irregular interval time series cannot have data array extracted using paired time series.")
if not self.intervals_match(ts, paired_ts):
# throw new UnequalTimeIntervalException(
raise ValueError(
"Time series from which to extract data has a different interval than paired time series.")
# Get valid dates because the ones passed in may have been null...
valid_dates = self.get_valid_period(ts, start_date, end_date)
start = valid_dates.get_date1()
end = valid_dates.get_date2()
interval_base = ts.get_data_interval_base()
interval_mult = ts.get_data_interval_mult()
size = 0
# if ts.get_data_interval_base() == TimeInterval.IRREGULAR:
# size = self.calculate_data_size(ts, start, end)
# else:
size = self.calculate_data_size(start, end, interval_base, interval_mult)
if return_type is None:
return_type = TSToArrayReturnType.DATA_VALUE
if return_type == TSToArrayReturnType.DATE_TIME:
# Only 1Year, 1Month, 1Day intervals are supported
if (interval_mult != 1) or ((interval_base != TimeInterval.YEAR) and
(interval_base != TimeInterval.YEAR) and (interval_base != TimeInterval.YEAR)):
# throw new InvalidTimeIntervalException(
raise ValueError(
"Interval must be Year, Month, or Day (no multiplier) to return date/time as array.")
include_months_mask = []
if (include_months is None) or (len(include_months) == 0):
for i in range(12):
include_months_mask[i] = True
else:
for i in range(12):
include_months_mask[i] = False
for i in range(len(include_months)):
include_months_mask[include_months[i] - 1] = True
if size == 0:
return []
data_array = [] # Initial size including missing
count = 0 # Number of values in array.
month = 0 # Month
if interval_base == TimeInterval.IRREGULAR:
# Get the data and loop through the vector...
irrts = ts
alltsdata = irrts.get_data()
if alltsdata is None:
# No data for the time series...
return None
nalltsdata = len(alltsdata)
tsdata = None
date = None
for i in range(nalltsdata):
tsdata = alltsdata[i]
date = tsdata.get_date()
if date.greater_than(end):
# Past the end of where we want to go so quit...
break
if date.greater_than_or_equal_to(start):
month = date.get_month()
if include_months_mask[month - 1]:
value = tsdata.get_data_value()
if include_missing or not ts.is_data_missing(value):
if return_type == TSToArrayReturnType.DATA_VALUE:
data_array[count] = value
count += 1
elif return_type == TSToArrayReturnType.DATE_TIME:
if interval_base == TimeInterval.YEAR:
data_array[count] = date.get_year()
count += 1
elif interval_base == TimeInterval.MONTH:
data_array[count] = date.get_absolute_month()
count += 1
elif interval_base == TimeInterval.DAY:
data_array[count] = date.get_absolute_day()
count += 1
else:
# Regular, increment the data by interval...
date = DateTime(date_time=start)
count = 0
do_transfer = False
is_missing = False
# for ; date.lessThanOrEqualTo( end); date.addInterval(interval_base, interval_mult):
first_iteration = True
while date.less_than_or_equal_to(end):
if first_iteration:
first_iteration = False
else:
date.add_interval(interval_base, interval_mult)
# First figure out if the data should be skipped because not in a requested month
month = date.get_month()
if not include_months_mask[month - 1]:
continue
# Now transfer the value while checking the paired time series
do_transfer = False # Do not transfer unless criteria are met below
value = ts.get_data_value(date)
is_missing = ts.is_data_missing(value)
if paired_ts is not None:
# Value in "ts" time series MUST be non-missing
if not is_missing:
value2 = paired_ts.get_data_value(date)
is_missing2 = paired_ts.is_data_missing(value2)
if match_other_nonmissing:
# Want non-missing in both "ts" and "pairedTS"
if not is_missing2:
do_transfer = True
else:
# Want non-missing in "ts" and missing in "pairedTS"
if is_missing2:
do_transfer = True
else:
if include_missing or not is_missing:
# Value is not missing.
do_transfer = True
# OK to transfer the value...
if do_transfer:
if return_type == TSToArrayReturnType.DATA_VALUE:
data_array[count] = value
count += 1
elif return_type == TSToArrayReturnType.DATE_TIME:
if interval_base == TimeInterval.YEAR:
data_array[count] = date.get_year()
count += 1
elif interval_base == TimeInterval.MONTH:
data_array[count] = date.get_absolute_month()
count += 1
elif interval_base == TimeInterval.DAY:
# TODO smalers 2020-01-04 need to enable
# data_array[count] = date.get_absolute_day()
count += 1
if count != size:
# The original array is too big and needs to be cut down to the exact size due to limited
# months or missing data being excluded)...
new_data_array = [count]
for j in range(count):
new_data_array[j] = data_array[j]
return new_data_array
# Return the full array...
return data_array
def calculate_data_limits(self, ts, start0, end0, refresh_flag):
"""
Calculate the total data limits for a time series between two dates.
This code was taken from the TSUtil.getDataLimits method.
@param ts Time series of interest.
@param start0 Starting date for the check.
@param end0 Ending date for the check.
@param refresh_flag Indicates whether the time series should be refreshed first
(in general this is used only within the TS package and the version of this
routine without the flag should be called).
"""
max = 1.0
mean = 0.0
min = 0.0
sum = 0.0
value = 0.0
base = 0
missing_count = 0
mult = 0
non_missing_count = 0
found = False
max_date = None
min_date = None
non_missing_data_date1 = None
non_missing_data_date2 = None
t = None
logger = logging.getLogger(__name__)
debug = False
try:
# Main try...
if ts is None:
message = "NULL time series"
logger.warning(message)
# throw new TSException ( message )
raise ValueError(message)
# Initialize the sum and the mean...
missing = ts.get_missing()
sum = missing
mean = missing
# Get valid date limits because the ones passed in may have been null...
valid_dates = self.get_valid_period(ts, start0, end0)
start = valid_dates.get_date1()
end = valid_dates.get_date2()
valid_dates = None
# Make sure that the time series has current limits...
base = ts.get_data_interval_base()
mult = ts.get_data_interval_mult()
if refresh_flag:
# Force a refresh of the time series.
ts.refresh()
# Get the variables that are used often in this function.
ts_date1 = ts.get_date1()
ts_date2 = ts.get_date2()
# Figure out if we are treating data <= 0 as missing...
ignore_lezero = False
if (self.flags & TSLimits.IGNORE_LESS_THAN_OR_EQUAL_ZERO) != 0:
ignore_lezero = True
# Loop through the dates and get max and min data values
# TODO SAM 2010-06-15 Need to consolidate code to use iterator
if base == TimeInterval.IRREGULAR:
# Loop through the dates and get max and min data values
# Need to cast as an irregular TS...
# IrregularTS its = (IrregularTS)ts
its = ts
data_array = its.get_data
if data_array is None:
message = "Null data for " + str(ts)
logger.warning(message)
# throw new TSException ( message )
raise ValueError(message)
size = len(data_array)
ptr = None
for i in range(size):
ptr = data_array[i]
date = ptr.get_date()
if date.less_than(ts_date1):
# Still looking for data...
continue
elif date.greater_than(ts_date2):
# No need to continue processing...
break
value = ptr.get_data_value()
if ts.is_data_missing(value) or (ignore_lezero and (value <= 0.0)):
# The value is missing
missing_count += 1
continue
# Else, data value is not missing...
if ts.is_data_missing(sum):
# Reset the sum...
sum = value
else:
# Add to the sum...
sum += value
non_missing_count += 1
if found:
# Already found the first non-missing point so
# all we need to do is check the limits. These
# should only result in new DateTime a few times...
if value > max:
max = value
max_date = DateTime(date_time=date)
if value < min:
min = value
min_date = DateTime(date_time=date)
else:
# Set the limits to the first value found...
# date = new DateTime ( t )
max = value
max_date = DateTime(date_time=date)
min = value
min_date = max_date
non_missing_data_date1 = max_date
non_missing_data_date2 = max_date
found = True
continue
# Now search backwards to find the first non-missing date...
if found:
for i in range((size - 1), 0, -1):
ptr = data_array[i]
date = ptr.get_date()
value = ptr.get_data_value()
if date.greater_than(end):
# Have not found data...
continue
elif date.less_than(start):
# Passed start...
break
if (not ignore_lezero and not ts.is_data_missing(value)) or \
(ignore_lezero and ((value > 0.0) and not ts.is_data_missing(value))):
# Found the one date we are after...
non_missing_data_date2 = DateTime(date_time=date)
break
else:
# A regular TS... easier to iterate...
# First loop through and find the data limits and the minimum non-missing date...
t = DateTime(date_time=start, flag=DateTime.DATE_FAST)
# Python for loops are not as clean as original Java code
# for ( ; t.lessThanOrEqualTo(end); t.addInterval( base, mult )) {
first_iteration = True
while t.less_than_or_equal_to(end):
if first_iteration:
first_iteration = False
else:
t.add_interval(base, mult)
value = ts.get_data_value(t)
if ts.is_data_missing(value) or (ignore_lezero and (value <= 0.0)):
# The value is missing
missing_count += 1
continue
# Else, data value is not missing...
if ts.is_data_missing(sum):
# Reset the sum...
sum = value
else:
# Add to the sum...
sum += value
non_missing_count += 1
if found:
# Already found the first non-missing point so
# all we need to do is check the limits. These
# should only result in new DateTime a few times...
if value > max:
max = value
max_date = DateTime(date_time=t)
if value < min:
min = value
min_date = DateTime(date_time=t)
else:
# First non-missing point so set the initial values...
date = DateTime(date_time=t)
max = value
max_date = date
min = value
min_date = date
non_missing_data_date1 = date
non_missing_data_date2 = date
found = True
# Now loop backwards and find the last non-missing value...
t = DateTime(date_time=end, flag=DateTime.DATE_FAST)
if found:
# for(; t.greaterThanOrEqualTo(start); t.addInterval( base, -mult )) {
first_iteration = True
while t.greater_than_or_equal_to(start):
if first_iteration:
first_iteration = False
else:
t.add_interval(base, -mult)
value = ts.get_data_value(t)
if (not ignore_lezero and not ts.is_data_missing(value)) or \
(ignore_lezero and ((value > 0.0) and not ts.is_data_missing(value))):
# The value is not missing...
non_missing_data_date2 = DateTime(date_time=t)
break
# TODO SAM 2010-06-15 This is a performance hit, but not too bad
# TODO SAM 2010-06-15 Consider treating other statistics similarly but need to define unit tests
# TODO SAM 2010-06-15 This code would need to be changed if doing Lag-1 correlation because order matters
# For newly added statistics, use helper method to get data, ignoring missing...
data_array = self.to_array(ts, start, end, 0, False)
# Check for <= 0 values if necessary
n_data_array = len(data_array)
if ignore_lezero:
for i in range(n_data_array):
if data_array[i] <= 0.0:
# Just exchange with the last value and reduce the size
temp = data_array[i]
data_array[i] = data_array[n_data_array - 1]
data_array[n_data_array - 1] = temp
n_data_array -= 1
if n_data_array > 0:
self.set_median(MathUtil.median(n_data_array, data_array))
if n_data_array > 1:
try:
self.set_std_dev(MathUtil.standard_deviation(n_data_array, data_array))
except Exception as e:
# Likely due to small sample size
pass
if n_data_array > 2:
try:
self.set_skew(MathUtil.skew(n_data_array, data_array))
except Exception as e:
# Likely due to small sample size
pass
if not found:
message = "\"" + ts.getIdentifierString() + "\": problems finding limits, whole POR missing!"
logger.warning(message)
# throw new TSException ( message )
raise ValueError(message)
if debug:
logger.debug("Overall date limits are: " + str(start) + " to " + str(end))
logger.debug("Found limits to be: " + str(min) + " on " + str(min_date) + " to " + str(max) +
" on " + str(max_date))
logger.debug("Found non-missing data dates to be: " + str(non_missing_data_date1) + " -> " +
str(non_missing_data_date2))
# Set the basic information...
self.set_date1(start)
self.set_date2(end)
self.set_max_value(max, max_date)
self.set_min_value(min, min_date)
self.set_non_missing_data_date1(non_missing_data_date1)
self.set_non_missing_data_date2(non_missing_data_date2)
self.set_missing_data_count(missing_count)
self.set_non_missing_data_count(non_missing_count)
# //int data_size = calculate_data_size(ts, start, end)
# //limits.set_non_missing_data_count(data_size - missing_count)
if not ts.is_data_missing(sum) and (non_missing_count > 0):
mean = sum/float(non_missing_count)
else:
mean = missing
self.set_sum(sum)
self.set_mean(mean)
except Exception as e:
message = "Error computing limits."
logger.warning(message)
# Put in debug because output sometimes is overwhelming when data are not available.
if debug:
logger.warning(e)
# throw new TSException ( message )
raise Exception(message)