def calculate_fallback_dt(hdf, fallback_dt=None, fallback_relative_to_start=True):
"""
Check the time parameters in the HDF5 file and update the fallback_dt.
Takes into account adjustment of fallback datetime if it's relative to
the end of data and if there's a constant timebase within the data it
will use this as the fallback datetime.
"""
if fallback_dt and not fallback_relative_to_start:
# fallback_dt is relative to the end of the data; remove the data
# duration to make it relative to the start of the data
secs = hdf.duration
fallback_dt -= timedelta(seconds=secs) # Q: minus the number of segments * gaps between them? # ADDRESS!
logger.info("Reduced fallback_dt by %ddays %dhr %dmin to %s",
secs // 86400, secs % 86400 // 3600,
secs % 86400 % 3600 // 60, fallback_dt)
if not has_constant_time(hdf):
# we don't need to do any further corrections
return fallback_dt
try:
timebase = calculate_timebase(*get_dt_arrays(hdf, fallback_dt))
except (KeyError, ValueError):
# The time parameters are not available/operational
return fallback_dt
else:
logger.warning("Time doesn't change, using the starting time as the fallback_dt")
return timebase
def calculate_fallback_dt(hdf,
fallback_dt=None,
validation_dt=None,
fallback_relative_to_start=True):
"""
Check the time parameters in the HDF5 file and update the fallback_dt.
Takes into account adjustment of fallback datetime if it's relative to
the end of data and if there's a constant timebase within the data it
will use this as the fallback datetime.
"""
if fallback_dt and not fallback_relative_to_start:
# fallback_dt is relative to the end of the data; remove the data
# duration to make it relative to the start of the data
secs = hdf.duration
fallback_dt -= timedelta(seconds=secs)
logger.info("Reduced fallback_dt by %ddays %dhr %dmin to %s",
secs // 86400, secs % 86400 // 3600,
secs % 86400 % 3600 // 60, fallback_dt)
if not has_constant_time(hdf):
# we don't need to do any further corrections
return fallback_dt
try:
timebase = calculate_timebase(
*get_dt_arrays(hdf, fallback_dt, validation_dt))
except (KeyError, ValueError):
# The time parameters are not available/operational
return fallback_dt
else:
logger.warning(
"Time doesn't change, using the starting time as the fallback_dt")
return timebase
def _calculate_start_datetime(hdf, fallback_dt):
"""
Calculate start datetime.
:param hdf: Flight data HDF file
:type hdf: hdf_access object
:param fallback_dt: Used to replace elements of datetimes which are not
available in the hdf file (e.g. YEAR not being recorded)
:type fallback_dt: datetime
HDF params used:
:Year: Optional (defaults to 1970)
:Month: Optional (defaults to 1)
:Day: Optional (defaults to 1)
:Hour: Required
:Minute: Required
:Second: Required
If required parameters are not available and fallback_dt is not provided,
a TimebaseError is raised
"""
now = datetime.utcnow().replace(tzinfo=pytz.utc)
if fallback_dt is not None:
if (fallback_dt.tzinfo is None or
fallback_dt.tzinfo.utcoffset(fallback_dt) is None):
# Assume fallback_dt is UTC.
fallback_dt = fallback_dt.replace(tzinfo=pytz.utc)
assert fallback_dt < now, (
"Fallback time '%s' in the future is not allowed. Current time "
"is '%s'." % (fallback_dt, now))
# align required parameters to 1Hz
dt_arrays = get_dt_arrays(hdf, fallback_dt)
length = max([len(a) for a in dt_arrays])
if length > 1:
# ensure all arrays are the same length
for n, arr in enumerate(dt_arrays):
if len(arr) == 1:
# repeat to the correct size
arr = np.repeat(arr, length)
dt_arrays[n] = arr
elif len(arr) != length:
raise ValueError("After align, all arrays should be the same "
"length")
else:
pass
# establish timebase for start of data
if has_constant_time(hdf):
timebase = fallback_dt
else:
try:
timebase = calculate_timebase(*dt_arrays)
except (KeyError, ValueError) as err:
raise TimebaseError("Error with timestamp values: %s" % err)
if timebase > now:
# Flight Data Analysis in the future is a challenge, lets see if we
# can correct this first...
if 'Day' not in hdf:
# unlikely to have year, month or day.
# Scenario: that fallback_dt is of the current day but recorded
# time is in the future of the fallback time, therefore resulting
# in a futuristic date.
a_day_before = timebase - relativedelta(days=1)
if a_day_before < now:
logger.info(
"Timebase was in the future, using a DAY before "
"satisfies requirements: %s", a_day_before)
return a_day_before
# continue to take away a Year
if 'Year' not in hdf:
# remove a year from the timebase
a_year_before = timebase - relativedelta(years=1)
if a_year_before < now:
logger.info("Timebase was in the future, using a YEAR before "
"satisfies requirements: %s", a_year_before)
return a_year_before
raise TimebaseError("Timebase '%s' is in the future.", timebase)
if settings.MAX_TIMEBASE_AGE and \
timebase < (now - timedelta(days=settings.MAX_TIMEBASE_AGE)):
# Only allow recent timebases.
error_msg = "Timebase '%s' older than the allowed '%d' days." % (
timebase, settings.MAX_TIMEBASE_AGE)
raise TimebaseError(error_msg)
logger.info("Valid timebase identified as %s", timebase)
return timebase
def _calculate_start_datetime(hdf, fallback_dt, validation_dt):
"""
Calculate start datetime.
:param hdf: Flight data HDF file
:type hdf: hdf_access object
:param fallback_dt: Used to replace elements of datetimes which are not
available in the hdf file (e.g. YEAR not being recorded)
:type fallback_dt: datetime
HDF params used:
:Year: Optional (defaults to 1970)
:Month: Optional (defaults to 1)
:Day: Optional (defaults to 1)
:Hour: Required
:Minute: Required
:Second: Required
If required parameters are not available and fallback_dt is not provided,
a TimebaseError is raised
"""
now = datetime.utcnow().replace(tzinfo=pytz.utc)
if fallback_dt is not None:
if (fallback_dt.tzinfo is None
or fallback_dt.tzinfo.utcoffset(fallback_dt) is None):
# Assume fallback_dt is UTC.
fallback_dt = fallback_dt.replace(tzinfo=pytz.utc)
assert fallback_dt <= now, (
"Fallback time '%s' in the future is not allowed. Current time "
"is '%s'." % (fallback_dt, now))
if validation_dt is not None:
assert fallback_dt <= validation_dt, (
"Fallback time '%s' ahead of validation time is not allowed. "
"Validation time is '%s'." % (fallback_dt, validation_dt))
# align required parameters to 1Hz
dt_arrays = get_dt_arrays(hdf, fallback_dt, validation_dt)
length = max([len(a) for a in dt_arrays])
if length > 1:
# ensure all arrays are the same length
for n, arr in enumerate(dt_arrays):
if len(arr) == 1:
# repeat to the correct size
arr = np.repeat(arr, length)
dt_arrays[n] = arr
elif len(arr) != length:
raise ValueError("After align, all arrays should be the same "
"length")
else:
pass
# establish timebase for start of data
if has_constant_time(hdf):
timebase = fallback_dt
else:
try:
timebase = calculate_timebase(*dt_arrays)
except (KeyError, ValueError) as err:
raise TimebaseError("Error with timestamp values: %s" % err)
if timebase > now:
# Flight Data Analysis in the future is a challenge, lets see if we
# can correct this first...
if 'Day' not in hdf:
# unlikely to have year, month or day.
# Scenario: that fallback_dt is of the current day but recorded
# time is in the future of the fallback time, therefore resulting
# in a futuristic date.
a_day_before = timebase - relativedelta(days=1)
if a_day_before < now:
logger.info(
"Timebase was in the future, using a DAY before "
"satisfies requirements: %s", a_day_before)
return a_day_before
# continue to take away a Year
if 'Year' not in hdf:
# remove a year from the timebase
a_year_before = timebase - relativedelta(years=1)
if a_year_before < now:
logger.info(
"Timebase was in the future, using a YEAR before "
"satisfies requirements: %s", a_year_before)
return a_year_before
raise TimebaseError("Timebase '%s' is in the future.", timebase)
if settings.MAX_TIMEBASE_AGE and \
timebase < (now - timedelta(days=settings.MAX_TIMEBASE_AGE)):
# Only allow recent timebases.
error_msg = "Timebase '%s' older than the allowed '%d' days." % (
timebase, settings.MAX_TIMEBASE_AGE)
raise TimebaseError(error_msg)
logger.info("Valid timebase identified as %s", timebase)
return timebase
def _calculate_start_datetime(hdf, fallback_dt=None):
"""
Calculate start datetime.
:param hdf: Flight data HDF file
:type hdf: hdf_access object
:param fallback_dt: Used to replace elements of datetimes which are not available in the hdf file (e.g. YEAR not being recorded)
:type fallback_dt: datetime
HDF params used:
:Year: Optional (defaults to 1970)
:Month: Optional (defaults to 1)
:Day: Optional (defaults to 1)
:Hour: Required
:Minute: Required
:Second: Required
If required parameters are not available and fallback_dt is not provided,
a TimebaseError is raised
"""
now = datetime.now()
if fallback_dt is not None:
assert fallback_dt < now, \
("Fallback time '%s' in the future is not allowed. Current time "
"is '%s'." % (fallback_dt, now))
# align required parameters to 1Hz
onehz = P(frequency = 1)
dt_arrays = []
for name in ('Year', 'Month', 'Day', 'Hour', 'Minute', 'Second'):
param = hdf.get(name)
if param:
if name == 'Year':
year = getattr(fallback_dt, 'year', None) or now.year
param.array = _mask_invalid_years(param.array, year)
# do not interpolate date/time parameters to avoid rollover issues
array = align(param, onehz, interpolate=False)
if len(array) == 0 or np.ma.count(array) == 0 or np.ma.all(array == 0):
# Other than the year 2000 or possibly 2100, no date values
# can be all 0's
logger.warning("No valid values returned for %s", name)
else:
# values returned, continue
dt_arrays.append(array)
continue
if fallback_dt:
array = [getattr(fallback_dt, name.lower())]
logger.warning("%s not available, using %d from fallback_dt %s",
name, array[0], fallback_dt)
dt_arrays.append(array)
continue
else:
raise TimebaseError("Required parameter '%s' not available" % name)
length = max([len(array) for array in dt_arrays])
if length > 1:
# ensure all arrays are the same length
for n, arr in enumerate(dt_arrays):
if len(arr) == 1:
# repeat to the correct size
arr = np.repeat(arr, length)
dt_arrays[n] = arr
elif len(arr) != length:
raise ValueError("After align, all arrays should be the same "
"length")
else:
pass
# establish timebase for start of data
try:
timebase = calculate_timebase(*dt_arrays)
except (KeyError, ValueError) as err:
raise TimebaseError("Error with timestamp values: %s" % err)
if timebase > now:
# Flight Data Analysis in the future is a challenge, lets see if we
# can correct this first...
if 'Day' not in hdf:
# unlikely to have year, month or day.
# Scenario: that fallback_dt is of the current day but recorded
# time is in the future of the fallback time, therefore resulting
# in a futuristic date.
a_day_before = timebase - relativedelta(days=1)
if a_day_before < now:
logger.info("Timebase was in the future, using a day before satisfies requirements")
return a_day_before
# continue to take away a Year
if 'Year' not in hdf:
# remove a year from the timebase
a_year_before = timebase - relativedelta(years=1)
if a_year_before < now:
logger.info("Timebase was in the future, using a day before satisfies requirements")
return a_year_before
raise TimebaseError("Timebase '%s' is in the future.", timebase)
if settings.MAX_TIMEBASE_AGE and \
timebase < (now - timedelta(days=settings.MAX_TIMEBASE_AGE)):
# Only allow recent timebases.
error_msg = "Timebase '%s' older than the allowed '%d' days." % (
timebase, settings.MAX_TIMEBASE_AGE)
raise TimebaseError(error_msg)
logger.info("Valid timebase identified as %s", timebase)
return timebase
def _calculate_start_datetime(hdf, fallback_dt=None):
"""
Calculate start datetime.
:param hdf: Flight data HDF file
:type hdf: hdf_access object
:param fallback_dt: Used to replace elements of datetimes which are not
available in the hdf file (e.g. YEAR not being recorded)
:type fallback_dt: datetime
HDF params used:
:Year: Optional (defaults to 1970)
:Month: Optional (defaults to 1)
:Day: Optional (defaults to 1)
:Hour: Required
:Minute: Required
:Second: Required
If required parameters are not available and fallback_dt is not provided,
a TimebaseError is raised
"""
now = datetime.utcnow().replace(tzinfo=pytz.utc)
if fallback_dt is not None:
if (fallback_dt.tzinfo is None or
fallback_dt.tzinfo.utcoffset(fallback_dt) is None):
# Assume fallback_dt is UTC.
fallback_dt = fallback_dt.replace(tzinfo=pytz.utc)
assert fallback_dt < now, (
"Fallback time '%s' in the future is not allowed. Current time "
"is '%s'." % (fallback_dt, now))
# align required parameters to 1Hz
onehz = P(frequency=1)
dt_arrays = []
for name in ('Year', 'Month', 'Day', 'Hour', 'Minute', 'Second'):
param = hdf.get(name)
if param:
if name == 'Year':
year = getattr(fallback_dt, 'year', None) or now.year
param.array = _mask_invalid_years(param.array, year)
# do not interpolate date/time parameters to avoid rollover issues
array = align(param, onehz, interpolate=False)
if len(array) == 0 or np.ma.count(array) == 0 \
or np.ma.all(array == 0):
# Other than the year 2000 or possibly 2100, no date values
# can be all 0's
logger.warning("No valid values returned for %s", name)
else:
# values returned, continue
dt_arrays.append(array)
continue
if fallback_dt:
array = [getattr(fallback_dt, name.lower())]
logger.warning("%s not available, using %d from fallback_dt %s",
name, array[0], fallback_dt)
dt_arrays.append(array)
continue
else:
raise TimebaseError("Required parameter '%s' not available" % name)
length = max([len(a) for a in dt_arrays])
if length > 1:
# ensure all arrays are the same length
for n, arr in enumerate(dt_arrays):
if len(arr) == 1:
# repeat to the correct size
arr = np.repeat(arr, length)
dt_arrays[n] = arr
elif len(arr) != length:
raise ValueError("After align, all arrays should be the same "
"length")
else:
pass
# establish timebase for start of data
try:
timebase = calculate_timebase(*dt_arrays)
except (KeyError, ValueError) as err:
raise TimebaseError("Error with timestamp values: %s" % err)
if timebase > now:
# Flight Data Analysis in the future is a challenge, lets see if we
# can correct this first...
if 'Day' not in hdf:
# unlikely to have year, month or day.
# Scenario: that fallback_dt is of the current day but recorded
# time is in the future of the fallback time, therefore resulting
# in a futuristic date.
a_day_before = timebase - relativedelta(days=1)
if a_day_before < now:
logger.info(
"Timebase was in the future, using a DAY before "
"satisfies requirements: %s", a_day_before)
return a_day_before
# continue to take away a Year
if 'Year' not in hdf:
# remove a year from the timebase
a_year_before = timebase - relativedelta(years=1)
if a_year_before < now:
logger.info("Timebase was in the future, using a YEAR before "
"satisfies requirements: %s", a_year_before)
return a_year_before
raise TimebaseError("Timebase '%s' is in the future.", timebase)
if settings.MAX_TIMEBASE_AGE and \
timebase < (now - timedelta(days=settings.MAX_TIMEBASE_AGE)):
# Only allow recent timebases.
error_msg = "Timebase '%s' older than the allowed '%d' days." % (
timebase, settings.MAX_TIMEBASE_AGE)
raise TimebaseError(error_msg)
logger.info("Valid timebase identified as %s", timebase)
return timebase
