def calculate_fallback_dt(hdf, fallback_dt=None, fallback_relative_to_start=True): """ Check the time parameters in the HDF5 file and update the fallback_dt. Takes into account adjustment of fallback datetime if it's relative to the end of data and if there's a constant timebase within the data it will use this as the fallback datetime. """ if fallback_dt and not fallback_relative_to_start: # fallback_dt is relative to the end of the data; remove the data # duration to make it relative to the start of the data secs = hdf.duration fallback_dt -= timedelta(seconds=secs) # Q: minus the number of segments * gaps between them? # ADDRESS! logger.info("Reduced fallback_dt by %ddays %dhr %dmin to %s", secs // 86400, secs % 86400 // 3600, secs % 86400 % 3600 // 60, fallback_dt) if not has_constant_time(hdf): # we don't need to do any further corrections return fallback_dt try: timebase = calculate_timebase(*get_dt_arrays(hdf, fallback_dt)) except (KeyError, ValueError): # The time parameters are not available/operational return fallback_dt else: logger.warning("Time doesn't change, using the starting time as the fallback_dt") return timebase
def calculate_fallback_dt(hdf, fallback_dt=None, validation_dt=None, fallback_relative_to_start=True): """ Check the time parameters in the HDF5 file and update the fallback_dt. Takes into account adjustment of fallback datetime if it's relative to the end of data and if there's a constant timebase within the data it will use this as the fallback datetime. """ if fallback_dt and not fallback_relative_to_start: # fallback_dt is relative to the end of the data; remove the data # duration to make it relative to the start of the data secs = hdf.duration fallback_dt -= timedelta(seconds=secs) logger.info("Reduced fallback_dt by %ddays %dhr %dmin to %s", secs // 86400, secs % 86400 // 3600, secs % 86400 % 3600 // 60, fallback_dt) if not has_constant_time(hdf): # we don't need to do any further corrections return fallback_dt try: timebase = calculate_timebase( *get_dt_arrays(hdf, fallback_dt, validation_dt)) except (KeyError, ValueError): # The time parameters are not available/operational return fallback_dt else: logger.warning( "Time doesn't change, using the starting time as the fallback_dt") return timebase
def _calculate_start_datetime(hdf, fallback_dt): """ Calculate start datetime. :param hdf: Flight data HDF file :type hdf: hdf_access object :param fallback_dt: Used to replace elements of datetimes which are not available in the hdf file (e.g. YEAR not being recorded) :type fallback_dt: datetime HDF params used: :Year: Optional (defaults to 1970) :Month: Optional (defaults to 1) :Day: Optional (defaults to 1) :Hour: Required :Minute: Required :Second: Required If required parameters are not available and fallback_dt is not provided, a TimebaseError is raised """ now = datetime.utcnow().replace(tzinfo=pytz.utc) if fallback_dt is not None: if (fallback_dt.tzinfo is None or fallback_dt.tzinfo.utcoffset(fallback_dt) is None): # Assume fallback_dt is UTC. fallback_dt = fallback_dt.replace(tzinfo=pytz.utc) assert fallback_dt < now, ( "Fallback time '%s' in the future is not allowed. Current time " "is '%s'." % (fallback_dt, now)) # align required parameters to 1Hz dt_arrays = get_dt_arrays(hdf, fallback_dt) length = max([len(a) for a in dt_arrays]) if length > 1: # ensure all arrays are the same length for n, arr in enumerate(dt_arrays): if len(arr) == 1: # repeat to the correct size arr = np.repeat(arr, length) dt_arrays[n] = arr elif len(arr) != length: raise ValueError("After align, all arrays should be the same " "length") else: pass # establish timebase for start of data if has_constant_time(hdf): timebase = fallback_dt else: try: timebase = calculate_timebase(*dt_arrays) except (KeyError, ValueError) as err: raise TimebaseError("Error with timestamp values: %s" % err) if timebase > now: # Flight Data Analysis in the future is a challenge, lets see if we # can correct this first... if 'Day' not in hdf: # unlikely to have year, month or day. # Scenario: that fallback_dt is of the current day but recorded # time is in the future of the fallback time, therefore resulting # in a futuristic date. a_day_before = timebase - relativedelta(days=1) if a_day_before < now: logger.info( "Timebase was in the future, using a DAY before " "satisfies requirements: %s", a_day_before) return a_day_before # continue to take away a Year if 'Year' not in hdf: # remove a year from the timebase a_year_before = timebase - relativedelta(years=1) if a_year_before < now: logger.info("Timebase was in the future, using a YEAR before " "satisfies requirements: %s", a_year_before) return a_year_before raise TimebaseError("Timebase '%s' is in the future.", timebase) if settings.MAX_TIMEBASE_AGE and \ timebase < (now - timedelta(days=settings.MAX_TIMEBASE_AGE)): # Only allow recent timebases. error_msg = "Timebase '%s' older than the allowed '%d' days." % ( timebase, settings.MAX_TIMEBASE_AGE) raise TimebaseError(error_msg) logger.info("Valid timebase identified as %s", timebase) return timebase
def _calculate_start_datetime(hdf, fallback_dt, validation_dt): """ Calculate start datetime. :param hdf: Flight data HDF file :type hdf: hdf_access object :param fallback_dt: Used to replace elements of datetimes which are not available in the hdf file (e.g. YEAR not being recorded) :type fallback_dt: datetime HDF params used: :Year: Optional (defaults to 1970) :Month: Optional (defaults to 1) :Day: Optional (defaults to 1) :Hour: Required :Minute: Required :Second: Required If required parameters are not available and fallback_dt is not provided, a TimebaseError is raised """ now = datetime.utcnow().replace(tzinfo=pytz.utc) if fallback_dt is not None: if (fallback_dt.tzinfo is None or fallback_dt.tzinfo.utcoffset(fallback_dt) is None): # Assume fallback_dt is UTC. fallback_dt = fallback_dt.replace(tzinfo=pytz.utc) assert fallback_dt <= now, ( "Fallback time '%s' in the future is not allowed. Current time " "is '%s'." % (fallback_dt, now)) if validation_dt is not None: assert fallback_dt <= validation_dt, ( "Fallback time '%s' ahead of validation time is not allowed. " "Validation time is '%s'." % (fallback_dt, validation_dt)) # align required parameters to 1Hz dt_arrays = get_dt_arrays(hdf, fallback_dt, validation_dt) length = max([len(a) for a in dt_arrays]) if length > 1: # ensure all arrays are the same length for n, arr in enumerate(dt_arrays): if len(arr) == 1: # repeat to the correct size arr = np.repeat(arr, length) dt_arrays[n] = arr elif len(arr) != length: raise ValueError("After align, all arrays should be the same " "length") else: pass # establish timebase for start of data if has_constant_time(hdf): timebase = fallback_dt else: try: timebase = calculate_timebase(*dt_arrays) except (KeyError, ValueError) as err: raise TimebaseError("Error with timestamp values: %s" % err) if timebase > now: # Flight Data Analysis in the future is a challenge, lets see if we # can correct this first... if 'Day' not in hdf: # unlikely to have year, month or day. # Scenario: that fallback_dt is of the current day but recorded # time is in the future of the fallback time, therefore resulting # in a futuristic date. a_day_before = timebase - relativedelta(days=1) if a_day_before < now: logger.info( "Timebase was in the future, using a DAY before " "satisfies requirements: %s", a_day_before) return a_day_before # continue to take away a Year if 'Year' not in hdf: # remove a year from the timebase a_year_before = timebase - relativedelta(years=1) if a_year_before < now: logger.info( "Timebase was in the future, using a YEAR before " "satisfies requirements: %s", a_year_before) return a_year_before raise TimebaseError("Timebase '%s' is in the future.", timebase) if settings.MAX_TIMEBASE_AGE and \ timebase < (now - timedelta(days=settings.MAX_TIMEBASE_AGE)): # Only allow recent timebases. error_msg = "Timebase '%s' older than the allowed '%d' days." % ( timebase, settings.MAX_TIMEBASE_AGE) raise TimebaseError(error_msg) logger.info("Valid timebase identified as %s", timebase) return timebase
def _calculate_start_datetime(hdf, fallback_dt=None): """ Calculate start datetime. :param hdf: Flight data HDF file :type hdf: hdf_access object :param fallback_dt: Used to replace elements of datetimes which are not available in the hdf file (e.g. YEAR not being recorded) :type fallback_dt: datetime HDF params used: :Year: Optional (defaults to 1970) :Month: Optional (defaults to 1) :Day: Optional (defaults to 1) :Hour: Required :Minute: Required :Second: Required If required parameters are not available and fallback_dt is not provided, a TimebaseError is raised """ now = datetime.now() if fallback_dt is not None: assert fallback_dt < now, \ ("Fallback time '%s' in the future is not allowed. Current time " "is '%s'." % (fallback_dt, now)) # align required parameters to 1Hz onehz = P(frequency = 1) dt_arrays = [] for name in ('Year', 'Month', 'Day', 'Hour', 'Minute', 'Second'): param = hdf.get(name) if param: if name == 'Year': year = getattr(fallback_dt, 'year', None) or now.year param.array = _mask_invalid_years(param.array, year) # do not interpolate date/time parameters to avoid rollover issues array = align(param, onehz, interpolate=False) if len(array) == 0 or np.ma.count(array) == 0 or np.ma.all(array == 0): # Other than the year 2000 or possibly 2100, no date values # can be all 0's logger.warning("No valid values returned for %s", name) else: # values returned, continue dt_arrays.append(array) continue if fallback_dt: array = [getattr(fallback_dt, name.lower())] logger.warning("%s not available, using %d from fallback_dt %s", name, array[0], fallback_dt) dt_arrays.append(array) continue else: raise TimebaseError("Required parameter '%s' not available" % name) length = max([len(array) for array in dt_arrays]) if length > 1: # ensure all arrays are the same length for n, arr in enumerate(dt_arrays): if len(arr) == 1: # repeat to the correct size arr = np.repeat(arr, length) dt_arrays[n] = arr elif len(arr) != length: raise ValueError("After align, all arrays should be the same " "length") else: pass # establish timebase for start of data try: timebase = calculate_timebase(*dt_arrays) except (KeyError, ValueError) as err: raise TimebaseError("Error with timestamp values: %s" % err) if timebase > now: # Flight Data Analysis in the future is a challenge, lets see if we # can correct this first... if 'Day' not in hdf: # unlikely to have year, month or day. # Scenario: that fallback_dt is of the current day but recorded # time is in the future of the fallback time, therefore resulting # in a futuristic date. a_day_before = timebase - relativedelta(days=1) if a_day_before < now: logger.info("Timebase was in the future, using a day before satisfies requirements") return a_day_before # continue to take away a Year if 'Year' not in hdf: # remove a year from the timebase a_year_before = timebase - relativedelta(years=1) if a_year_before < now: logger.info("Timebase was in the future, using a day before satisfies requirements") return a_year_before raise TimebaseError("Timebase '%s' is in the future.", timebase) if settings.MAX_TIMEBASE_AGE and \ timebase < (now - timedelta(days=settings.MAX_TIMEBASE_AGE)): # Only allow recent timebases. error_msg = "Timebase '%s' older than the allowed '%d' days." % ( timebase, settings.MAX_TIMEBASE_AGE) raise TimebaseError(error_msg) logger.info("Valid timebase identified as %s", timebase) return timebase
def _calculate_start_datetime(hdf, fallback_dt=None): """ Calculate start datetime. :param hdf: Flight data HDF file :type hdf: hdf_access object :param fallback_dt: Used to replace elements of datetimes which are not available in the hdf file (e.g. YEAR not being recorded) :type fallback_dt: datetime HDF params used: :Year: Optional (defaults to 1970) :Month: Optional (defaults to 1) :Day: Optional (defaults to 1) :Hour: Required :Minute: Required :Second: Required If required parameters are not available and fallback_dt is not provided, a TimebaseError is raised """ now = datetime.utcnow().replace(tzinfo=pytz.utc) if fallback_dt is not None: if (fallback_dt.tzinfo is None or fallback_dt.tzinfo.utcoffset(fallback_dt) is None): # Assume fallback_dt is UTC. fallback_dt = fallback_dt.replace(tzinfo=pytz.utc) assert fallback_dt < now, ( "Fallback time '%s' in the future is not allowed. Current time " "is '%s'." % (fallback_dt, now)) # align required parameters to 1Hz onehz = P(frequency=1) dt_arrays = [] for name in ('Year', 'Month', 'Day', 'Hour', 'Minute', 'Second'): param = hdf.get(name) if param: if name == 'Year': year = getattr(fallback_dt, 'year', None) or now.year param.array = _mask_invalid_years(param.array, year) # do not interpolate date/time parameters to avoid rollover issues array = align(param, onehz, interpolate=False) if len(array) == 0 or np.ma.count(array) == 0 \ or np.ma.all(array == 0): # Other than the year 2000 or possibly 2100, no date values # can be all 0's logger.warning("No valid values returned for %s", name) else: # values returned, continue dt_arrays.append(array) continue if fallback_dt: array = [getattr(fallback_dt, name.lower())] logger.warning("%s not available, using %d from fallback_dt %s", name, array[0], fallback_dt) dt_arrays.append(array) continue else: raise TimebaseError("Required parameter '%s' not available" % name) length = max([len(a) for a in dt_arrays]) if length > 1: # ensure all arrays are the same length for n, arr in enumerate(dt_arrays): if len(arr) == 1: # repeat to the correct size arr = np.repeat(arr, length) dt_arrays[n] = arr elif len(arr) != length: raise ValueError("After align, all arrays should be the same " "length") else: pass # establish timebase for start of data try: timebase = calculate_timebase(*dt_arrays) except (KeyError, ValueError) as err: raise TimebaseError("Error with timestamp values: %s" % err) if timebase > now: # Flight Data Analysis in the future is a challenge, lets see if we # can correct this first... if 'Day' not in hdf: # unlikely to have year, month or day. # Scenario: that fallback_dt is of the current day but recorded # time is in the future of the fallback time, therefore resulting # in a futuristic date. a_day_before = timebase - relativedelta(days=1) if a_day_before < now: logger.info( "Timebase was in the future, using a DAY before " "satisfies requirements: %s", a_day_before) return a_day_before # continue to take away a Year if 'Year' not in hdf: # remove a year from the timebase a_year_before = timebase - relativedelta(years=1) if a_year_before < now: logger.info("Timebase was in the future, using a YEAR before " "satisfies requirements: %s", a_year_before) return a_year_before raise TimebaseError("Timebase '%s' is in the future.", timebase) if settings.MAX_TIMEBASE_AGE and \ timebase < (now - timedelta(days=settings.MAX_TIMEBASE_AGE)): # Only allow recent timebases. error_msg = "Timebase '%s' older than the allowed '%d' days." % ( timebase, settings.MAX_TIMEBASE_AGE) raise TimebaseError(error_msg) logger.info("Valid timebase identified as %s", timebase) return timebase