def append_segment_info(hdf_segment_path, segment_type, segment_slice, part,
fallback_dt=None):
"""
Get information about a segment such as type, hash, etc. and return a
named tuple.
If a valid timestamp can't be found, it creates start_dt as epoch(0)
i.e. datetime(1970,1,1,1,0). Go-fast dt and Stop dt are relative to this
point in time.
:param hdf_segment_path: path to HDF segment to analyse
:type hdf_segment_path: string
:param segment_slice: Slice of this segment relative to original file.
:type segment_slice: slice
:param part: Numeric part this segment was in the original data file (1 indexed)
:type part: Integer
:param fallback_dt: Used to replace elements of datetimes which are not available in the hdf file (e.g. YEAR not being recorded)
:type fallback_dt: datetime
:returns: Segment named tuple
:rtype: Segment
"""
# build information about a slice
with hdf_file(hdf_segment_path) as hdf:
airspeed = hdf['Airspeed'].array
duration = hdf.duration
# For now, raise TimebaseError up rather than using EPOCH
# TODO: Review whether to revert to epoch again.
##try:
start_datetime = _calculate_start_datetime(hdf, fallback_dt)
##except TimebaseError:
##logger.warning("Unable to calculate timebase, using epoch "
##"1.1.1970!")
##start_datetime = datetime.fromtimestamp(0)
stop_datetime = start_datetime + timedelta(seconds=duration)
hdf.start_datetime = start_datetime
if segment_type in ('START_AND_STOP', 'START_ONLY', 'STOP_ONLY'):
# we went fast, so get the index
spd_above_threshold = \
np.ma.where(airspeed > settings.AIRSPEED_THRESHOLD)
go_fast_index = spd_above_threshold[0][0]
go_fast_datetime = \
start_datetime + timedelta(seconds=int(go_fast_index))
# Identification of raw data airspeed hash
airspeed_hash_sections = runs_of_ones(airspeed.data > settings.AIRSPEED_THRESHOLD)
airspeed_hash = hash_array(airspeed.data,airspeed_hash_sections,
settings.AIRSPEED_HASH_MIN_SAMPLES)
#elif segment_type == 'GROUND_ONLY':
##Q: Create a groundspeed hash?
#pass
else:
go_fast_index = None
go_fast_datetime = None
# if not go_fast, create hash from entire file
airspeed_hash = sha_hash_file(hdf_segment_path)
# ('slice type part path hash start_dt go_fast_dt stop_dt')
segment = Segment(segment_slice, segment_type, part, hdf_segment_path, airspeed_hash, start_datetime, go_fast_datetime, stop_datetime)
return segment
def test_deprecated_function(self):
msg = re.escape(
'Call to deprecated function (or staticmethod) sha_hash_file. '
'(Superseded by flightdatautilities.filesystem_tools:checksum().)',
)
with self.assertWarnsRegex(DeprecationWarning, msg):
with tempfile.NamedTemporaryFile(mode='xb', suffix='.bz2') as f:
f.write(
b'\x42\x5a\x68\x39\x17\x72\x45\x38\x50\x90\x00\x00\x00\x00'
)
f.flush()
self.assertEqual(
fst.sha_hash_file(f.name),
'd3dda84eb03b9738d118eb2be78e246106900493c0ae07819ad60815134a8058'
)
def append_segment_info(hdf_segment_path, segment_type, segment_slice, part,
fallback_dt=None):
"""
Get information about a segment such as type, hash, etc. and return a
named tuple.
If a valid timestamp can't be found, it creates start_dt as epoch(0)
i.e. datetime(1970,1,1,1,0). Go-fast dt and Stop dt are relative to this
point in time.
:param hdf_segment_path: path to HDF segment to analyse
:type hdf_segment_path: string
:param segment_slice: Slice of this segment relative to original file.
:type segment_slice: slice
:param part: Numeric part this segment was in the original data file (1
indexed)
:type part: Integer
:param fallback_dt: Used to replace elements of datetimes which are not
available in the hdf file (e.g. YEAR not being recorded)
:type fallback_dt: datetime
:returns: Segment named tuple
:rtype: Segment
"""
# build information about a slice
with hdf_file(hdf_segment_path) as hdf:
airspeed = hdf['Airspeed']
duration = hdf.duration
try:
start_datetime = _calculate_start_datetime(hdf, fallback_dt)
except TimebaseError:
# Warn the user and store the fake datetime. The code on the other
# side should check the datetime and avoid processing this file
logger.exception('Unable to calculate timebase, using '
'1970-01-01 00:00:00+0000!')
start_datetime = datetime.utcfromtimestamp(0).replace(tzinfo=pytz.utc)
stop_datetime = start_datetime + timedelta(seconds=duration)
hdf.start_datetime = start_datetime
if segment_type in ('START_AND_STOP', 'START_ONLY', 'STOP_ONLY'):
# we went fast, so get the index
spd_above_threshold = \
np.ma.where(airspeed.array > settings.AIRSPEED_THRESHOLD)
go_fast_index = spd_above_threshold[0][0] / airspeed.frequency
go_fast_datetime = \
start_datetime + timedelta(seconds=int(go_fast_index))
# Identification of raw data airspeed hash
airspeed_hash_sections = runs_of_ones(airspeed.array.data >
settings.AIRSPEED_THRESHOLD)
airspeed_hash = hash_array(airspeed.array.data, airspeed_hash_sections,
settings.AIRSPEED_HASH_MIN_SAMPLES)
#elif segment_type == 'GROUND_ONLY':
##Q: Create a groundspeed hash?
#pass
else:
go_fast_index = None
go_fast_datetime = None
# if not go_fast, create hash from entire file
airspeed_hash = sha_hash_file(hdf_segment_path)
segment = Segment(
segment_slice,
segment_type,
part,
hdf_segment_path,
airspeed_hash,
start_datetime,
go_fast_datetime,
stop_datetime
)
return segment
def append_segment_info(hdf_segment_path,
segment_type,
segment_slice,
part,
fallback_dt=None,
validation_dt=None,
aircraft_info={}):
"""
Get information about a segment such as type, hash, etc. and return a
named tuple.
If a valid timestamp can't be found, it creates start_dt as epoch(0)
i.e. datetime(1970,1,1,1,0). Go-fast dt and Stop dt are relative to this
point in time.
:param hdf_segment_path: path to HDF segment to analyse
:type hdf_segment_path: string
:param segment_slice: Slice of this segment relative to original file.
:type segment_slice: slice
:param part: Numeric part this segment was in the original data file (1
indexed)
:type part: Integer
:param fallback_dt: Used to replace elements of datetimes which are not
available in the hdf file (e.g. YEAR not being recorded)
:type fallback_dt: datetime
:returns: Segment named tuple
:rtype: Segment
"""
# build information about a slice
with hdf_file(hdf_segment_path) as hdf:
speed, thresholds = _get_speed_parameter(hdf, aircraft_info)
duration = hdf.duration
try:
start_datetime = _calculate_start_datetime(hdf, fallback_dt,
validation_dt)
except TimebaseError:
# Warn the user and store the fake datetime. The code on the other
# side should check the datetime and avoid processing this file
logger.exception(
'Unable to calculate timebase, using 1970-01-01 00:00:00+0000!'
)
start_datetime = datetime.utcfromtimestamp(0).replace(
tzinfo=pytz.utc)
stop_datetime = start_datetime + timedelta(seconds=duration)
hdf.start_datetime = start_datetime
if segment_type in ('START_AND_STOP', 'START_ONLY', 'STOP_ONLY'):
# we went fast, so get the index
spd_above_threshold = \
np.ma.where(speed.array > thresholds['speed_threshold'])
go_fast_index = spd_above_threshold[0][0] / speed.frequency
go_fast_datetime = \
start_datetime + timedelta(seconds=int(go_fast_index))
# Identification of raw data speed hash
speed_hash_sections = runs_of_ones(
speed.array.data > thresholds['speed_threshold'])
speed_hash = hash_array(speed.array.data, speed_hash_sections,
thresholds['hash_min_samples'])
#elif segment_type == 'GROUND_ONLY':
##Q: Create a groundspeed hash?
#pass
else:
go_fast_index = None
go_fast_datetime = None
# if not go_fast, create hash from entire file
speed_hash = sha_hash_file(hdf_segment_path)
segment = Segment(segment_slice, segment_type, part, hdf_segment_path,
speed_hash, start_datetime, go_fast_datetime,
stop_datetime)
return segment