def test_extracted_bouts():
one_bouts = counts.bouts(1,1)
zero_bouts = counts.bouts(0,0)
# Bouts where counts == 0 and counts == 1 should be mutually excluse
# So there should be no intersections between them
intersections = Bout.bout_list_intersection(one_bouts, zero_bouts)
assert(len(intersections) == 0)
# A bout that spans the whole time period should completely intersect with bouts where counts == 1
one_big_bout = Bout.Bout(counts.timestamps[0]-timedelta(days=1), counts.timestamps[-1]+timedelta(days=1))
one_intersections = Bout.bout_list_intersection(one_bouts, [one_big_bout])
assert(Bout.total_time(one_intersections) == Bout.total_time(one_bouts))
# Same for zeros
zero_intersections = Bout.bout_list_intersection(zero_bouts, [one_big_bout])
assert(Bout.total_time(zero_intersections) == Bout.total_time(zero_bouts))
# Filling in the bout gaps of one bouts should recreate the zero bouts
inverse_of_one_bouts = Bout.time_period_minus_bouts((counts.timeframe[0], counts.timeframe[1]+timedelta(minutes=1)), one_bouts)
# They should have the same n
assert(len(inverse_of_one_bouts) == len(zero_bouts))
# Same total amount of time
assert(Bout.total_time(inverse_of_one_bouts) == Bout.total_time(zero_bouts))
def test_extracted_bouts():
one_bouts = counts.bouts(1, 1)
zero_bouts = counts.bouts(0, 0)
# Bouts where counts == 0 and counts == 1 should be mutually excluse
# So there should be no intersections between them
intersections = Bout.bout_list_intersection(one_bouts, zero_bouts)
assert (len(intersections) == 0)
# A bout that spans the whole time period should completely intersect with bouts where counts == 1
one_big_bout = Bout.Bout(counts.timestamps[0] - timedelta(days=1),
counts.timestamps[-1] + timedelta(days=1))
one_intersections = Bout.bout_list_intersection(one_bouts, [one_big_bout])
assert (Bout.total_time(one_intersections) == Bout.total_time(one_bouts))
# Same for zeros
zero_intersections = Bout.bout_list_intersection(zero_bouts,
[one_big_bout])
assert (Bout.total_time(zero_intersections) == Bout.total_time(zero_bouts))
# Filling in the bout gaps of one bouts should recreate the zero bouts
inverse_of_one_bouts = Bout.time_period_minus_bouts(
(counts.timeframe[0], counts.timeframe[1] + timedelta(minutes=1)),
one_bouts)
# They should have the same n
assert (len(inverse_of_one_bouts) == len(zero_bouts))
# Same total amount of time
assert (
Bout.total_time(inverse_of_one_bouts) == Bout.total_time(zero_bouts))
def test_bouts():
# There are 8 bouts of 0s
zero_bouts = counts.bouts(0, 0)
assert (len(zero_bouts) == 8)
# There are 8 bouts of 1s
one_bouts = counts.bouts(1, 1)
assert (len(one_bouts) == 8)
# Since there are only 1s and 0s in the file, there should be 1 bout of 0 to 1
both_bouts = counts.bouts(0, 1)
assert (len(both_bouts) == 1)
# The timestamps of that 1 bout should match the start and end of the channel timestamps
# But "end" of bout occurs 1 minute after end of channel
assert (both_bouts[0].start_timestamp == counts.timestamps[0])
assert (both_bouts[0].end_timestamp == counts.timestamps[-1] +
timedelta(minutes=1))
# Changing the max value shouldn't change anything
bouts = counts.bouts(0, 23)
assert (len(bouts) == 1)
# Same for the minimum value
bouts = counts.bouts(-340, 23)
assert (len(bouts) == 1)
# Should be no bouts 2 or above
bouts = counts.bouts(2, 23)
assert (len(bouts) == 0)
# Same for below 0
bouts = counts.bouts(-32323, -2)
assert (len(bouts) == 0)
# The data is in 1 minute epochs
total_zero_time = Bout.total_time(zero_bouts)
total_one_time = Bout.total_time(one_bouts)
total_both_time = Bout.total_time(both_bouts)
assert (total_zero_time == timedelta(minutes=10 * 30))
assert (total_one_time == timedelta(minutes=16 * 30))
assert (total_both_time == total_zero_time + total_one_time)
# Integer seconds spent at 0 should be 300 minutes * 60 = 18000 seconds
total_zero_time_seconds = total_zero_time.total_seconds()
assert (total_zero_time_seconds == 10 * 30 * 60)
# Inverting bouts within a period
# Since the file is 0s and 1s, the total time - the time spent @ 0 should = time spent @ 1
not_zero_bouts = Bout.time_period_minus_bouts(
(counts.timestamps[0], counts.timestamps[-1] + timedelta(minutes=1)),
zero_bouts)
total_not_zero_time = Bout.total_time(not_zero_bouts)
assert (total_not_zero_time == total_one_time)
def test_bouts():
# There are 8 bouts of 0s
zero_bouts = counts.bouts(0,0)
assert(len(zero_bouts) == 8)
# There are 8 bouts of 1s
one_bouts = counts.bouts(1,1)
assert(len(one_bouts) == 8)
# Since there are only 1s and 0s in the file, there should be 1 bout of 0 to 1
both_bouts = counts.bouts(0,1)
assert(len(both_bouts) == 1)
# The timestamps of that 1 bout should match the start and end of the channel timestamps
# But "end" of bout occurs 1 minute after end of channel
assert(both_bouts[0].start_timestamp == counts.timestamps[0])
assert(both_bouts[0].end_timestamp == counts.timestamps[-1]+timedelta(minutes=1))
# Changing the max value shouldn't change anything
bouts = counts.bouts(0,23)
assert(len(bouts) == 1)
# Same for the minimum value
bouts = counts.bouts(-340,23)
assert(len(bouts) == 1)
# Should be no bouts 2 or above
bouts = counts.bouts(2,23)
assert(len(bouts) == 0)
# Same for below 0
bouts = counts.bouts(-32323,-2)
assert(len(bouts) == 0)
# The data is in 1 minute epochs
total_zero_time = Bout.total_time(zero_bouts)
total_one_time = Bout.total_time(one_bouts)
total_both_time = Bout.total_time(both_bouts)
assert(total_zero_time == timedelta(minutes=10*30))
assert(total_one_time == timedelta(minutes=16*30))
assert(total_both_time == total_zero_time + total_one_time)
# Integer seconds spent at 0 should be 300 minutes * 60 = 18000 seconds
total_zero_time_seconds = total_zero_time.total_seconds()
assert(total_zero_time_seconds == 10*30*60)
# Inverting bouts within a period
# Since the file is 0s and 1s, the total time - the time spent @ 0 should = time spent @ 1
not_zero_bouts = Bout.time_period_minus_bouts((counts.timestamps[0],counts.timestamps[-1]+timedelta(minutes=1)), zero_bouts)
total_not_zero_time = Bout.total_time(not_zero_bouts)
assert(total_not_zero_time == total_one_time)
def infer_nonwear_actigraph(counts, zero_minutes=timedelta(minutes=60)):
"""Given an Actigraph counts signal, infer nonwear as consecutive zeros of a given duration. """
# List all bouts where the signal was <= 0
nonwear_bouts = counts.bouts(-999999, 0)
# Limit those bouts to the minimum duration specified in "zero_minutes"
nonwear_bouts = Bout.limit_to_lengths(nonwear_bouts, min_length=zero_minutes)
# Invert the nonwear bouts to get wear bouts
wear_bouts = Bout.time_period_minus_bouts([counts.timeframe[0], counts.timeframe[1]], nonwear_bouts)
return nonwear_bouts, wear_bouts
def qc_analysis(job_details):
id_num = str(job_details["pid"])
filename = job_details["filename"]
filename_short = os.path.basename(filename).split('.')[0]
battery_max = 0
if filetype == "GeneActiv":
battery_max = GA_battery_max
elif filetype == "Axivity":
battery_max = AX_battery_max
# Load the data from the hdf5 file
ts, header = data_loading.fast_load(filename, filetype)
header["QC_filename"] = os.path.basename(filename)
x, y, z, battery, temperature = ts.get_channels(["X", "Y", "Z", "Battery", "Temperature"])
# create a channel of battery percentage, based on the assumed battery maximum value
battery_pct = Channel.Channel.clone(battery)
battery_pct.data = (battery.data / battery_max) * 100
channels = [x, y, z, battery, temperature, battery_pct]
anomalies = diagnostics.diagnose_fix_anomalies(channels, discrepancy_threshold=2)
# create dictionary of anomalies types
anomalies_dict = dict()
# check whether any anomalies have been found:
if len(anomalies) > 0:
anomalies_file = os.path.join(anomalies_folder, "{}_anomalies.csv".format(filename_short))
df = pd.DataFrame(anomalies)
for type in anomaly_types:
anomalies_dict["QC_anomaly_{}".format(type)] = (df.anomaly_type.values == type).sum()
df = df.set_index("anomaly_type")
# print record of anomalies to anomalies_file
df.to_csv(anomalies_file)
else:
for type in anomaly_types:
anomalies_dict["QC_anomaly_{}".format(type)] = 0
# check for axis anomalies
axes_dict = diagnostics.diagnose_axes(x, y, z, noise_cutoff_mg=13)
axis_anomaly = False
for key, val in axes_dict.items():
anomalies_dict["QC_{}".format(key)] = val
if key.endswith("max"):
if val > axis_max:
axis_anomaly = True
elif key.endswith("min"):
if val < axis_min:
axis_anomaly = True
# create a "check battery" flag:
check_battery = False
# calculate first and last battery percentages
first_battery_pct = round((battery_pct.data[1]),2)
last_battery_pct = round((battery_pct.data[-1]),2)
header["QC_first_battery_pct"] = first_battery_pct
header["QC_last_battery_pct"] = last_battery_pct
# calculate lowest battery percentage
# check if battery.pct has a missing_value, exclude those values if they exist
if battery_pct.missing_value == "None":
lowest_battery_pct = min(battery_pct.data)
else:
test_array = np.delete(battery_pct.data, np.where(battery_pct.data == battery_pct.missing_value))
lowest_battery_pct = min(test_array)
header["QC_lowest_battery_pct"] = round(lowest_battery_pct,2)
header["QC_lowest_battery_threshold"] = battery_minimum
# find the maximum battery discharge in any 24hr period:
max_discharge = battery_pct.channel_max_decrease(time_period=timedelta(hours=discharge_hours))
header["QC_max_discharge"] = round(max_discharge, 2)
header["QC_discharge_time_period"] = "{} hours".format(discharge_hours)
header["QC_discharge_threshold"] = discharge_pct
# change flag if lowest battery percentage dips below battery_minimum at any point
# OR maximum discharge greater than discharge_pct over time period "hours = discharge_hours"
if lowest_battery_pct < battery_minimum or max_discharge > discharge_pct:
check_battery = True
header["QC_check_battery"] = str(check_battery)
header["QC_axis_anomaly"] = str(axis_anomaly)
# Calculate the time frame to use
start = time_utilities.start_of_day(x.timeframe[0])
end = time_utilities.end_of_day(x.timeframe[-1])
tp = (start, end)
results_ts = Time_Series.Time_Series("")
# Derive some signal features
vm = channel_inference.infer_vector_magnitude(x, y, z)
enmo = channel_inference.infer_enmo(vm)
enmo.minimum = 0
enmo.maximum = enmo_max
# Infer nonwear
nonwear_bouts = channel_inference.infer_nonwear_for_qc(x, y, z, noise_cutoff_mg=noise_cutoff_mg)
# Use nonwear bouts to calculate wear bouts
wear_bouts = Bout.time_period_minus_bouts(enmo.timeframe, nonwear_bouts)
# Use wear bouts to calculate the amount of wear time in the file in hours, save to meta data
total_wear = Bout.total_time(wear_bouts)
total_seconds_wear = total_wear.total_seconds()
total_hours_wear = round(total_seconds_wear/3600)
header["QC_total_hours_wear"] = total_hours_wear
# Split the enmo channel into lists of bouts for each quadrant:
''' quadrant_0 = 00:00 -> 06: 00
quadrant_1 = 06:00 -> 12: 00
quadrant_2 = 12:00 -> 18: 00
quadrant_3 = 18:00 -> 00: 00 '''
q_0, q_1, q_2, q_3 = channel_inference.create_quadrant_bouts(enmo)
# calculate the intersection of each set of bouts with wear_bouts, then calculate the wear time in each quadrant.
sum_quadrant_wear = 0
for quadrant, name1, name2 in ([q_0, "QC_hours_wear_quadrant_0", "QC_pct_wear_quadrant_0"],
[q_1, "QC_hours_wear_quadrant_1", "QC_pct_wear_quadrant_1"],
[q_2, "QC_hours_wear_quadrant_2", "QC_pct_wear_quadrant_2"],
[q_3, "QC_hours_wear_quadrant_3", "QC_pct_wear_quadrant_3"]):
quadrant_wear = Bout.bout_list_intersection(quadrant, wear_bouts)
seconds_wear = Bout.total_time(quadrant_wear).total_seconds()
hours_wear = round(seconds_wear / 3600)
header[name1] = hours_wear
header[name2] = round(((hours_wear / total_hours_wear) * 100), 2)
for bout in nonwear_bouts:
# Show non-wear bouts in purple
bout.draw_properties = {'lw': 0, 'alpha': 0.75, 'facecolor': '#764af9'}
for channel, channel_name in zip([enmo, battery_pct],["ENMO", "Battery_percentage"]):
channel.name = channel_name
results_ts.add_channel(channel)
if PLOT == "YES":
# Plot statistics as subplots in one plot file per data file
results_ts["ENMO"].add_annotations(nonwear_bouts)
results_ts.draw_qc(plotting_df, file_target=os.path.join(charts_folder,"{}_plots.png".format(filename_short)))
header["QC_script"] = version
# file of metadata from qc process
qc_output = os.path.join(results_folder, "qc_meta_{}.csv".format(filename_short))
# check if qc_output already exists...
if os.path.isfile(qc_output):
os.remove(qc_output)
metadata = {**header, **anomalies_dict}
# write metadata to file
pampro_utilities.dict_write(qc_output, id_num, metadata)
for c in ts:
del c.data
del c.timestamps
del c.indices
del c.cached_indices
