def window_statistics(self, start_dts, end_dts, statistics): window = Bout.Bout(start_dts, end_dts) bouts = self.bouts_involved(window) output_row = [] if (len(bouts) > 0): for stat in statistics: if stat[0] == "generic": for val1 in stat[1]: if val1 == "sum": intersection = Bout.bout_list_intersection([window],bouts) Bout.cache_lengths(intersection) sum_seconds = Bout.total_time(intersection).total_seconds() output_row.append(sum_seconds) elif val1 == "mean": intersection = Bout.bout_list_intersection([window],bouts) Bout.cache_lengths(intersection) sum_seconds = Bout.total_time(intersection).total_seconds() if sum_seconds >0 and len(bouts) > 0: output_row.append( sum_seconds / len(bouts) ) else: output_row.append(0) elif val1 == "n": output_row.append( len(bouts) ) else: print("nooooooooo") print(stat) print(statistics) output_row.append(-1) elif stat[0] == "sdx": # ("sdx", [10,20,30,40,50,60,70,80,90]) sdx_results = sdx(bouts, stat[1]) for r in sdx_results: output_row.append(r) else: # No bouts in this Bout_Collection overlapping this window # There was no data for the time period # Output -1 for each missing variable for i in range(self.expected_results(statistics)): output_row.append(-1) return output_row
def test_bouts(): # There are 8 bouts of 0s zero_bouts = counts.bouts(0,0) assert(len(zero_bouts) == 8) # There are 8 bouts of 1s one_bouts = counts.bouts(1,1) assert(len(one_bouts) == 8) # Since there are only 1s and 0s in the file, there should be 1 bout of 0 to 1 both_bouts = counts.bouts(0,1) assert(len(both_bouts) == 1) # The timestamps of that 1 bout should match the start and end of the channel timestamps # But "end" of bout occurs 1 minute after end of channel assert(both_bouts[0].start_timestamp == counts.timestamps[0]) assert(both_bouts[0].end_timestamp == counts.timestamps[-1]+timedelta(minutes=1)) # Changing the max value shouldn't change anything bouts = counts.bouts(0,23) assert(len(bouts) == 1) # Same for the minimum value bouts = counts.bouts(-340,23) assert(len(bouts) == 1) # Should be no bouts 2 or above bouts = counts.bouts(2,23) assert(len(bouts) == 0) # Same for below 0 bouts = counts.bouts(-32323,-2) assert(len(bouts) == 0) # The data is in 1 minute epochs total_zero_time = Bout.total_time(zero_bouts) total_one_time = Bout.total_time(one_bouts) total_both_time = Bout.total_time(both_bouts) assert(total_zero_time == timedelta(minutes=10*30)) assert(total_one_time == timedelta(minutes=16*30)) assert(total_both_time == total_zero_time + total_one_time) # Integer seconds spent at 0 should be 300 minutes * 60 = 18000 seconds total_zero_time_seconds = total_zero_time.total_seconds() assert(total_zero_time_seconds == 10*30*60) # Inverting bouts within a period # Since the file is 0s and 1s, the total time - the time spent @ 0 should = time spent @ 1 not_zero_bouts = Bout.time_period_minus_bouts((counts.timestamps[0],counts.timestamps[-1]+timedelta(minutes=1)), zero_bouts) total_not_zero_time = Bout.total_time(not_zero_bouts) assert(total_not_zero_time == total_one_time)
def test_bouts(): one_bouts = counts.bouts(1, 1) # There are 7 bouts assert len(one_bouts) == 7 # Their length is 1, 2, 3, .. 7 assert Bout.total_time(one_bouts) == timedelta(minutes=7 + 6 + 5 + 4 + 3 + 2 + 1) # Keeping bouts >= i minutes means there should be 7-(i-1) left for i in range(1, 7): i_or_longer = Bout.limit_to_lengths(one_bouts, min_length=timedelta(minutes=i)) assert len(i_or_longer) == 7 - (i - 1) # One manual check three_or_longer = Bout.limit_to_lengths(one_bouts, min_length=timedelta(minutes=3)) assert len(three_or_longer) == 5 # This should exclude the 1 bout at exactly 3 minutes three_plus_bit_or_longer = Bout.limit_to_lengths(one_bouts, min_length=timedelta(minutes=3, seconds=1)) assert len(three_plus_bit_or_longer) == 4 # No bouts should be this long eight_or_longer = Bout.limit_to_lengths(one_bouts, min_length=timedelta(minutes=8)) assert len(eight_or_longer) == 0 # There is nothing above 1 in the file, should be 0 bouts two_bouts = counts.bouts(2, 989) assert len(two_bouts) == 0
def infer_valid_days(channel, wear_bouts, valid_criterion=timedelta(hours=10)): #Generate day-long windows start = time_utilities.start_of_day(channel.timestamps[0]) day_windows = [] while start < channel.timeframe[1]: day_windows.append(Bout.Bout(start, start+timedelta(days=1))) start += timedelta(days=1) valid_windows = [] invalid_windows = [] for window in day_windows: #how much does all of wear_bouts intersect with window? intersections = Bout.bout_list_intersection([window], wear_bouts) total = Bout.total_time(intersections) # If the amount of overlap exceeds the valid criterion, it is valid if total >= valid_criterion: #window.draw_properties={"lw":0, "facecolor":[1,0,0], "alpha":0.25} valid_windows.append(window) else: invalid_windows.append(window) return(invalid_windows, valid_windows)
def sdx(bouts, percentages): total_time_minutes = Bout.total_time(bouts).total_seconds()/60 Bout.cache_lengths(bouts) bouts.sort(key=lambda x : x.length) highest_length_minutes = int(bouts[-1].length.total_seconds()/60) targets_minutes = [int((total_time_minutes)/100.0 * percentage) for percentage in percentages] results = [] #print("Number of bouts: ", len(bouts)) #print("Total time mins: ", total_time_minutes) #print("Highest length mins", highest_length_minutes) #print(targets_minutes) current_target_index = 0 target_minutes = targets_minutes[current_target_index] for length in range(1, highest_length_minutes+1): included_bouts = [b for b in bouts if b.length.total_seconds()/60 <= length] #print(included_bouts) total_included_time_minutes = Bout.total_time(included_bouts).total_seconds()/60 #print(length, total_included_time_minutes) while total_included_time_minutes >= target_minutes: #print(">target_minutes", target_minutes) #length is the result results.append(length) current_target_index += 1 if current_target_index == len(targets_minutes): target_minutes = 999999999 else: target_minutes = targets_minutes[current_target_index] if current_target_index == len(targets_minutes): break #print(results) return results
def test_extracted_bouts(): one_bouts = counts.bouts(1,1) zero_bouts = counts.bouts(0,0) # Bouts where counts == 0 and counts == 1 should be mutually excluse # So there should be no intersections between them intersections = Bout.bout_list_intersection(one_bouts, zero_bouts) assert(len(intersections) == 0) # A bout that spans the whole time period should completely intersect with bouts where counts == 1 one_big_bout = Bout.Bout(counts.timestamps[0]-timedelta(days=1), counts.timestamps[-1]+timedelta(days=1)) one_intersections = Bout.bout_list_intersection(one_bouts, [one_big_bout]) assert(Bout.total_time(one_intersections) == Bout.total_time(one_bouts)) # Same for zeros zero_intersections = Bout.bout_list_intersection(zero_bouts, [one_big_bout]) assert(Bout.total_time(zero_intersections) == Bout.total_time(zero_bouts)) # Filling in the bout gaps of one bouts should recreate the zero bouts inverse_of_one_bouts = Bout.time_period_minus_bouts((counts.timeframe[0], counts.timeframe[1]+timedelta(minutes=1)), one_bouts) # They should have the same n assert(len(inverse_of_one_bouts) == len(zero_bouts)) # Same total amount of time assert(Bout.total_time(inverse_of_one_bouts) == Bout.total_time(zero_bouts))
def test_nonwear_positions(): # Case 1: Nonwear at very beginning of file ts1, header1 = data_loading.load(os.path.abspath(__file__).replace(os.path.basename(__file__), "") + "_data/testfile23.dat", "Actigraph", datetime_format="%d/%m/%Y") counts1 = ts1.get_channel("AG_Counts") nonwear_bouts1, wear_bouts1 = channel_inference.infer_nonwear_actigraph(counts1) # Case 2: Nonwear in middle of file ts2, header2 = data_loading.load(os.path.abspath(__file__).replace(os.path.basename(__file__), "") + "_data/testfile24.dat", "Actigraph", datetime_format="%d/%m/%Y") counts2 = ts2.get_channel("AG_Counts") nonwear_bouts2, wear_bouts2 = channel_inference.infer_nonwear_actigraph(counts2) # Case 3: Nonwear at very end of file ts3, header3 = data_loading.load(os.path.abspath(__file__).replace(os.path.basename(__file__), "") + "_data/testfile25.dat", "Actigraph", datetime_format="%d/%m/%Y") counts3 = ts3.get_channel("AG_Counts") nonwear_bouts3, wear_bouts3 = channel_inference.infer_nonwear_actigraph(counts3) # They should all have the same duration of wear & nonwear assert(Bout.total_time(nonwear_bouts1) == timedelta(hours=2)) assert(Bout.total_time(nonwear_bouts1) == Bout.total_time(nonwear_bouts2)) assert(Bout.total_time(nonwear_bouts1) == Bout.total_time(nonwear_bouts3)) assert(Bout.total_time(wear_bouts1) == Bout.total_time(wear_bouts2)) assert(Bout.total_time(wear_bouts1) == Bout.total_time(wear_bouts3)) # Delete the relevant nonwear bouts from each channel counts1.delete_windows(nonwear_bouts1) counts2.delete_windows(nonwear_bouts2) counts3.delete_windows(nonwear_bouts3) # Total data should be equal assert(sum(counts1.data) == sum(counts2.data)) assert(sum(counts1.data) == sum(counts3.data)) # Summary level mean should also be the same s1 = counts1.summary_statistics()[0] s2 = counts2.summary_statistics()[0] s3 = counts3.summary_statistics()[0] assert(s1.data[0] == s2.data[0]) assert(s1.data[0] == s3.data[0])
def test_bouts(): one_bouts = counts.bouts(1, 1) # There are 7 bouts assert (len(one_bouts) == 7) # Their length is 1, 2, 3, .. 7 assert (Bout.total_time(one_bouts) == timedelta(minutes=7 + 6 + 5 + 4 + 3 + 2 + 1)) # Keeping bouts >= i minutes means there should be 7-(i-1) left for i in range(1, 7): i_or_longer = Bout.limit_to_lengths(one_bouts, min_length=timedelta(minutes=i)) assert (len(i_or_longer) == 7 - (i - 1)) # One manual check three_or_longer = Bout.limit_to_lengths(one_bouts, min_length=timedelta(minutes=3)) assert (len(three_or_longer) == 5) # This should exclude the 1 bout at exactly 3 minutes three_plus_bit_or_longer = Bout.limit_to_lengths(one_bouts, min_length=timedelta( minutes=3, seconds=1)) assert (len(three_plus_bit_or_longer) == 4) # No bouts should be this long eight_or_longer = Bout.limit_to_lengths(one_bouts, min_length=timedelta(minutes=8)) assert (len(eight_or_longer) == 0) # There is nothing above 1 in the file, should be 0 bouts two_bouts = counts.bouts(2, 989) assert (len(two_bouts) == 0)
def qc_analysis(job_details): id_num = str(job_details["pid"]) filename = job_details["filename"] filename_short = os.path.basename(filename).split('.')[0] battery_max = 0 if filetype == "GeneActiv": battery_max = GA_battery_max elif filetype == "Axivity": battery_max = AX_battery_max # Load the data from the hdf5 file ts, header = data_loading.fast_load(filename, filetype) header["QC_filename"] = os.path.basename(filename) x, y, z, battery, temperature = ts.get_channels(["X", "Y", "Z", "Battery", "Temperature"]) # create a channel of battery percentage, based on the assumed battery maximum value battery_pct = Channel.Channel.clone(battery) battery_pct.data = (battery.data / battery_max) * 100 channels = [x, y, z, battery, temperature, battery_pct] anomalies = diagnostics.diagnose_fix_anomalies(channels, discrepancy_threshold=2) # create dictionary of anomalies types anomalies_dict = dict() # check whether any anomalies have been found: if len(anomalies) > 0: anomalies_file = os.path.join(anomalies_folder, "{}_anomalies.csv".format(filename_short)) df = pd.DataFrame(anomalies) for type in anomaly_types: anomalies_dict["QC_anomaly_{}".format(type)] = (df.anomaly_type.values == type).sum() df = df.set_index("anomaly_type") # print record of anomalies to anomalies_file df.to_csv(anomalies_file) else: for type in anomaly_types: anomalies_dict["QC_anomaly_{}".format(type)] = 0 # check for axis anomalies axes_dict = diagnostics.diagnose_axes(x, y, z, noise_cutoff_mg=13) axis_anomaly = False for key, val in axes_dict.items(): anomalies_dict["QC_{}".format(key)] = val if key.endswith("max"): if val > axis_max: axis_anomaly = True elif key.endswith("min"): if val < axis_min: axis_anomaly = True # create a "check battery" flag: check_battery = False # calculate first and last battery percentages first_battery_pct = round((battery_pct.data[1]),2) last_battery_pct = round((battery_pct.data[-1]),2) header["QC_first_battery_pct"] = first_battery_pct header["QC_last_battery_pct"] = last_battery_pct # calculate lowest battery percentage # check if battery.pct has a missing_value, exclude those values if they exist if battery_pct.missing_value == "None": lowest_battery_pct = min(battery_pct.data) else: test_array = np.delete(battery_pct.data, np.where(battery_pct.data == battery_pct.missing_value)) lowest_battery_pct = min(test_array) header["QC_lowest_battery_pct"] = round(lowest_battery_pct,2) header["QC_lowest_battery_threshold"] = battery_minimum # find the maximum battery discharge in any 24hr period: max_discharge = battery_pct.channel_max_decrease(time_period=timedelta(hours=discharge_hours)) header["QC_max_discharge"] = round(max_discharge, 2) header["QC_discharge_time_period"] = "{} hours".format(discharge_hours) header["QC_discharge_threshold"] = discharge_pct # change flag if lowest battery percentage dips below battery_minimum at any point # OR maximum discharge greater than discharge_pct over time period "hours = discharge_hours" if lowest_battery_pct < battery_minimum or max_discharge > discharge_pct: check_battery = True header["QC_check_battery"] = str(check_battery) header["QC_axis_anomaly"] = str(axis_anomaly) # Calculate the time frame to use start = time_utilities.start_of_day(x.timeframe[0]) end = time_utilities.end_of_day(x.timeframe[-1]) tp = (start, end) results_ts = Time_Series.Time_Series("") # Derive some signal features vm = channel_inference.infer_vector_magnitude(x, y, z) enmo = channel_inference.infer_enmo(vm) enmo.minimum = 0 enmo.maximum = enmo_max # Infer nonwear nonwear_bouts = channel_inference.infer_nonwear_for_qc(x, y, z, noise_cutoff_mg=noise_cutoff_mg) # Use nonwear bouts to calculate wear bouts wear_bouts = Bout.time_period_minus_bouts(enmo.timeframe, nonwear_bouts) # Use wear bouts to calculate the amount of wear time in the file in hours, save to meta data total_wear = Bout.total_time(wear_bouts) total_seconds_wear = total_wear.total_seconds() total_hours_wear = round(total_seconds_wear/3600) header["QC_total_hours_wear"] = total_hours_wear # Split the enmo channel into lists of bouts for each quadrant: ''' quadrant_0 = 00:00 -> 06: 00 quadrant_1 = 06:00 -> 12: 00 quadrant_2 = 12:00 -> 18: 00 quadrant_3 = 18:00 -> 00: 00 ''' q_0, q_1, q_2, q_3 = channel_inference.create_quadrant_bouts(enmo) # calculate the intersection of each set of bouts with wear_bouts, then calculate the wear time in each quadrant. sum_quadrant_wear = 0 for quadrant, name1, name2 in ([q_0, "QC_hours_wear_quadrant_0", "QC_pct_wear_quadrant_0"], [q_1, "QC_hours_wear_quadrant_1", "QC_pct_wear_quadrant_1"], [q_2, "QC_hours_wear_quadrant_2", "QC_pct_wear_quadrant_2"], [q_3, "QC_hours_wear_quadrant_3", "QC_pct_wear_quadrant_3"]): quadrant_wear = Bout.bout_list_intersection(quadrant, wear_bouts) seconds_wear = Bout.total_time(quadrant_wear).total_seconds() hours_wear = round(seconds_wear / 3600) header[name1] = hours_wear header[name2] = round(((hours_wear / total_hours_wear) * 100), 2) for bout in nonwear_bouts: # Show non-wear bouts in purple bout.draw_properties = {'lw': 0, 'alpha': 0.75, 'facecolor': '#764af9'} for channel, channel_name in zip([enmo, battery_pct],["ENMO", "Battery_percentage"]): channel.name = channel_name results_ts.add_channel(channel) if PLOT == "YES": # Plot statistics as subplots in one plot file per data file results_ts["ENMO"].add_annotations(nonwear_bouts) results_ts.draw_qc(plotting_df, file_target=os.path.join(charts_folder,"{}_plots.png".format(filename_short))) header["QC_script"] = version # file of metadata from qc process qc_output = os.path.join(results_folder, "qc_meta_{}.csv".format(filename_short)) # check if qc_output already exists... if os.path.isfile(qc_output): os.remove(qc_output) metadata = {**header, **anomalies_dict} # write metadata to file pampro_utilities.dict_write(qc_output, id_num, metadata) for c in ts: del c.data del c.timestamps del c.indices del c.cached_indices
def calibrate(x,y,z, allow_overwrite=True, budget=1000, noise_cutoff_mg=13): """ Use still bouts in the given triaxial data to calibrate it and return the calibrated channels """ calibration_diagnostics = OrderedDict() vm = channel_inference.infer_vector_magnitude(x,y,z) # Get a list of bouts where standard deviation in each axis is below given threshold ("still") still_bouts = channel_inference.infer_still_bouts_triaxial(x,y,z, noise_cutoff_mg=noise_cutoff_mg, minimum_length=timedelta(minutes=1)) num_still_bouts = len(still_bouts) num_still_seconds = Bout.total_time(still_bouts).total_seconds() # Summarise VM in 10s intervals vm_windows = vm.piecewise_statistics(timedelta(seconds=10), [("generic", ["mean"])], time_period=vm.timeframe)[0] # Get a list where VM was between 0.5 and 1.5g ("reasonable") reasonable_bouts = vm_windows.bouts(0.5, 1.5) num_reasonable_bouts = len(reasonable_bouts) num_reasonable_seconds = Bout.total_time(reasonable_bouts).total_seconds() # We only want still bouts where the VM level was within 0.5g of 1g # Therefore insersect "still" time with "reasonable" time still_bouts = Bout.bout_list_intersection(reasonable_bouts, still_bouts) # And we only want bouts where it was still and reasonable for 10s or longer still_bouts = Bout.limit_to_lengths(still_bouts, min_length = timedelta(seconds=10)) num_final_bouts = len(still_bouts) num_final_seconds = Bout.total_time(still_bouts).total_seconds() # Get the average X,Y,Z for each still bout (inside which, by definition, XYZ should not change) still_x, num_samples = x.build_statistics_channels(still_bouts, [("generic", ["mean", "n"])]) still_y = y.build_statistics_channels(still_bouts, [("generic", ["mean"])])[0] still_z = z.build_statistics_channels(still_bouts, [("generic", ["mean"])])[0] # Get the octant positions of the points to calibrate on occupancy = octant_occupancy(still_x.data, still_y.data, still_z.data) # Are they fairly distributed? comparisons = {"x<0":[0,1,2,3], "x>0":[4,5,6,7], "y<0":[0,1,4,5], "y>0":[2,3,6,7], "z<0":[0,2,4,6], "z>0":[1,3,5,7]} for axis in ["x", "y", "z"]: mt = sum(occupancy[comparisons[axis + ">0"]]) lt = sum(occupancy[comparisons[axis + "<0"]]) calibration_diagnostics[axis + "_inequality"] = abs(mt-lt)/sum(occupancy) # Calculate the initial error without doing any calibration start_error = evaluate_solution(still_x, still_y, still_z, num_samples, [0,1,0,1,0,1]) # Do offset and scale calibration by default offset_only_calibration = False calibration_diagnostics["calibration_method"] = "offset and scale" # If we have less than 500 points to calibrate with, or if more than 2 octants are empty if len(still_x.data) < 500 or sum(occupancy == 0) > 2: offset_only_calibration = True calibration_diagnostics["calibration_method"] = "offset only" # Search for the correct way to calibrate the data calibration_parameters = find_calibration_parameters(still_x.data, still_y.data, still_z.data, offset_only=offset_only_calibration) for param,value in zip("x_offset,x_scale,y_offset,y_scale,z_offset,z_scale".split(","), calibration_parameters): calibration_diagnostics[param] = value for i,occ in enumerate(occupancy): calibration_diagnostics["octant_"+str(i)] = occ # Calculate the final error after calibration end_error = evaluate_solution(still_x, still_y, still_z, num_samples, calibration_parameters) calibration_diagnostics["start_error"] = start_error calibration_diagnostics["end_error"] = end_error calibration_diagnostics["num_final_bouts"] = num_final_bouts calibration_diagnostics["num_final_seconds"] = num_final_seconds calibration_diagnostics["num_still_bouts"] = num_still_bouts calibration_diagnostics["num_still_seconds"] = num_still_seconds calibration_diagnostics["num_reasonable_bouts"] = num_reasonable_bouts calibration_diagnostics["num_reasonable_seconds"] = num_reasonable_seconds if allow_overwrite: # If we do not need to preserve the original x,y,z values, we can just calibrate that data # Apply the best calibration factors to the data do_calibration(x, y, z, calibration_parameters) return (x, y, z, calibration_diagnostics) else: # Else we create an independent copy of the raw data and calibrate that instead cal_x = copy.deepcopy(x) cal_y = copy.deepcopy(y) cal_z = copy.deepcopy(z) # Apply the best calibration factors to the data do_calibration(cal_x, cal_y, cal_z, calibration_parameters) return (cal_x, cal_y, cal_z, calibration_diagnostics)