def test_artificial_bouts(): start_a = datetime.strptime("01/01/2000", "%d/%m/%Y") end_a = start_a + timedelta(hours=1) bout_a = Bout.Bout(start_a, end_a) # Hour long bout assert (bout_a.length == timedelta(hours=1)) start_b = datetime.strptime("01/01/2000", "%d/%m/%Y") end_b = start_a + timedelta(minutes=15) bout_b = Bout.Bout(start_b, end_b) # They share common time assert (bout_a.overlaps(bout_b)) # 15 minutes, to be precise intersection = bout_a.intersection(bout_b) assert (intersection.length == timedelta(minutes=15)) start_c = datetime.strptime("01/02/2000", "%d/%m/%Y") end_c = start_c + timedelta(days=1) bout_c = Bout.Bout(start_c, end_c) # No overlap of those bouts assert (not bout_a.overlaps(bout_c)) # bout_a ends exactly as bout_d starts # there should be no overlap (0 common time) start_d = end_a end_d = start_d + timedelta(minutes=1) bout_d = Bout.Bout(start_d, end_d) assert (not bout_a.overlaps(bout_d))
def summary_statistics(self, statistics=[("generic", "mean")], time_period=False, name=""): if time_period == False: windows = [Bout(self.timeframe[0], self.timeframe[1]+timedelta(days=1111))] else: windows = [Bout(time_period[0],time_period[1])] return self.build_statistics_channels(windows, statistics, name=name)
def test_f(): # Case F # Multiple deletions producing consistent results origin = counts.timestamps[0] # Delete first 2 hours start = origin end = origin + timedelta(hours=2) # Summarise the data before deletion summary_before = Time_Series.Time_Series("") summary_before.add_channels( counts.summary_statistics(statistics=[( "generic", ["sum", "n", "missing"]), ("cutpoints", [[0, 0], [0, 1], [1, 1]])])) counts.delete_windows([Bout.Bout(start, end)]) # Summarise the data after deletion summary_after_a = Time_Series.Time_Series("") summary_after_a.add_channels( counts.summary_statistics(statistics=[( "generic", ["sum", "n", "missing"]), ("cutpoints", [[0, 0], [0, 1], [1, 1]])])) # Delete midday to 2pm start = origin + timedelta(hours=12) end = origin + timedelta(hours=14) counts.delete_windows([Bout.Bout(start, end)]) # Summarise the data after deletion summary_after_b = Time_Series.Time_Series("") summary_after_b.add_channels( counts.summary_statistics(statistics=[( "generic", ["sum", "n", "missing"]), ("cutpoints", [[0, 0], [0, 1], [1, 1]])])) # 20 hours left assert (summary_after_b.get_channel("AG_Counts_n").data[0] == 20 * 60) # 4 hours missing assert (summary_after_b.get_channel("AG_Counts_missing").data[0] == 4 * 60) # Sum data should be 20 1s assert (summary_after_b.get_channel("AG_Counts_sum").data[0] == 20 * 60)
def piecewise_statistics(self, window_size, statistics=[("generic", "mean")], time_period=False, name=""): if time_period == False: start = self.timeframe[0] - timedelta(hours=self.timeframe[0].hour, minutes=self.timeframe[0].minute, seconds=self.timeframe[0].second, microseconds=self.timeframe[0].microsecond) end = self.timeframe[1] + timedelta(hours=23-self.timeframe[1].hour, minutes=59-self.timeframe[1].minute, seconds=59-self.timeframe[1].second, microseconds=999999-self.timeframe[1].microsecond) else: start = time_period[0] end = time_period[1] #print("Piecewise statistics: {}".format(self.name)) windows = [] start_dts = start end_dts = start + window_size while start_dts < end: window = Bout(start_dts, end_dts) windows.append(window) start_dts = start_dts + window_size end_dts = end_dts + window_size return self.build_statistics_channels(windows, statistics, name=name)
def test_a(): # Case A # Both timestamps preceed data origin = counts.timestamps[0] start = origin - timedelta(days=2) end = origin - timedelta(days=1) # Summarise the data before deletion summary_before = Time_Series.Time_Series("") summary_before.add_channels( counts.summary_statistics(statistics=[( "generic", ["sum", "n", "missing"]), ("cutpoints", [[0, 0], [0, 1], [1, 1]])])) counts.delete_windows([Bout.Bout(start, end)]) # Summarise the data after deletion summary_after = Time_Series.Time_Series("") summary_after.add_channels( counts.summary_statistics(statistics=[( "generic", ["sum", "n", "missing"]), ("cutpoints", [[0, 0], [0, 1], [1, 1]])])) # All values should be identical, loop through them and assert equality suffixes = "sum n missing 0_0 0_1 1_1".split(" ") for suffix in suffixes: assert (summary_before.get_channel("AG_Counts_" + suffix).data[0] == summary_after.get_channel("AG_Counts_" + suffix).data[0])
def test_extracted_bouts(): one_bouts = counts.bouts(1, 1) zero_bouts = counts.bouts(0, 0) # Bouts where counts == 0 and counts == 1 should be mutually excluse # So there should be no intersections between them intersections = Bout.bout_list_intersection(one_bouts, zero_bouts) assert (len(intersections) == 0) # A bout that spans the whole time period should completely intersect with bouts where counts == 1 one_big_bout = Bout.Bout(counts.timestamps[0] - timedelta(days=1), counts.timestamps[-1] + timedelta(days=1)) one_intersections = Bout.bout_list_intersection(one_bouts, [one_big_bout]) assert (Bout.total_time(one_intersections) == Bout.total_time(one_bouts)) # Same for zeros zero_intersections = Bout.bout_list_intersection(zero_bouts, [one_big_bout]) assert (Bout.total_time(zero_intersections) == Bout.total_time(zero_bouts)) # Filling in the bout gaps of one bouts should recreate the zero bouts inverse_of_one_bouts = Bout.time_period_minus_bouts( (counts.timeframe[0], counts.timeframe[1] + timedelta(minutes=1)), one_bouts) # They should have the same n assert (len(inverse_of_one_bouts) == len(zero_bouts)) # Same total amount of time assert ( Bout.total_time(inverse_of_one_bouts) == Bout.total_time(zero_bouts))
def window_statistics(self, start_dts, end_dts, statistics): window = Bout(start_dts, end_dts) bouts = self.bouts_involved(window) output_row = [] if (len(bouts) > 0): for stat in statistics: if stat[0] == "generic": for val1 in stat[1]: if val1 == "sum": intersection = bout_list_intersection([window],bouts) cache_lengths(intersection) sum_seconds = total_time(intersection).total_seconds() output_row.append(sum_seconds) elif val1 == "mean": intersection = bout_list_intersection([window],bouts) cache_lengths(intersection) sum_seconds = total_time(intersection).total_seconds() if sum_seconds >0 and len(bouts) > 0: output_row.append( sum_seconds / len(bouts) ) else: output_row.append(0) elif val1 == "n": output_row.append( len(bouts) ) else: print("nooooooooo") print(stat) print(statistics) output_row.append(-1) elif stat[0] == "sdx": # ("sdx", [10,20,30,40,50,60,70,80,90]) sdx_results = sdx(bouts, stat[1]) for r in sdx_results: output_row.append(r) else: # No bouts in this Bout_Collection overlapping this window # There was no data for the time period # Output -1 for each missing variable for i in range(self.expected_results(statistics)): output_row.append(-1) return output_row
def test_b(): # Case B # First timestamp preceeds data, second doesn't origin = counts.timestamps[0] start = origin - timedelta(hours=12) end = origin + timedelta(hours=12) # Summarise the data before deletion summary_before = Time_Series.Time_Series("") summary_before.add_channels( counts.summary_statistics(statistics=[( "generic", ["sum", "n", "missing"]), ("cutpoints", [[0, 0], [0, 1], [1, 1]])])) counts.delete_windows([Bout.Bout(start, end)]) # Summarise the data after deletion summary_after = Time_Series.Time_Series("") summary_after.add_channels( counts.summary_statistics(statistics=[( "generic", ["sum", "n", "missing"]), ("cutpoints", [[0, 0], [0, 1], [1, 1]])])) # n should go down and missing should go up assert (summary_before.get_channel("AG_Counts_n").data[0] > summary_after.get_channel("AG_Counts_n").data[0]) assert (summary_before.get_channel("AG_Counts_missing").data[0] < summary_after.get_channel("AG_Counts_missing").data[0]) # Should only be 12 hours left assert (summary_after.get_channel("AG_Counts_n").data[0] == 12 * 60) # And 12 hours missing assert (summary_after.get_channel("AG_Counts_missing").data[0] == 12 * 60)
def test_c(): # Case C # Both timestamps inside data origin = counts.timestamps[0] start = origin + timedelta(hours=6) end = origin + timedelta(hours=7) # Summarise the data before deletion summary_before = Time_Series.Time_Series("") summary_before.add_channels( counts.summary_statistics(statistics=[( "generic", ["sum", "n", "missing"]), ("cutpoints", [[0, 0], [0, 1], [1, 1]])])) counts.delete_windows([Bout.Bout(start, end)]) # Summarise the data after deletion summary_after = Time_Series.Time_Series("") summary_after.add_channels( counts.summary_statistics(statistics=[( "generic", ["sum", "n", "missing"]), ("cutpoints", [[0, 0], [0, 1], [1, 1]])])) # n should go down and missing should go up assert (summary_before.get_channel("AG_Counts_n").data[0] > summary_after.get_channel("AG_Counts_n").data[0]) assert (summary_before.get_channel("AG_Counts_missing").data[0] < summary_after.get_channel("AG_Counts_missing").data[0]) # Should only be 23 hours left assert (summary_after.get_channel("AG_Counts_n").data[0] == 23 * 60) # And 1 hours missing assert (summary_after.get_channel("AG_Counts_missing").data[0] == 1 * 60)
def process_file(job_details): id_num = str(job_details["pid"]) filename = job_details["filename"] filename_short = os.path.basename(filename).split('.')[0] meta = os.path.join(results_folder, "metadata_{}.csv".format(filename_short)) # check if analysis_meta already exists... if os.path.isfile(meta): os.remove(meta) battery_max = 0 if monitor_type == "GeneActiv": battery_max = GA_battery_max elif monitor_type == "Axivity": battery_max = AX_battery_max epochs = [timedelta(minutes=n) for n in epoch_minutes] # Use 'epochs_minutes' variable to create the corresponding names to the epochs defined names = [] plots_list = [] for n in epoch_minutes: name = "" if n % 60 == 0: # If the epoch is a multiple of 60, it will be named in hours, e.g. '1h' name = "{}h".format(int(n / 60)) elif n % 60 != 0: # If the epoch is NOT a multiple of 60, it will be named in seconds, e.g. '15m' name = "{}m".format(n) names.append(name) if n in epoch_plot: plots_list.append(name) # fast-load the data to identify any anomalies: qc_ts, qc_header = data_loading.fast_load(filename, monitor_type) qc_channels = qc_ts.get_channels(["X", "Y", "Z"]) anomalies = diagnostics.diagnose_fix_anomalies(qc_channels, discrepancy_threshold=2) # Load the data ts, header = data_loading.load(filename, monitor_type, compress=False) header["processed_file"] = os.path.basename(filename) # some monitors have manufacturers parameters applied to them, let's preserve these but rename them: var_list = [ "x_gain", "x_offset", "y_gain", "y_offset", "z_gain", "z_offset", "calibration_date" ] for var in var_list: if var in header.keys(): header[("manufacturers_%s" % var)] = header[var] header.pop(var) x, y, z, battery, temperature, integrity = ts.get_channels( ["X", "Y", "Z", "Battery", "Temperature", "Integrity"]) initial_channels = [x, y, z, battery, temperature, integrity] # create dictionary of anomalies total and types anomalies_dict = {"QC_anomalies_total": len(anomalies)} # check whether any anomalies have been found: if len(anomalies) > 0: anomalies_file = os.path.join( results_folder, "{}_anomalies.csv".format(filename_short)) df = pd.DataFrame(anomalies) for type in anomaly_types: anomalies_dict["QC_anomaly_{}".format(type)] = ( df.anomaly_type.values == type).sum() df = df.set_index("anomaly_type") # print record of anomalies to anomalies_file df.to_csv(anomalies_file) # if anomalies have been found, fix these anomalies channels = diagnostics.fix_anomalies(anomalies, initial_channels) else: for type in anomaly_types: anomalies_dict["QC_anomaly_{}".format(type)] = 0 # if no anomalies channels = initial_channels first_channel = channels[0] # Convert timestamps to offsets from the first timestamp start, offsets = Channel.timestamps_to_offsets(first_channel.timestamps) # As timestamps are sparse, expand them to 1 per observation offsets = Channel.interpolate_offsets(offsets, len(first_channel.data)) # For each channel, convert to offset timestamps for c in channels: c.start = start c.set_contents(c.data, offsets, timestamp_policy="offset") # find approximate first and last battery percentage values first_battery_pct = round((battery.data[1] / battery_max) * 100, 2) last_battery_pct = round((battery.data[-1] / battery_max) * 100, 2) # Calculate the time frame to use start = time_utilities.start_of_day(x.timeframe[0]) end = time_utilities.end_of_day(x.timeframe[-1]) tp = (start, end) # if the sampling frequency is greater than 40Hz if x.frequency > 40: # apply a low pass filter x = pampro_fourier.low_pass_filter(x, 20, frequency=x.frequency, order=4) x.name = "X" # because LPF^ changes the name, we want to override that y = pampro_fourier.low_pass_filter(y, 20, frequency=y.frequency, order=4) y.name = "Y" z = pampro_fourier.low_pass_filter(z, 20, frequency=z.frequency, order=4) z.name = "Z" # find any bouts where data is "missing" BEFORE calibration missing_bouts = [] if -111 in x.data: # extract the bouts of the data channels where the data == -111 (the missing value) missing = x.bouts(-111, -111) # add a buffer of 2 minutes (120 seconds) to the beginning and end of each bout for item in missing: bout_start = max(item.start_timestamp - timedelta(seconds=120), x.timeframe[0]) bout_end = min(item.end_timestamp + timedelta(seconds=120), x.timeframe[1]) new_bout = Bout.Bout(start_timestamp=bout_start, end_timestamp=bout_end) missing_bouts.append(new_bout) else: pass x.delete_windows(missing_bouts) y.delete_windows(missing_bouts) z.delete_windows(missing_bouts) integrity.fill_windows(missing_bouts, fill_value=1) ################ CALIBRATION ####################### # extract still bouts calibration_ts, calibration_header = triaxial_calibration.calibrate_stepone( x, y, z, noise_cutoff_mg=noise_cutoff_mg) # Calibrate the acceleration to local gravity cal_diagnostics = triaxial_calibration.calibrate_steptwo( calibration_ts, calibration_header, calibration_statistics=False) # calibrate data triaxial_calibration.do_calibration(x, y, z, temperature=None, cp=cal_diagnostics) x.delete_windows(missing_bouts) y.delete_windows(missing_bouts) z.delete_windows(missing_bouts) temperature.delete_windows(missing_bouts) battery.delete_windows(missing_bouts) # Derive some signal features vm = channel_inference.infer_vector_magnitude(x, y, z) vm.delete_windows(missing_bouts) if "HPFVM" in stats: vm_hpf = channel_inference.infer_vm_hpf(vm) else: vm_hpf = None if "ENMO" in stats: enmo = channel_inference.infer_enmo(vm) else: enmo = None if "PITCH" and "ROLL" in stats: pitch, roll = channel_inference.infer_pitch_roll(x, y, z) else: pitch = roll = None # Infer nonwear and mask those data points in the signal nonwear_bouts = channel_inference.infer_nonwear_triaxial( x, y, z, noise_cutoff_mg=noise_cutoff_mg) for bout in nonwear_bouts: # Show non-wear bouts in purple bout.draw_properties = {'lw': 0, 'alpha': 0.75, 'facecolor': '#764af9'} for channel, channel_name in zip( [enmo, vm_hpf, pitch, roll, temperature, battery], ["ENMO", "HPFVM", "PITCH", "ROLL", "Temperature", "Battery"]): if channel_name in stats: # Collapse the sample data to a processing epoch (in seconds) so data is summarised epoch_level_channel = channel.piecewise_statistics( timedelta(seconds=processing_epoch), time_period=tp)[0] epoch_level_channel.name = channel_name if channel_name in ["Temperature", "Battery"]: pass else: epoch_level_channel.delete_windows(nonwear_bouts) epoch_level_channel.delete_windows(missing_bouts) ts.add_channel(epoch_level_channel) # collapse binary integrity channel epoch_level_channel = integrity.piecewise_statistics( timedelta(seconds=int(processing_epoch)), statistics=[("binary", ["flag"])], time_period=tp)[0] epoch_level_channel.name = "Integrity" epoch_level_channel.fill_windows(missing_bouts, fill_value=1) ts.add_channel(epoch_level_channel) # create and open results files results_files = [ os.path.join(results_folder, "{}_{}.csv".format(name, filename_short)) for name in names ] files = [open(file, "w") for file in results_files] # Write the column headers to the created files for f in files: f.write(pampro_utilities.design_file_header(stats) + "\n") # writing out and plotting results for epoch, name, f in zip(epochs, names, files): results_ts = ts.piecewise_statistics(epoch, statistics=stats, time_period=tp, name=id_num) results_ts.write_channels_to_file(file_target=f) f.flush() if name in plots_list: # for each statistic in the plotting dictionary, produce a plot in the results folder for stat, plot in plotting_dict.items(): try: results_ts[stat].add_annotations(nonwear_bouts) results_ts.draw([[stat]], file_target=os.path.join( results_folder, plot.format(filename_short, name))) except KeyError: pass header["processing_script"] = version header["analysis_resolutions"] = names header["noise_cutoff_mg"] = noise_cutoff_mg header["processing_epoch"] = processing_epoch header["QC_first_battery_pct"] = first_battery_pct header["QC_last_battery_pct"] = last_battery_pct metadata = {**header, **anomalies_dict, **cal_diagnostics} # write metadata to file pampro_utilities.dict_write(meta, id_num, metadata) for c in ts: del c.data del c.timestamps del c.indices del c.cached_indices