def evaluate_solution(still_x, still_y, still_z, still_n, calibration_parameters): """ Calculates the RMSE of the input XYZ signal if calibrated according to input calibration parameters""" # Temporarily adjust the channels of still data, which has collapsed x,y,z values do_calibration(still_x, still_y, still_z, calibration_parameters) # Get the VM of the calibrated channel vm = channel_inference.infer_vector_magnitude(still_x, still_y, still_z) # se = sum error se = 0.0 for vm_val,n in zip(vm.data, still_n.data): se += (abs(1.0 - vm_val)**2)*n rmse = math.sqrt(se / len(vm.data)) # Undo the temporary calibration undo_calibration(still_x, still_y, still_z, calibration_parameters) return rmse
from datetime import datetime, date, time, timedelta from pampro import data_loading, Time_Series, Channel, channel_inference, triaxial_calibration # Change filenames as appropriate # Read the data - yields 1 channel per axis ts, header = data_loading.load("/pa/data/BIOBANK/example.cwa", "Axivity") x, y, z = ts.get_channels(["X", "Y", "Z"]) # Autocalibrate the raw acceleration data x, y, z, calibration_diagnostics = triaxial_calibration.calibrate(x, y, z) # Infer some sample level information - Vector Magnitude (VM), Euclidean Norm Minus One (ENMO) vm = channel_inference.infer_vector_magnitude(x, y, z) enmo = channel_inference.infer_enmo(vm) # Create a time series object and add channels to it ts.add_channels([vm, enmo]) # Uncomment this line to write the raw data as CSV #ts.write_channels_to_file("C:/Data/3.csv") # Request some interesting statistics - mean of ENMO stats = {"ENMO":[("generic", ["mean"])]} # Get the above statistics on an hourly level - returned as channels hourly_results = ts.piecewise_statistics(timedelta(hours=1), statistics=stats) # Write the hourly analysis to a file
from datetime import datetime, date, time, timedelta from pampro import Time_Series, Channel, channel_inference, triaxial_calibration # Change the filenames as appropriate # Load sample activPAL data x, y, z = Channel.load_channels( "/pa/data/STVS/_data/activpal_data/714952C-AP1335893 18Nov13 10-00am for 7d 23h 14m.datx", "activPAL") # Autocalibrate the raw acceleration data x, y, z, (cal_params), (results), (misc) = triaxial_calibration.calibrate( x, y, z) # Infer some sample level info from the three channels - VM, ENMO, Pitch & Roll vm = channel_inference.infer_vector_magnitude(x, y, z) enmo = channel_inference.infer_enmo(vm) pitch, roll = channel_inference.infer_pitch_roll(x, y, z) # Create a time series object and add all signals to it ts = Time_Series.Time_Series("activPAL") ts.add_channels([x, y, z, vm, enmo, pitch, roll]) # Request some stats about the time series # In this case: mean ENMO, pitch and roll, and 10 degree cutpoints of pitch and roll angle_levels = [[-90, -80], [-80, -70], [-70, -60], [-60, -50], [-50, -40], [-40, -30], [-30, -20], [-20, -10], [-10, 0], [0, 10], [10, 20], [20, 30], [30, 40], [40, 50], [50, 60], [60, 70], [70, 80], [80, 90]] stat_dict = { "Pitch": angle_levels + ["mean"],
def qc_analysis(job_details): id_num = str(job_details["pid"]) filename = job_details["filename"] filename_short = os.path.basename(filename).split('.')[0] battery_max = 0 if filetype == "GeneActiv": battery_max = GA_battery_max elif filetype == "Axivity": battery_max = AX_battery_max # Load the data from the hdf5 file ts, header = data_loading.fast_load(filename, filetype) header["QC_filename"] = os.path.basename(filename) x, y, z, battery, temperature = ts.get_channels(["X", "Y", "Z", "Battery", "Temperature"]) # create a channel of battery percentage, based on the assumed battery maximum value battery_pct = Channel.Channel.clone(battery) battery_pct.data = (battery.data / battery_max) * 100 channels = [x, y, z, battery, temperature, battery_pct] anomalies = diagnostics.diagnose_fix_anomalies(channels, discrepancy_threshold=2) # create dictionary of anomalies types anomalies_dict = dict() # check whether any anomalies have been found: if len(anomalies) > 0: anomalies_file = os.path.join(anomalies_folder, "{}_anomalies.csv".format(filename_short)) df = pd.DataFrame(anomalies) for type in anomaly_types: anomalies_dict["QC_anomaly_{}".format(type)] = (df.anomaly_type.values == type).sum() df = df.set_index("anomaly_type") # print record of anomalies to anomalies_file df.to_csv(anomalies_file) else: for type in anomaly_types: anomalies_dict["QC_anomaly_{}".format(type)] = 0 # check for axis anomalies axes_dict = diagnostics.diagnose_axes(x, y, z, noise_cutoff_mg=13) axis_anomaly = False for key, val in axes_dict.items(): anomalies_dict["QC_{}".format(key)] = val if key.endswith("max"): if val > axis_max: axis_anomaly = True elif key.endswith("min"): if val < axis_min: axis_anomaly = True # create a "check battery" flag: check_battery = False # calculate first and last battery percentages first_battery_pct = round((battery_pct.data[1]),2) last_battery_pct = round((battery_pct.data[-1]),2) header["QC_first_battery_pct"] = first_battery_pct header["QC_last_battery_pct"] = last_battery_pct # calculate lowest battery percentage # check if battery.pct has a missing_value, exclude those values if they exist if battery_pct.missing_value == "None": lowest_battery_pct = min(battery_pct.data) else: test_array = np.delete(battery_pct.data, np.where(battery_pct.data == battery_pct.missing_value)) lowest_battery_pct = min(test_array) header["QC_lowest_battery_pct"] = round(lowest_battery_pct,2) header["QC_lowest_battery_threshold"] = battery_minimum # find the maximum battery discharge in any 24hr period: max_discharge = battery_pct.channel_max_decrease(time_period=timedelta(hours=discharge_hours)) header["QC_max_discharge"] = round(max_discharge, 2) header["QC_discharge_time_period"] = "{} hours".format(discharge_hours) header["QC_discharge_threshold"] = discharge_pct # change flag if lowest battery percentage dips below battery_minimum at any point # OR maximum discharge greater than discharge_pct over time period "hours = discharge_hours" if lowest_battery_pct < battery_minimum or max_discharge > discharge_pct: check_battery = True header["QC_check_battery"] = str(check_battery) header["QC_axis_anomaly"] = str(axis_anomaly) # Calculate the time frame to use start = time_utilities.start_of_day(x.timeframe[0]) end = time_utilities.end_of_day(x.timeframe[-1]) tp = (start, end) results_ts = Time_Series.Time_Series("") # Derive some signal features vm = channel_inference.infer_vector_magnitude(x, y, z) enmo = channel_inference.infer_enmo(vm) enmo.minimum = 0 enmo.maximum = enmo_max # Infer nonwear nonwear_bouts = channel_inference.infer_nonwear_for_qc(x, y, z, noise_cutoff_mg=noise_cutoff_mg) # Use nonwear bouts to calculate wear bouts wear_bouts = Bout.time_period_minus_bouts(enmo.timeframe, nonwear_bouts) # Use wear bouts to calculate the amount of wear time in the file in hours, save to meta data total_wear = Bout.total_time(wear_bouts) total_seconds_wear = total_wear.total_seconds() total_hours_wear = round(total_seconds_wear/3600) header["QC_total_hours_wear"] = total_hours_wear # Split the enmo channel into lists of bouts for each quadrant: ''' quadrant_0 = 00:00 -> 06: 00 quadrant_1 = 06:00 -> 12: 00 quadrant_2 = 12:00 -> 18: 00 quadrant_3 = 18:00 -> 00: 00 ''' q_0, q_1, q_2, q_3 = channel_inference.create_quadrant_bouts(enmo) # calculate the intersection of each set of bouts with wear_bouts, then calculate the wear time in each quadrant. sum_quadrant_wear = 0 for quadrant, name1, name2 in ([q_0, "QC_hours_wear_quadrant_0", "QC_pct_wear_quadrant_0"], [q_1, "QC_hours_wear_quadrant_1", "QC_pct_wear_quadrant_1"], [q_2, "QC_hours_wear_quadrant_2", "QC_pct_wear_quadrant_2"], [q_3, "QC_hours_wear_quadrant_3", "QC_pct_wear_quadrant_3"]): quadrant_wear = Bout.bout_list_intersection(quadrant, wear_bouts) seconds_wear = Bout.total_time(quadrant_wear).total_seconds() hours_wear = round(seconds_wear / 3600) header[name1] = hours_wear header[name2] = round(((hours_wear / total_hours_wear) * 100), 2) for bout in nonwear_bouts: # Show non-wear bouts in purple bout.draw_properties = {'lw': 0, 'alpha': 0.75, 'facecolor': '#764af9'} for channel, channel_name in zip([enmo, battery_pct],["ENMO", "Battery_percentage"]): channel.name = channel_name results_ts.add_channel(channel) if PLOT == "YES": # Plot statistics as subplots in one plot file per data file results_ts["ENMO"].add_annotations(nonwear_bouts) results_ts.draw_qc(plotting_df, file_target=os.path.join(charts_folder,"{}_plots.png".format(filename_short))) header["QC_script"] = version # file of metadata from qc process qc_output = os.path.join(results_folder, "qc_meta_{}.csv".format(filename_short)) # check if qc_output already exists... if os.path.isfile(qc_output): os.remove(qc_output) metadata = {**header, **anomalies_dict} # write metadata to file pampro_utilities.dict_write(qc_output, id_num, metadata) for c in ts: del c.data del c.timestamps del c.indices del c.cached_indices
def process_file(job_details): id_num = str(job_details["pid"]) filename = job_details["filename"] filename_short = os.path.basename(filename).split('.')[0] meta = os.path.join(results_folder, "metadata_{}.csv".format(filename_short)) # check if analysis_meta already exists... if os.path.isfile(meta): os.remove(meta) battery_max = 0 if monitor_type == "GeneActiv": battery_max = GA_battery_max elif monitor_type == "Axivity": battery_max = AX_battery_max epochs = [timedelta(minutes=n) for n in epoch_minutes] # Use 'epochs_minutes' variable to create the corresponding names to the epochs defined names = [] plots_list = [] for n in epoch_minutes: name = "" if n % 60 == 0: # If the epoch is a multiple of 60, it will be named in hours, e.g. '1h' name = "{}h".format(int(n / 60)) elif n % 60 != 0: # If the epoch is NOT a multiple of 60, it will be named in seconds, e.g. '15m' name = "{}m".format(n) names.append(name) if n in epoch_plot: plots_list.append(name) # fast-load the data to identify any anomalies: qc_ts, qc_header = data_loading.fast_load(filename, monitor_type) qc_channels = qc_ts.get_channels(["X", "Y", "Z"]) anomalies = diagnostics.diagnose_fix_anomalies(qc_channels, discrepancy_threshold=2) # Load the data ts, header = data_loading.load(filename, monitor_type, compress=False) header["processed_file"] = os.path.basename(filename) # some monitors have manufacturers parameters applied to them, let's preserve these but rename them: var_list = [ "x_gain", "x_offset", "y_gain", "y_offset", "z_gain", "z_offset", "calibration_date" ] for var in var_list: if var in header.keys(): header[("manufacturers_%s" % var)] = header[var] header.pop(var) x, y, z, battery, temperature, integrity = ts.get_channels( ["X", "Y", "Z", "Battery", "Temperature", "Integrity"]) initial_channels = [x, y, z, battery, temperature, integrity] # create dictionary of anomalies total and types anomalies_dict = {"QC_anomalies_total": len(anomalies)} # check whether any anomalies have been found: if len(anomalies) > 0: anomalies_file = os.path.join( results_folder, "{}_anomalies.csv".format(filename_short)) df = pd.DataFrame(anomalies) for type in anomaly_types: anomalies_dict["QC_anomaly_{}".format(type)] = ( df.anomaly_type.values == type).sum() df = df.set_index("anomaly_type") # print record of anomalies to anomalies_file df.to_csv(anomalies_file) # if anomalies have been found, fix these anomalies channels = diagnostics.fix_anomalies(anomalies, initial_channels) else: for type in anomaly_types: anomalies_dict["QC_anomaly_{}".format(type)] = 0 # if no anomalies channels = initial_channels first_channel = channels[0] # Convert timestamps to offsets from the first timestamp start, offsets = Channel.timestamps_to_offsets(first_channel.timestamps) # As timestamps are sparse, expand them to 1 per observation offsets = Channel.interpolate_offsets(offsets, len(first_channel.data)) # For each channel, convert to offset timestamps for c in channels: c.start = start c.set_contents(c.data, offsets, timestamp_policy="offset") # find approximate first and last battery percentage values first_battery_pct = round((battery.data[1] / battery_max) * 100, 2) last_battery_pct = round((battery.data[-1] / battery_max) * 100, 2) # Calculate the time frame to use start = time_utilities.start_of_day(x.timeframe[0]) end = time_utilities.end_of_day(x.timeframe[-1]) tp = (start, end) # if the sampling frequency is greater than 40Hz if x.frequency > 40: # apply a low pass filter x = pampro_fourier.low_pass_filter(x, 20, frequency=x.frequency, order=4) x.name = "X" # because LPF^ changes the name, we want to override that y = pampro_fourier.low_pass_filter(y, 20, frequency=y.frequency, order=4) y.name = "Y" z = pampro_fourier.low_pass_filter(z, 20, frequency=z.frequency, order=4) z.name = "Z" # find any bouts where data is "missing" BEFORE calibration missing_bouts = [] if -111 in x.data: # extract the bouts of the data channels where the data == -111 (the missing value) missing = x.bouts(-111, -111) # add a buffer of 2 minutes (120 seconds) to the beginning and end of each bout for item in missing: bout_start = max(item.start_timestamp - timedelta(seconds=120), x.timeframe[0]) bout_end = min(item.end_timestamp + timedelta(seconds=120), x.timeframe[1]) new_bout = Bout.Bout(start_timestamp=bout_start, end_timestamp=bout_end) missing_bouts.append(new_bout) else: pass x.delete_windows(missing_bouts) y.delete_windows(missing_bouts) z.delete_windows(missing_bouts) integrity.fill_windows(missing_bouts, fill_value=1) ################ CALIBRATION ####################### # extract still bouts calibration_ts, calibration_header = triaxial_calibration.calibrate_stepone( x, y, z, noise_cutoff_mg=noise_cutoff_mg) # Calibrate the acceleration to local gravity cal_diagnostics = triaxial_calibration.calibrate_steptwo( calibration_ts, calibration_header, calibration_statistics=False) # calibrate data triaxial_calibration.do_calibration(x, y, z, temperature=None, cp=cal_diagnostics) x.delete_windows(missing_bouts) y.delete_windows(missing_bouts) z.delete_windows(missing_bouts) temperature.delete_windows(missing_bouts) battery.delete_windows(missing_bouts) # Derive some signal features vm = channel_inference.infer_vector_magnitude(x, y, z) vm.delete_windows(missing_bouts) if "HPFVM" in stats: vm_hpf = channel_inference.infer_vm_hpf(vm) else: vm_hpf = None if "ENMO" in stats: enmo = channel_inference.infer_enmo(vm) else: enmo = None if "PITCH" and "ROLL" in stats: pitch, roll = channel_inference.infer_pitch_roll(x, y, z) else: pitch = roll = None # Infer nonwear and mask those data points in the signal nonwear_bouts = channel_inference.infer_nonwear_triaxial( x, y, z, noise_cutoff_mg=noise_cutoff_mg) for bout in nonwear_bouts: # Show non-wear bouts in purple bout.draw_properties = {'lw': 0, 'alpha': 0.75, 'facecolor': '#764af9'} for channel, channel_name in zip( [enmo, vm_hpf, pitch, roll, temperature, battery], ["ENMO", "HPFVM", "PITCH", "ROLL", "Temperature", "Battery"]): if channel_name in stats: # Collapse the sample data to a processing epoch (in seconds) so data is summarised epoch_level_channel = channel.piecewise_statistics( timedelta(seconds=processing_epoch), time_period=tp)[0] epoch_level_channel.name = channel_name if channel_name in ["Temperature", "Battery"]: pass else: epoch_level_channel.delete_windows(nonwear_bouts) epoch_level_channel.delete_windows(missing_bouts) ts.add_channel(epoch_level_channel) # collapse binary integrity channel epoch_level_channel = integrity.piecewise_statistics( timedelta(seconds=int(processing_epoch)), statistics=[("binary", ["flag"])], time_period=tp)[0] epoch_level_channel.name = "Integrity" epoch_level_channel.fill_windows(missing_bouts, fill_value=1) ts.add_channel(epoch_level_channel) # create and open results files results_files = [ os.path.join(results_folder, "{}_{}.csv".format(name, filename_short)) for name in names ] files = [open(file, "w") for file in results_files] # Write the column headers to the created files for f in files: f.write(pampro_utilities.design_file_header(stats) + "\n") # writing out and plotting results for epoch, name, f in zip(epochs, names, files): results_ts = ts.piecewise_statistics(epoch, statistics=stats, time_period=tp, name=id_num) results_ts.write_channels_to_file(file_target=f) f.flush() if name in plots_list: # for each statistic in the plotting dictionary, produce a plot in the results folder for stat, plot in plotting_dict.items(): try: results_ts[stat].add_annotations(nonwear_bouts) results_ts.draw([[stat]], file_target=os.path.join( results_folder, plot.format(filename_short, name))) except KeyError: pass header["processing_script"] = version header["analysis_resolutions"] = names header["noise_cutoff_mg"] = noise_cutoff_mg header["processing_epoch"] = processing_epoch header["QC_first_battery_pct"] = first_battery_pct header["QC_last_battery_pct"] = last_battery_pct metadata = {**header, **anomalies_dict, **cal_diagnostics} # write metadata to file pampro_utilities.dict_write(meta, id_num, metadata) for c in ts: del c.data del c.timestamps del c.indices del c.cached_indices
def calibrate(x,y,z, allow_overwrite=True, budget=1000, noise_cutoff_mg=13): """ Use still bouts in the given triaxial data to calibrate it and return the calibrated channels """ calibration_diagnostics = OrderedDict() vm = channel_inference.infer_vector_magnitude(x,y,z) # Get a list of bouts where standard deviation in each axis is below given threshold ("still") still_bouts = channel_inference.infer_still_bouts_triaxial(x,y,z, noise_cutoff_mg=noise_cutoff_mg, minimum_length=timedelta(minutes=1)) num_still_bouts = len(still_bouts) num_still_seconds = Bout.total_time(still_bouts).total_seconds() # Summarise VM in 10s intervals vm_windows = vm.piecewise_statistics(timedelta(seconds=10), [("generic", ["mean"])], time_period=vm.timeframe)[0] # Get a list where VM was between 0.5 and 1.5g ("reasonable") reasonable_bouts = vm_windows.bouts(0.5, 1.5) num_reasonable_bouts = len(reasonable_bouts) num_reasonable_seconds = Bout.total_time(reasonable_bouts).total_seconds() # We only want still bouts where the VM level was within 0.5g of 1g # Therefore insersect "still" time with "reasonable" time still_bouts = Bout.bout_list_intersection(reasonable_bouts, still_bouts) # And we only want bouts where it was still and reasonable for 10s or longer still_bouts = Bout.limit_to_lengths(still_bouts, min_length = timedelta(seconds=10)) num_final_bouts = len(still_bouts) num_final_seconds = Bout.total_time(still_bouts).total_seconds() # Get the average X,Y,Z for each still bout (inside which, by definition, XYZ should not change) still_x, num_samples = x.build_statistics_channels(still_bouts, [("generic", ["mean", "n"])]) still_y = y.build_statistics_channels(still_bouts, [("generic", ["mean"])])[0] still_z = z.build_statistics_channels(still_bouts, [("generic", ["mean"])])[0] # Get the octant positions of the points to calibrate on occupancy = octant_occupancy(still_x.data, still_y.data, still_z.data) # Are they fairly distributed? comparisons = {"x<0":[0,1,2,3], "x>0":[4,5,6,7], "y<0":[0,1,4,5], "y>0":[2,3,6,7], "z<0":[0,2,4,6], "z>0":[1,3,5,7]} for axis in ["x", "y", "z"]: mt = sum(occupancy[comparisons[axis + ">0"]]) lt = sum(occupancy[comparisons[axis + "<0"]]) calibration_diagnostics[axis + "_inequality"] = abs(mt-lt)/sum(occupancy) # Calculate the initial error without doing any calibration start_error = evaluate_solution(still_x, still_y, still_z, num_samples, [0,1,0,1,0,1]) # Do offset and scale calibration by default offset_only_calibration = False calibration_diagnostics["calibration_method"] = "offset and scale" # If we have less than 500 points to calibrate with, or if more than 2 octants are empty if len(still_x.data) < 500 or sum(occupancy == 0) > 2: offset_only_calibration = True calibration_diagnostics["calibration_method"] = "offset only" # Search for the correct way to calibrate the data calibration_parameters = find_calibration_parameters(still_x.data, still_y.data, still_z.data, offset_only=offset_only_calibration) for param,value in zip("x_offset,x_scale,y_offset,y_scale,z_offset,z_scale".split(","), calibration_parameters): calibration_diagnostics[param] = value for i,occ in enumerate(occupancy): calibration_diagnostics["octant_"+str(i)] = occ # Calculate the final error after calibration end_error = evaluate_solution(still_x, still_y, still_z, num_samples, calibration_parameters) calibration_diagnostics["start_error"] = start_error calibration_diagnostics["end_error"] = end_error calibration_diagnostics["num_final_bouts"] = num_final_bouts calibration_diagnostics["num_final_seconds"] = num_final_seconds calibration_diagnostics["num_still_bouts"] = num_still_bouts calibration_diagnostics["num_still_seconds"] = num_still_seconds calibration_diagnostics["num_reasonable_bouts"] = num_reasonable_bouts calibration_diagnostics["num_reasonable_seconds"] = num_reasonable_seconds if allow_overwrite: # If we do not need to preserve the original x,y,z values, we can just calibrate that data # Apply the best calibration factors to the data do_calibration(x, y, z, calibration_parameters) return (x, y, z, calibration_diagnostics) else: # Else we create an independent copy of the raw data and calibrate that instead cal_x = copy.deepcopy(x) cal_y = copy.deepcopy(y) cal_z = copy.deepcopy(z) # Apply the best calibration factors to the data do_calibration(cal_x, cal_y, cal_z, calibration_parameters) return (cal_x, cal_y, cal_z, calibration_diagnostics)