def build_statistics_channels(self, windows, statistics, name=""): channel_list = [] for stat in statistics: #print(stat) channel_names = design_variable_names(self.name, stat) #print(channel_names) for cn in channel_names: channel_list.append(Channel(cn)) num_expected_results = len(channel_list) for window in windows: results = self.window_statistics(window.start_timestamp, window.end_timestamp, statistics) if len(results) != num_expected_results: raise Exception("Incorrect number of statistics yielded. {} expected, {} given. Channel: {}. Statistics: {}.".format(num_expected_results, len(results), self.name, statistics)) for i in range(len(results)): #print len(results) channel_list[i].append_data(window.start_timestamp, results[i]) for channel in channel_list: channel.calculate_timeframe() channel.data = np.array(channel.data) channel.timestamps = np.array(channel.timestamps) ts = Time_Series(name) ts.add_channels(channel_list) return ts
def infer_nonwear_triaxial(x, y, z, minimum_length=timedelta(hours=1), noise_cutoff_mg=13, return_nonwear_binary=False): ''' Use the 3 channels of triaxial acceleration to infer periods of nonwear ''' # Get an exhaustive list of bouts where the monitor was still x_intersect_y_intersect_z = infer_still_bouts_triaxial(x,y,z, noise_cutoff_mg=noise_cutoff_mg, minimum_length=minimum_length) # Restrict those bouts to only those with a length that exceeds the minimum length criterion x_intersect_y_intersect_z = Bout.limit_to_lengths(x_intersect_y_intersect_z, min_length=minimum_length) # Legacy code - probably going to delete this if return_nonwear_binary: # Create a parallel, binary channel indicating if that time point was in or out of wear nonwear_binary = Channel.channel_from_bouts(x_intersect_y_intersect_z, x.timeframe, False, "nonwear", skeleton=x) return (x_intersect_y_intersect_z, nonwear_binary) else: return x_intersect_y_intersect_z
def produce_binary_channels(bouts, lengths, skeleton_channel): Bout.cache_lengths(bouts) bouts.sort(key=lambda x: x.length, reverse=True) channels = [] for length in lengths: # Drop bouts from list if their length is less than x minutes bouts = Bout.limit_to_lengths(bouts, min_length=length, sorted=True) channel_name = "{}_mt{}".format(skeleton_channel.name,length) # Clone the blank channel, set data to 1 where time is inside any of the bouts skeleton_copy = copy.deepcopy(skeleton_channel) chan = Channel.channel_from_bouts(bouts, False, False, channel_name, skeleton=skeleton_copy) channels.append(chan) return channels
from datetime import datetime, date, time, timedelta from pampro import Time_Series, Channel, channel_inference, triaxial_calibration # Change the filenames as appropriate # Load sample activPAL data x, y, z = Channel.load_channels( "/pa/data/STVS/_data/activpal_data/714952C-AP1335893 18Nov13 10-00am for 7d 23h 14m.datx", "activPAL") # Autocalibrate the raw acceleration data x, y, z, (cal_params), (results), (misc) = triaxial_calibration.calibrate( x, y, z) # Infer some sample level info from the three channels - VM, ENMO, Pitch & Roll vm = channel_inference.infer_vector_magnitude(x, y, z) enmo = channel_inference.infer_enmo(vm) pitch, roll = channel_inference.infer_pitch_roll(x, y, z) # Create a time series object and add all signals to it ts = Time_Series.Time_Series("activPAL") ts.add_channels([x, y, z, vm, enmo, pitch, roll]) # Request some stats about the time series # In this case: mean ENMO, pitch and roll, and 10 degree cutpoints of pitch and roll angle_levels = [[-90, -80], [-80, -70], [-70, -60], [-60, -50], [-50, -40], [-40, -30], [-30, -20], [-20, -10], [-10, 0], [0, 10], [10, 20], [20, 30], [30, 40], [40, 50], [50, 60], [60, 70], [70, 80], [80, 90]] stat_dict = { "Pitch": angle_levels + ["mean"],
from pampro import Channel, Time_Series import numpy as np from datetime import datetime, timedelta import copy import os start = datetime.strptime("01/01/2000 00:00", "%d/%m/%Y %H:%M") data = np.array([0 for i in range(500)] + [1 for i in range(500)]) full_timestamps = np.array( [start + (timedelta(seconds=1) / 100) * i for i in range(len(data))]) c = Channel.Channel("") indices = list(range(0, len(data), 100)) + [999] sparse_timestamps = full_timestamps[indices] def setup_func(): global c global data global full_timestamps global indices global sparse_timestamps c.set_contents(data, sparse_timestamps) c.frequency = 100 c.sparsely_timestamped = True c.indices = indices print(len(data)) print(indices)
from datetime import datetime, date, time, timedelta from pampro import Time_Series, Channel, channel_inference, triaxial_calibration # Change the filenames as appropriate # Load sample activPAL data x, y, z = Channel.load_channels("/pa/data/STVS/_data/activpal_data/714952C-AP1335893 18Nov13 10-00am for 7d 23h 14m.datx", "activPAL") # Autocalibrate the raw acceleration data x, y, z, (cal_params), (results), (misc) = triaxial_calibration.calibrate(x, y, z) # Infer some sample level info from the three channels - VM, ENMO, Pitch & Roll vm = channel_inference.infer_vector_magnitude(x, y, z) enmo = channel_inference.infer_enmo(vm) pitch, roll = channel_inference.infer_pitch_roll(x, y, z) # Create a time series object and add all signals to it ts = Time_Series.Time_Series("activPAL") ts.add_channels([x,y,z,vm,enmo,pitch,roll]) # Request some stats about the time series # In this case: mean ENMO, pitch and roll, and 10 degree cutpoints of pitch and roll angle_levels = [[-90,-80],[-80,-70],[-70,-60],[-60,-50],[-50,-40],[-40,-30],[-30,-20],[-20,-10],[-10,0],[0,10],[10,20],[20,30],[30,40],[40,50],[50,60],[60,70],[70,80],[80,90]] stat_dict = {"Pitch":angle_levels+["mean"], "Roll":angle_levels+["mean"], "ENMO":["mean"]} # Get the output at 15 minute level quarter_hourly_results = ts.piecewise_statistics(timedelta(minutes=15), stat_dict) # Create a time series object to put the results in
from datetime import datetime, date, time, timedelta from pampro import Time_Series, Channel, channel_inference, Bout # Change filenames as appropriate # Request some interesting statistics - mean, min and max of the counts signal # ...plus basic cutpoints for Sedentary, Light, and Moderate to Vigorous stats = {"AG_Counts": [("generic", ["mean", "min", "max"]), ("cutpoints", [[0,99],[100,2999],[3000,99999]])]} # Load Actigraph data counts, header = Channel.load_channels("/pa/data/Tom/pampro/data/example_actigraph.DAT", "Actigraph", datetime_format="%m/%d/%Y") ts = Time_Series.Time_Series("Actigraph") ts.add_channel(counts) # Get a list of bouts where the monitor was & wasn't worn nonwear_bouts = channel_inference.infer_nonwear_actigraph(counts, zero_minutes=timedelta(minutes=90)) # Use that list to get a list of days of valid & invalid time invalid_bouts = channel_inference.infer_valid_days(counts, wear_bouts) # Since the cutpoints defined above only count positive data, negative values will be ignored # Where the monitor wasn't worn, set the count value to -1 # Where the monitor wasn't valid, set the count value to -2 counts.fill_windows(nonwear_bouts, fill_value=-1) counts.fill_windows(nonwear_bouts, fill_value=-2) # Get the summary level results summary_results = ts.summary_statistics(statistics=stats)
import random import copy from pampro import Time_Series, Channel, channel_inference, Bout execution_start = datetime.now() ts = Time_Series.Time_Series("Actiheart") # Load sample Actiheart data filename = os.path.join(os.path.dirname(__file__), '..', 'data\ARBOTW.txt') chans = Channel.load_channels(filename, "Actiheart") #ts.add_channels(chans) activity = chans[0] ecg = chans[1] # Calculate moving averages of the channels ecg_ma = ecg.moving_average(15) activity_ma = activity.moving_average(15) ts.add_channel(ecg_ma) ts.add_channel(activity_ma) blah = activity.time_derivative() blah = blah.moving_average(121) #ts.add_channel(blah)
def process_file(job_details): id_num = str(job_details["pid"]) filename = job_details["filename"] filename_short = os.path.basename(filename).split('.')[0] meta = os.path.join(results_folder, "metadata_{}.csv".format(filename_short)) # check if analysis_meta already exists... if os.path.isfile(meta): os.remove(meta) battery_max = 0 if monitor_type == "GeneActiv": battery_max = GA_battery_max elif monitor_type == "Axivity": battery_max = AX_battery_max epochs = [timedelta(minutes=n) for n in epoch_minutes] # Use 'epochs_minutes' variable to create the corresponding names to the epochs defined names = [] plots_list = [] for n in epoch_minutes: name = "" if n % 60 == 0: # If the epoch is a multiple of 60, it will be named in hours, e.g. '1h' name = "{}h".format(int(n / 60)) elif n % 60 != 0: # If the epoch is NOT a multiple of 60, it will be named in seconds, e.g. '15m' name = "{}m".format(n) names.append(name) if n in epoch_plot: plots_list.append(name) # fast-load the data to identify any anomalies: qc_ts, qc_header = data_loading.fast_load(filename, monitor_type) qc_channels = qc_ts.get_channels(["X", "Y", "Z"]) anomalies = diagnostics.diagnose_fix_anomalies(qc_channels, discrepancy_threshold=2) # Load the data ts, header = data_loading.load(filename, monitor_type, compress=False) header["processed_file"] = os.path.basename(filename) # some monitors have manufacturers parameters applied to them, let's preserve these but rename them: var_list = [ "x_gain", "x_offset", "y_gain", "y_offset", "z_gain", "z_offset", "calibration_date" ] for var in var_list: if var in header.keys(): header[("manufacturers_%s" % var)] = header[var] header.pop(var) x, y, z, battery, temperature, integrity = ts.get_channels( ["X", "Y", "Z", "Battery", "Temperature", "Integrity"]) initial_channels = [x, y, z, battery, temperature, integrity] # create dictionary of anomalies total and types anomalies_dict = {"QC_anomalies_total": len(anomalies)} # check whether any anomalies have been found: if len(anomalies) > 0: anomalies_file = os.path.join( results_folder, "{}_anomalies.csv".format(filename_short)) df = pd.DataFrame(anomalies) for type in anomaly_types: anomalies_dict["QC_anomaly_{}".format(type)] = ( df.anomaly_type.values == type).sum() df = df.set_index("anomaly_type") # print record of anomalies to anomalies_file df.to_csv(anomalies_file) # if anomalies have been found, fix these anomalies channels = diagnostics.fix_anomalies(anomalies, initial_channels) else: for type in anomaly_types: anomalies_dict["QC_anomaly_{}".format(type)] = 0 # if no anomalies channels = initial_channels first_channel = channels[0] # Convert timestamps to offsets from the first timestamp start, offsets = Channel.timestamps_to_offsets(first_channel.timestamps) # As timestamps are sparse, expand them to 1 per observation offsets = Channel.interpolate_offsets(offsets, len(first_channel.data)) # For each channel, convert to offset timestamps for c in channels: c.start = start c.set_contents(c.data, offsets, timestamp_policy="offset") # find approximate first and last battery percentage values first_battery_pct = round((battery.data[1] / battery_max) * 100, 2) last_battery_pct = round((battery.data[-1] / battery_max) * 100, 2) # Calculate the time frame to use start = time_utilities.start_of_day(x.timeframe[0]) end = time_utilities.end_of_day(x.timeframe[-1]) tp = (start, end) # if the sampling frequency is greater than 40Hz if x.frequency > 40: # apply a low pass filter x = pampro_fourier.low_pass_filter(x, 20, frequency=x.frequency, order=4) x.name = "X" # because LPF^ changes the name, we want to override that y = pampro_fourier.low_pass_filter(y, 20, frequency=y.frequency, order=4) y.name = "Y" z = pampro_fourier.low_pass_filter(z, 20, frequency=z.frequency, order=4) z.name = "Z" # find any bouts where data is "missing" BEFORE calibration missing_bouts = [] if -111 in x.data: # extract the bouts of the data channels where the data == -111 (the missing value) missing = x.bouts(-111, -111) # add a buffer of 2 minutes (120 seconds) to the beginning and end of each bout for item in missing: bout_start = max(item.start_timestamp - timedelta(seconds=120), x.timeframe[0]) bout_end = min(item.end_timestamp + timedelta(seconds=120), x.timeframe[1]) new_bout = Bout.Bout(start_timestamp=bout_start, end_timestamp=bout_end) missing_bouts.append(new_bout) else: pass x.delete_windows(missing_bouts) y.delete_windows(missing_bouts) z.delete_windows(missing_bouts) integrity.fill_windows(missing_bouts, fill_value=1) ################ CALIBRATION ####################### # extract still bouts calibration_ts, calibration_header = triaxial_calibration.calibrate_stepone( x, y, z, noise_cutoff_mg=noise_cutoff_mg) # Calibrate the acceleration to local gravity cal_diagnostics = triaxial_calibration.calibrate_steptwo( calibration_ts, calibration_header, calibration_statistics=False) # calibrate data triaxial_calibration.do_calibration(x, y, z, temperature=None, cp=cal_diagnostics) x.delete_windows(missing_bouts) y.delete_windows(missing_bouts) z.delete_windows(missing_bouts) temperature.delete_windows(missing_bouts) battery.delete_windows(missing_bouts) # Derive some signal features vm = channel_inference.infer_vector_magnitude(x, y, z) vm.delete_windows(missing_bouts) if "HPFVM" in stats: vm_hpf = channel_inference.infer_vm_hpf(vm) else: vm_hpf = None if "ENMO" in stats: enmo = channel_inference.infer_enmo(vm) else: enmo = None if "PITCH" and "ROLL" in stats: pitch, roll = channel_inference.infer_pitch_roll(x, y, z) else: pitch = roll = None # Infer nonwear and mask those data points in the signal nonwear_bouts = channel_inference.infer_nonwear_triaxial( x, y, z, noise_cutoff_mg=noise_cutoff_mg) for bout in nonwear_bouts: # Show non-wear bouts in purple bout.draw_properties = {'lw': 0, 'alpha': 0.75, 'facecolor': '#764af9'} for channel, channel_name in zip( [enmo, vm_hpf, pitch, roll, temperature, battery], ["ENMO", "HPFVM", "PITCH", "ROLL", "Temperature", "Battery"]): if channel_name in stats: # Collapse the sample data to a processing epoch (in seconds) so data is summarised epoch_level_channel = channel.piecewise_statistics( timedelta(seconds=processing_epoch), time_period=tp)[0] epoch_level_channel.name = channel_name if channel_name in ["Temperature", "Battery"]: pass else: epoch_level_channel.delete_windows(nonwear_bouts) epoch_level_channel.delete_windows(missing_bouts) ts.add_channel(epoch_level_channel) # collapse binary integrity channel epoch_level_channel = integrity.piecewise_statistics( timedelta(seconds=int(processing_epoch)), statistics=[("binary", ["flag"])], time_period=tp)[0] epoch_level_channel.name = "Integrity" epoch_level_channel.fill_windows(missing_bouts, fill_value=1) ts.add_channel(epoch_level_channel) # create and open results files results_files = [ os.path.join(results_folder, "{}_{}.csv".format(name, filename_short)) for name in names ] files = [open(file, "w") for file in results_files] # Write the column headers to the created files for f in files: f.write(pampro_utilities.design_file_header(stats) + "\n") # writing out and plotting results for epoch, name, f in zip(epochs, names, files): results_ts = ts.piecewise_statistics(epoch, statistics=stats, time_period=tp, name=id_num) results_ts.write_channels_to_file(file_target=f) f.flush() if name in plots_list: # for each statistic in the plotting dictionary, produce a plot in the results folder for stat, plot in plotting_dict.items(): try: results_ts[stat].add_annotations(nonwear_bouts) results_ts.draw([[stat]], file_target=os.path.join( results_folder, plot.format(filename_short, name))) except KeyError: pass header["processing_script"] = version header["analysis_resolutions"] = names header["noise_cutoff_mg"] = noise_cutoff_mg header["processing_epoch"] = processing_epoch header["QC_first_battery_pct"] = first_battery_pct header["QC_last_battery_pct"] = last_battery_pct metadata = {**header, **anomalies_dict, **cal_diagnostics} # write metadata to file pampro_utilities.dict_write(meta, id_num, metadata) for c in ts: del c.data del c.timestamps del c.indices del c.cached_indices
from datetime import datetime, date, time, timedelta from pampro import Time_Series, Channel, channel_inference, Bout # Change filenames as appropriate # Request some interesting statistics - mean, min and max of the counts signal # ...plus basic cutpoints for Sedentary, Light, and Moderate to Vigorous stats = { "AG_Counts": [("generic", ["mean", "min", "max"]), ("cutpoints", [[0, 99], [100, 2999], [3000, 99999]])] } # Load Actigraph data counts, header = Channel.load_channels( "/pa/data/Tom/pampro/data/example_actigraph.DAT", "Actigraph", datetime_format="%m/%d/%Y") ts = Time_Series.Time_Series("Actigraph") ts.add_channel(counts) # Get a list of bouts where the monitor was & wasn't worn nonwear_bouts = channel_inference.infer_nonwear_actigraph( counts, zero_minutes=timedelta(minutes=90)) # Use that list to get a list of days of valid & invalid time invalid_bouts = channel_inference.infer_valid_days(counts, wear_bouts) # Since the cutpoints defined above only count positive data, negative values will be ignored # Where the monitor wasn't worn, set the count value to -1 # Where the monitor wasn't valid, set the count value to -2
from matplotlib.dates import DayLocator, HourLocator, DateFormatter, drange from datetime import datetime, date, time, timedelta from scipy import stats import random import copy from pampro import Time_Series, Channel, channel_inference, Bout execution_start = datetime.now() ts = Time_Series.Time_Series("Actiheart") # Load sample Actiheart data filename = os.path.join(os.path.dirname(__file__), '..', 'data\ARBOTW.txt') chans = Channel.load_channels(filename, "Actiheart") #ts.add_channels(chans) activity = chans[0] ecg = chans[1] # Calculate moving averages of the channels ecg_ma = ecg.moving_average(15) activity_ma = activity.moving_average(15) ts.add_channel(ecg_ma) ts.add_channel(activity_ma) blah = activity.time_derivative() blah = blah.moving_average(121) #ts.add_channel(blah) # Infer sleep from Actiheart channels