def build_statistics_channels(self, windows, statistics, name=""):
channel_list = []
for stat in statistics:
#print(stat)
channel_names = design_variable_names(self.name, stat)
#print(channel_names)
for cn in channel_names:
channel_list.append(Channel(cn))
num_expected_results = len(channel_list)
for window in windows:
results = self.window_statistics(window.start_timestamp, window.end_timestamp, statistics)
if len(results) != num_expected_results:
raise Exception("Incorrect number of statistics yielded. {} expected, {} given. Channel: {}. Statistics: {}.".format(num_expected_results, len(results), self.name, statistics))
for i in range(len(results)):
#print len(results)
channel_list[i].append_data(window.start_timestamp, results[i])
for channel in channel_list:
channel.calculate_timeframe()
channel.data = np.array(channel.data)
channel.timestamps = np.array(channel.timestamps)
ts = Time_Series(name)
ts.add_channels(channel_list)
return ts
def infer_nonwear_triaxial(x, y, z, minimum_length=timedelta(hours=1), noise_cutoff_mg=13, return_nonwear_binary=False):
''' Use the 3 channels of triaxial acceleration to infer periods of nonwear '''
# Get an exhaustive list of bouts where the monitor was still
x_intersect_y_intersect_z = infer_still_bouts_triaxial(x,y,z, noise_cutoff_mg=noise_cutoff_mg, minimum_length=minimum_length)
# Restrict those bouts to only those with a length that exceeds the minimum length criterion
x_intersect_y_intersect_z = Bout.limit_to_lengths(x_intersect_y_intersect_z, min_length=minimum_length)
# Legacy code - probably going to delete this
if return_nonwear_binary:
# Create a parallel, binary channel indicating if that time point was in or out of wear
nonwear_binary = Channel.channel_from_bouts(x_intersect_y_intersect_z, x.timeframe, False, "nonwear", skeleton=x)
return (x_intersect_y_intersect_z, nonwear_binary)
else:
return x_intersect_y_intersect_z
def produce_binary_channels(bouts, lengths, skeleton_channel):
Bout.cache_lengths(bouts)
bouts.sort(key=lambda x: x.length, reverse=True)
channels = []
for length in lengths:
# Drop bouts from list if their length is less than x minutes
bouts = Bout.limit_to_lengths(bouts, min_length=length, sorted=True)
channel_name = "{}_mt{}".format(skeleton_channel.name,length)
# Clone the blank channel, set data to 1 where time is inside any of the bouts
skeleton_copy = copy.deepcopy(skeleton_channel)
chan = Channel.channel_from_bouts(bouts, False, False, channel_name, skeleton=skeleton_copy)
channels.append(chan)
return channels
from datetime import datetime, date, time, timedelta
from pampro import Time_Series, Channel, channel_inference, triaxial_calibration
# Change the filenames as appropriate
# Load sample activPAL data
x, y, z = Channel.load_channels(
"/pa/data/STVS/_data/activpal_data/714952C-AP1335893 18Nov13 10-00am for 7d 23h 14m.datx",
"activPAL")
# Autocalibrate the raw acceleration data
x, y, z, (cal_params), (results), (misc) = triaxial_calibration.calibrate(
x, y, z)
# Infer some sample level info from the three channels - VM, ENMO, Pitch & Roll
vm = channel_inference.infer_vector_magnitude(x, y, z)
enmo = channel_inference.infer_enmo(vm)
pitch, roll = channel_inference.infer_pitch_roll(x, y, z)
# Create a time series object and add all signals to it
ts = Time_Series.Time_Series("activPAL")
ts.add_channels([x, y, z, vm, enmo, pitch, roll])
# Request some stats about the time series
# In this case: mean ENMO, pitch and roll, and 10 degree cutpoints of pitch and roll
angle_levels = [[-90, -80], [-80, -70], [-70, -60], [-60, -50], [-50, -40],
[-40, -30], [-30, -20], [-20, -10], [-10, 0], [0, 10],
[10, 20], [20, 30], [30, 40], [40, 50], [50, 60], [60, 70],
[70, 80], [80, 90]]
stat_dict = {
"Pitch": angle_levels + ["mean"],
from pampro import Channel, Time_Series
import numpy as np
from datetime import datetime, timedelta
import copy
import os
start = datetime.strptime("01/01/2000 00:00", "%d/%m/%Y %H:%M")
data = np.array([0 for i in range(500)] + [1 for i in range(500)])
full_timestamps = np.array(
[start + (timedelta(seconds=1) / 100) * i for i in range(len(data))])
c = Channel.Channel("")
indices = list(range(0, len(data), 100)) + [999]
sparse_timestamps = full_timestamps[indices]
def setup_func():
global c
global data
global full_timestamps
global indices
global sparse_timestamps
c.set_contents(data, sparse_timestamps)
c.frequency = 100
c.sparsely_timestamped = True
c.indices = indices
print(len(data))
print(indices)
from datetime import datetime, date, time, timedelta
from pampro import Time_Series, Channel, channel_inference, triaxial_calibration
# Change the filenames as appropriate
# Load sample activPAL data
x, y, z = Channel.load_channels("/pa/data/STVS/_data/activpal_data/714952C-AP1335893 18Nov13 10-00am for 7d 23h 14m.datx", "activPAL")
# Autocalibrate the raw acceleration data
x, y, z, (cal_params), (results), (misc) = triaxial_calibration.calibrate(x, y, z)
# Infer some sample level info from the three channels - VM, ENMO, Pitch & Roll
vm = channel_inference.infer_vector_magnitude(x, y, z)
enmo = channel_inference.infer_enmo(vm)
pitch, roll = channel_inference.infer_pitch_roll(x, y, z)
# Create a time series object and add all signals to it
ts = Time_Series.Time_Series("activPAL")
ts.add_channels([x,y,z,vm,enmo,pitch,roll])
# Request some stats about the time series
# In this case: mean ENMO, pitch and roll, and 10 degree cutpoints of pitch and roll
angle_levels = [[-90,-80],[-80,-70],[-70,-60],[-60,-50],[-50,-40],[-40,-30],[-30,-20],[-20,-10],[-10,0],[0,10],[10,20],[20,30],[30,40],[40,50],[50,60],[60,70],[70,80],[80,90]]
stat_dict = {"Pitch":angle_levels+["mean"], "Roll":angle_levels+["mean"], "ENMO":["mean"]}
# Get the output at 15 minute level
quarter_hourly_results = ts.piecewise_statistics(timedelta(minutes=15), stat_dict)
# Create a time series object to put the results in
from datetime import datetime, date, time, timedelta
from pampro import Time_Series, Channel, channel_inference, Bout
# Change filenames as appropriate
# Request some interesting statistics - mean, min and max of the counts signal
# ...plus basic cutpoints for Sedentary, Light, and Moderate to Vigorous
stats = {"AG_Counts": [("generic", ["mean", "min", "max"]), ("cutpoints", [[0,99],[100,2999],[3000,99999]])]}
# Load Actigraph data
counts, header = Channel.load_channels("/pa/data/Tom/pampro/data/example_actigraph.DAT", "Actigraph", datetime_format="%m/%d/%Y")
ts = Time_Series.Time_Series("Actigraph")
ts.add_channel(counts)
# Get a list of bouts where the monitor was & wasn't worn
nonwear_bouts = channel_inference.infer_nonwear_actigraph(counts, zero_minutes=timedelta(minutes=90))
# Use that list to get a list of days of valid & invalid time
invalid_bouts = channel_inference.infer_valid_days(counts, wear_bouts)
# Since the cutpoints defined above only count positive data, negative values will be ignored
# Where the monitor wasn't worn, set the count value to -1
# Where the monitor wasn't valid, set the count value to -2
counts.fill_windows(nonwear_bouts, fill_value=-1)
counts.fill_windows(nonwear_bouts, fill_value=-2)
# Get the summary level results
summary_results = ts.summary_statistics(statistics=stats)
import random
import copy
from pampro import Time_Series, Channel, channel_inference, Bout
execution_start = datetime.now()
ts = Time_Series.Time_Series("Actiheart")
# Load sample Actiheart data
filename = os.path.join(os.path.dirname(__file__), '..', 'data\ARBOTW.txt')
chans = Channel.load_channels(filename, "Actiheart")
#ts.add_channels(chans)
activity = chans[0]
ecg = chans[1]
# Calculate moving averages of the channels
ecg_ma = ecg.moving_average(15)
activity_ma = activity.moving_average(15)
ts.add_channel(ecg_ma)
ts.add_channel(activity_ma)
blah = activity.time_derivative()
blah = blah.moving_average(121)
#ts.add_channel(blah)
def process_file(job_details):
id_num = str(job_details["pid"])
filename = job_details["filename"]
filename_short = os.path.basename(filename).split('.')[0]
meta = os.path.join(results_folder,
"metadata_{}.csv".format(filename_short))
# check if analysis_meta already exists...
if os.path.isfile(meta):
os.remove(meta)
battery_max = 0
if monitor_type == "GeneActiv":
battery_max = GA_battery_max
elif monitor_type == "Axivity":
battery_max = AX_battery_max
epochs = [timedelta(minutes=n) for n in epoch_minutes]
# Use 'epochs_minutes' variable to create the corresponding names to the epochs defined
names = []
plots_list = []
for n in epoch_minutes:
name = ""
if n % 60 == 0: # If the epoch is a multiple of 60, it will be named in hours, e.g. '1h'
name = "{}h".format(int(n / 60))
elif n % 60 != 0: # If the epoch is NOT a multiple of 60, it will be named in seconds, e.g. '15m'
name = "{}m".format(n)
names.append(name)
if n in epoch_plot:
plots_list.append(name)
# fast-load the data to identify any anomalies:
qc_ts, qc_header = data_loading.fast_load(filename, monitor_type)
qc_channels = qc_ts.get_channels(["X", "Y", "Z"])
anomalies = diagnostics.diagnose_fix_anomalies(qc_channels,
discrepancy_threshold=2)
# Load the data
ts, header = data_loading.load(filename, monitor_type, compress=False)
header["processed_file"] = os.path.basename(filename)
# some monitors have manufacturers parameters applied to them, let's preserve these but rename them:
var_list = [
"x_gain", "x_offset", "y_gain", "y_offset", "z_gain", "z_offset",
"calibration_date"
]
for var in var_list:
if var in header.keys():
header[("manufacturers_%s" % var)] = header[var]
header.pop(var)
x, y, z, battery, temperature, integrity = ts.get_channels(
["X", "Y", "Z", "Battery", "Temperature", "Integrity"])
initial_channels = [x, y, z, battery, temperature, integrity]
# create dictionary of anomalies total and types
anomalies_dict = {"QC_anomalies_total": len(anomalies)}
# check whether any anomalies have been found:
if len(anomalies) > 0:
anomalies_file = os.path.join(
results_folder, "{}_anomalies.csv".format(filename_short))
df = pd.DataFrame(anomalies)
for type in anomaly_types:
anomalies_dict["QC_anomaly_{}".format(type)] = (
df.anomaly_type.values == type).sum()
df = df.set_index("anomaly_type")
# print record of anomalies to anomalies_file
df.to_csv(anomalies_file)
# if anomalies have been found, fix these anomalies
channels = diagnostics.fix_anomalies(anomalies, initial_channels)
else:
for type in anomaly_types:
anomalies_dict["QC_anomaly_{}".format(type)] = 0
# if no anomalies
channels = initial_channels
first_channel = channels[0]
# Convert timestamps to offsets from the first timestamp
start, offsets = Channel.timestamps_to_offsets(first_channel.timestamps)
# As timestamps are sparse, expand them to 1 per observation
offsets = Channel.interpolate_offsets(offsets, len(first_channel.data))
# For each channel, convert to offset timestamps
for c in channels:
c.start = start
c.set_contents(c.data, offsets, timestamp_policy="offset")
# find approximate first and last battery percentage values
first_battery_pct = round((battery.data[1] / battery_max) * 100, 2)
last_battery_pct = round((battery.data[-1] / battery_max) * 100, 2)
# Calculate the time frame to use
start = time_utilities.start_of_day(x.timeframe[0])
end = time_utilities.end_of_day(x.timeframe[-1])
tp = (start, end)
# if the sampling frequency is greater than 40Hz
if x.frequency > 40:
# apply a low pass filter
x = pampro_fourier.low_pass_filter(x,
20,
frequency=x.frequency,
order=4)
x.name = "X" # because LPF^ changes the name, we want to override that
y = pampro_fourier.low_pass_filter(y,
20,
frequency=y.frequency,
order=4)
y.name = "Y"
z = pampro_fourier.low_pass_filter(z,
20,
frequency=z.frequency,
order=4)
z.name = "Z"
# find any bouts where data is "missing" BEFORE calibration
missing_bouts = []
if -111 in x.data:
# extract the bouts of the data channels where the data == -111 (the missing value)
missing = x.bouts(-111, -111)
# add a buffer of 2 minutes (120 seconds) to the beginning and end of each bout
for item in missing:
bout_start = max(item.start_timestamp - timedelta(seconds=120),
x.timeframe[0])
bout_end = min(item.end_timestamp + timedelta(seconds=120),
x.timeframe[1])
new_bout = Bout.Bout(start_timestamp=bout_start,
end_timestamp=bout_end)
missing_bouts.append(new_bout)
else:
pass
x.delete_windows(missing_bouts)
y.delete_windows(missing_bouts)
z.delete_windows(missing_bouts)
integrity.fill_windows(missing_bouts, fill_value=1)
################ CALIBRATION #######################
# extract still bouts
calibration_ts, calibration_header = triaxial_calibration.calibrate_stepone(
x, y, z, noise_cutoff_mg=noise_cutoff_mg)
# Calibrate the acceleration to local gravity
cal_diagnostics = triaxial_calibration.calibrate_steptwo(
calibration_ts, calibration_header, calibration_statistics=False)
# calibrate data
triaxial_calibration.do_calibration(x,
y,
z,
temperature=None,
cp=cal_diagnostics)
x.delete_windows(missing_bouts)
y.delete_windows(missing_bouts)
z.delete_windows(missing_bouts)
temperature.delete_windows(missing_bouts)
battery.delete_windows(missing_bouts)
# Derive some signal features
vm = channel_inference.infer_vector_magnitude(x, y, z)
vm.delete_windows(missing_bouts)
if "HPFVM" in stats:
vm_hpf = channel_inference.infer_vm_hpf(vm)
else:
vm_hpf = None
if "ENMO" in stats:
enmo = channel_inference.infer_enmo(vm)
else:
enmo = None
if "PITCH" and "ROLL" in stats:
pitch, roll = channel_inference.infer_pitch_roll(x, y, z)
else:
pitch = roll = None
# Infer nonwear and mask those data points in the signal
nonwear_bouts = channel_inference.infer_nonwear_triaxial(
x, y, z, noise_cutoff_mg=noise_cutoff_mg)
for bout in nonwear_bouts:
# Show non-wear bouts in purple
bout.draw_properties = {'lw': 0, 'alpha': 0.75, 'facecolor': '#764af9'}
for channel, channel_name in zip(
[enmo, vm_hpf, pitch, roll, temperature, battery],
["ENMO", "HPFVM", "PITCH", "ROLL", "Temperature", "Battery"]):
if channel_name in stats:
# Collapse the sample data to a processing epoch (in seconds) so data is summarised
epoch_level_channel = channel.piecewise_statistics(
timedelta(seconds=processing_epoch), time_period=tp)[0]
epoch_level_channel.name = channel_name
if channel_name in ["Temperature", "Battery"]:
pass
else:
epoch_level_channel.delete_windows(nonwear_bouts)
epoch_level_channel.delete_windows(missing_bouts)
ts.add_channel(epoch_level_channel)
# collapse binary integrity channel
epoch_level_channel = integrity.piecewise_statistics(
timedelta(seconds=int(processing_epoch)),
statistics=[("binary", ["flag"])],
time_period=tp)[0]
epoch_level_channel.name = "Integrity"
epoch_level_channel.fill_windows(missing_bouts, fill_value=1)
ts.add_channel(epoch_level_channel)
# create and open results files
results_files = [
os.path.join(results_folder, "{}_{}.csv".format(name, filename_short))
for name in names
]
files = [open(file, "w") for file in results_files]
# Write the column headers to the created files
for f in files:
f.write(pampro_utilities.design_file_header(stats) + "\n")
# writing out and plotting results
for epoch, name, f in zip(epochs, names, files):
results_ts = ts.piecewise_statistics(epoch,
statistics=stats,
time_period=tp,
name=id_num)
results_ts.write_channels_to_file(file_target=f)
f.flush()
if name in plots_list:
# for each statistic in the plotting dictionary, produce a plot in the results folder
for stat, plot in plotting_dict.items():
try:
results_ts[stat].add_annotations(nonwear_bouts)
results_ts.draw([[stat]],
file_target=os.path.join(
results_folder,
plot.format(filename_short, name)))
except KeyError:
pass
header["processing_script"] = version
header["analysis_resolutions"] = names
header["noise_cutoff_mg"] = noise_cutoff_mg
header["processing_epoch"] = processing_epoch
header["QC_first_battery_pct"] = first_battery_pct
header["QC_last_battery_pct"] = last_battery_pct
metadata = {**header, **anomalies_dict, **cal_diagnostics}
# write metadata to file
pampro_utilities.dict_write(meta, id_num, metadata)
for c in ts:
del c.data
del c.timestamps
del c.indices
del c.cached_indices
from datetime import datetime, date, time, timedelta
from pampro import Time_Series, Channel, channel_inference, Bout
# Change filenames as appropriate
# Request some interesting statistics - mean, min and max of the counts signal
# ...plus basic cutpoints for Sedentary, Light, and Moderate to Vigorous
stats = {
"AG_Counts": [("generic", ["mean", "min", "max"]),
("cutpoints", [[0, 99], [100, 2999], [3000, 99999]])]
}
# Load Actigraph data
counts, header = Channel.load_channels(
"/pa/data/Tom/pampro/data/example_actigraph.DAT",
"Actigraph",
datetime_format="%m/%d/%Y")
ts = Time_Series.Time_Series("Actigraph")
ts.add_channel(counts)
# Get a list of bouts where the monitor was & wasn't worn
nonwear_bouts = channel_inference.infer_nonwear_actigraph(
counts, zero_minutes=timedelta(minutes=90))
# Use that list to get a list of days of valid & invalid time
invalid_bouts = channel_inference.infer_valid_days(counts, wear_bouts)
# Since the cutpoints defined above only count positive data, negative values will be ignored
# Where the monitor wasn't worn, set the count value to -1
# Where the monitor wasn't valid, set the count value to -2
from matplotlib.dates import DayLocator, HourLocator, DateFormatter, drange
from datetime import datetime, date, time, timedelta
from scipy import stats
import random
import copy
from pampro import Time_Series, Channel, channel_inference, Bout
execution_start = datetime.now()
ts = Time_Series.Time_Series("Actiheart")
# Load sample Actiheart data
filename = os.path.join(os.path.dirname(__file__), '..', 'data\ARBOTW.txt')
chans = Channel.load_channels(filename, "Actiheart")
#ts.add_channels(chans)
activity = chans[0]
ecg = chans[1]
# Calculate moving averages of the channels
ecg_ma = ecg.moving_average(15)
activity_ma = activity.moving_average(15)
ts.add_channel(ecg_ma)
ts.add_channel(activity_ma)
blah = activity.time_derivative()
blah = blah.moving_average(121)
#ts.add_channel(blah)
# Infer sleep from Actiheart channels
