def main():
"""
Application entry point responsible for parsing command line requests
"""
parser = argparse.ArgumentParser(
description="A script to plot acc time series data.", add_help=True)
# required
parser.add_argument('timeSeriesFile',
metavar='input file',
type=str,
help="input .csv.gz time series file to plot")
parser.add_argument('plotFile',
metavar='output file',
type=str,
help="output .png file to plot to")
parser.add_argument('--activityModel',
type=str,
default="activityModels/doherty2018-apr20Update.tar",
help="""trained activity model .tar file""")
# check input is ok
if len(sys.argv) < 3:
msg = "\nInvalid input, please enter at least 2 parameters, e.g."
msg += "\npython accPlot.py timeSeries.csv.gz plot.png \n"
accUtils.toScreen(msg)
parser.print_help()
sys.exit(-1)
args = parser.parse_args()
# and then call plot function
plotTimeSeries(args.timeSeriesFile,
args.plotFile,
activityModel=args.activityModel)
def main():
"""
Application entry point responsible for parsing command line requests
"""
parser = argparse.ArgumentParser(
description="A script to plot acc time series data.", add_help=True)
# required
parser.add_argument('timeSeriesFile',
metavar='input file',
type=str,
help="input .csv.gz time series file to plot")
parser.add_argument('plotFile',
metavar='output file',
type=str,
help="output .png file to plot to")
parser.add_argument('--activityModel',
type=str,
default="activityModels/walmsley-nov20.tar",
help="""trained activity model .tar file""")
parser.add_argument('--useRecommendedImputation',
metavar='True/False',
default=True,
type=str2bool,
help="""Highly recommended method to show imputed
missing data (default : %(default)s)""")
parser.add_argument('--imputedLabels',
metavar='True/False',
default=False,
type=str2bool,
help="""If activity classification during imputed
period will be displayed (default : %(default)s)"""
)
parser.add_argument('--imputedLabelsHeight',
metavar='Proportion i.e. 0-1.0',
default=0.9,
type=float,
help="""Proportion of plot labels take
if activity classification during imputed
period will be displayed (default : %(default)s)"""
)
# check input is ok
if len(sys.argv) < 3:
msg = "\nInvalid input, please enter at least 2 parameters, e.g."
msg += "\npython accPlot.py timeSeries.csv.gz plot.png \n"
accUtils.toScreen(msg)
parser.print_help()
sys.exit(-1)
args = parser.parse_args()
# and then call plot function
plotTimeSeries(args.timeSeriesFile,
args.plotFile,
activityModel=args.activityModel,
useRecommendedImputation=args.useRecommendedImputation,
imputedLabels=args.imputedLabels,
imputedLabelsHeight=args.imputedLabelsHeight)
def processInputFileToEpoch(inputFile,
epochFile,
stationaryFile,
summary,
skipCalibration=False,
stationaryStd=13,
xyzIntercept=[0.0, 0.0, 0.0],
xyzSlope=[1.0, 1.0, 1.0],
xyzTemp=[0.0, 0.0, 0.0],
meanTemp=20.0,
rawDataParser="AccelerometerParser",
javaHeapSpace=None,
useFilter=True,
sampleRate=100,
epochPeriod=30,
activityClassification=True,
rawOutput=False,
rawFile=None,
npyOutput=False,
npyFile=None,
startTime=None,
endTime=None,
verbose=False,
timeZoneOffset=0,
csvStartTime=None,
csvSampleRate=None,
csvTimeFormat=None,
csvStartRow=None,
csvXYZTCols=None):
"""Process raw accelerometer file, writing summary epoch stats to file
This is usually achieved by
1) identify 10sec stationary epochs
2) record calibrated axes scale/offset/temp vals + static point stats
3) use calibration coefficients and then write filtered avgVm epochs
to <epochFile> from <inputFile>
:param str inputFile: Input <cwa/cwa.gz/bin/gt3x> raw accelerometer file
:param str epochFile: Output csv.gz file of processed epoch data
:param str stationaryFile: Output/temporary file for calibration
:param dict summary: Output dictionary containing all summary metrics
:param bool skipCalibration: Perform software calibration (process data twice)
:param int stationaryStd: Gravity threshold (in mg units) for stationary vs not
:param list(float) xyzIntercept: Calbiration offset [x, y, z]
:param list(float) xyzSlope: Calbiration slope [x, y, z]
:param list(float) xyzTemp: Calbiration temperature coefficient [x, y, z]
:param float meanTemp: Calibration mean temperature in file
:param str rawDataParser: External helper process to read raw acc file. If a
java class, it must omit .class ending.
:param str javaHeapSpace: Amount of heap space allocated to java subprocesses.
Useful for limiting RAM usage.
:param bool useFilter: Filter ENMOtrunc signal
:param int sampleRate: Resample data to n Hz
:param int epochPeriod: Size of epoch time window (in seconds)
:param bool activityClassification: Extract features for machine learning
:param bool rawOutput: Output calibrated and resampled raw data to a .csv.gz
file? requires ~50MB/day.
:param str rawFile: Output raw data ".csv.gz" filename
:param bool npyOutput: Output calibrated and resampled raw data to a .npy
file? requires ~60MB/day.
:param str npyFile: Output raw data ".npy" filename
:param datetime startTime: Remove data before this time in analysis
:param datetime endTime: Remove data after this time in analysis
:param bool verbose: Print verbose output
:param int timeZoneOffset: timezone difference between configure and deployment (in minutes)
:param datetime csvStartTime: start time for csv file when time column is not available
:param float csvSampleRate: sample rate for csv file when time column is not available
:param str csvTimeFormat: time format for csv file when time column is available
:param int csvStartRow: start row for accelerometer data in csv file
:param str csvXYZTCols: index of column positions for XYZT columns, e.g. "0,1,2,3"
:return: Raw processing summary values written to dict <summary>
:rtype: void
:Example:
>>> import device
>>> summary = {}
>>> device.processInputFileToEpoch('inputFile.cwa', 'epochFile.csv.gz',
'stationary.csv.gz', summary)
<epoch file written to "epochFile.csv.gz", and calibration points to
'stationary.csv.gz'>
"""
summary['file-size'] = os.path.getsize(inputFile)
summary['file-deviceID'] = getDeviceId(inputFile)
useJava = True
javaClassPath = "java:java/JTransforms-3.1-with-dependencies.jar"
staticStdG = stationaryStd / 1000.0 #java expects units of G (not mg)
if xyzIntercept != [0, 0, 0] or xyzSlope != [1, 1, 1
] or xyzTemp != [0, 0, 0]:
skipCalibration = True
print('\nSkipping calibration as input parameter supplied')
if 'omconvert' in rawDataParser:
useJava = False
if useJava:
if not skipCalibration:
# identify 10sec stationary epochs
accUtils.toScreen("=== Calibrating ===")
commandArgs = [
"java", "-classpath", javaClassPath, "-XX:ParallelGCThreads=1",
rawDataParser, inputFile, "outputFile:" + stationaryFile,
"verbose:" + str(verbose), "filter:" + str(useFilter),
"getStationaryBouts:true", "epochPeriod:10",
"stationaryStd:" + str(staticStdG),
"sampleRate:" + str(sampleRate)
]
if javaHeapSpace:
commandArgs.insert(1, javaHeapSpace)
if timeZoneOffset:
commandArgs.append("timeZoneOffset:" + str(timeZoneOffset))
if csvStartTime:
commandArgs.append("csvStartTime:" +
csvStartTime.strftime("%Y-%m-%dT%H:%M"))
if csvSampleRate:
commandArgs.append("csvSampleRate:" + str(csvSampleRate))
if csvTimeFormat:
commandArgs.append("csvTimeFormat:" + str(csvTimeFormat))
if csvStartRow:
commandArgs.append("csvStartRow:" + str(csvStartRow))
if csvXYZTCols:
commandArgs.append("csvXYZTCols:" + str(csvXYZTCols))
# call process to identify stationary epochs
exitCode = call(commandArgs)
if exitCode != 0:
print(commandArgs)
print("Error: java calibration failed, exit ", exitCode)
sys.exit(-6)
# record calibrated axes scale/offset/temp vals + static point stats
getCalibrationCoefs(stationaryFile, summary)
xyzIntercept = [
summary['calibration-xOffset(g)'],
summary['calibration-yOffset(g)'],
summary['calibration-zOffset(g)']
]
xyzSlope = [
summary['calibration-xSlope(g)'],
summary['calibration-ySlope(g)'],
summary['calibration-zSlope(g)']
]
xyzTemp = [
summary['calibration-xTemp(C)'],
summary['calibration-yTemp(C)'],
summary['calibration-zTemp(C)']
]
meanTemp = summary['calibration-meanDeviceTemp(C)']
else:
storeCalibrationParams(summary, xyzIntercept, xyzSlope, xyzTemp,
meanTemp)
summary['quality-calibratedOnOwnData'] = 0
summary['quality-goodCalibration'] = 1
accUtils.toScreen('=== Extracting features ===')
commandArgs = [
"java", "-classpath", javaClassPath, "-XX:ParallelGCThreads=1",
rawDataParser, inputFile, "outputFile:" + epochFile,
"verbose:" + str(verbose), "filter:" + str(useFilter),
"sampleRate:" + str(sampleRate), "xIntercept:" +
str(xyzIntercept[0]), "yIntercept:" + str(xyzIntercept[1]),
"zIntercept:" + str(xyzIntercept[2]), "xSlope:" + str(xyzSlope[0]),
"ySlope:" + str(xyzSlope[1]), "zSlope:" + str(xyzSlope[2]),
"xTemp:" + str(xyzTemp[0]), "yTemp:" + str(xyzTemp[1]),
"zTemp:" + str(xyzTemp[2]), "meanTemp:" + str(meanTemp),
"epochPeriod:" + str(epochPeriod), "rawOutput:" + str(rawOutput),
"rawFile:" + str(rawFile), "npyOutput:" + str(npyOutput),
"npyFile:" + str(npyFile),
"getFeatures:" + str(activityClassification)
]
if javaHeapSpace:
commandArgs.insert(1, javaHeapSpace)
if startTime:
commandArgs.append("startTime:" +
startTime.strftime("%Y-%m-%dT%H:%M"))
if endTime:
commandArgs.append("endTime:" + endTime.strftime("%Y-%m-%dT%H:%M"))
if timeZoneOffset:
commandArgs.append("timeZoneOffset:" + str(timeZoneOffset))
if csvStartTime:
commandArgs.append("csvStartTime:" +
csvStartTime.strftime("%Y-%m-%dT%H:%M"))
if csvSampleRate:
commandArgs.append("csvSampleRate:" + str(csvSampleRate))
if csvTimeFormat:
commandArgs.append("csvTimeFormat:" + str(csvTimeFormat))
if csvStartRow:
commandArgs.append("csvStartRow:" + str(csvStartRow))
if csvXYZTCols:
commandArgs.append("csvXYZTCols:" + str(csvXYZTCols))
exitCode = call(commandArgs)
if exitCode != 0:
print(commandArgs)
print("Error: Java epoch generation failed, exit ", exitCode)
sys.exit(-7)
else:
if not skipCalibration:
commandArgs = [
rawDataParser, inputFile, "-svm-file", epochFile, "-info",
stationaryFile, "-svm-extended", "3", "-calibrate", "1",
"-interpolate-mode", "2", "-svm-mode", "1", "-svm-epoch",
str(epochPeriod), "-svm-filter", "2"
]
else:
calArgs = str(xSlope) + ','
calArgs += str(ySlope) + ','
calArgs += str(zSlope) + ','
calArgs += str(xyzIntercept[0]) + ','
calArgs += str(xyzIntercept[1]) + ','
calArgs += str(xyzIntercept[2]) + ','
calArgs += str(xyzTemp[0]) + ','
calArgs += str(xyzTemp[1]) + ','
calArgs += str(xyzTemp[2]) + ','
calArgs += str(meanTemp)
commandArgs = [
rawDataParser, inputFile, "-svm-file", epochFile, "-info",
stationaryFile, "-svm-extended", "3", "-calibrate", "0",
"-calibration", calArgs, "-interpolate-mode", "2", "-svm-mode",
"1", "-svm-epoch",
str(epochPeriod), "-svm-filter", "2"
]
call(commandArgs)
getOmconvertInfo(stationaryFile, summary)
def processRawFileToEpoch(rawFile, epochFile, stationaryFile, summary,
skipCalibration = False, stationaryStd = 13, xIntercept = 0.0,
yIntercept = 0.0, zIntercept = 0.0, xSlope = 0.0, ySlope = 0.0,
zSlope = 0.0, xTemp = 0.0, yTemp = 0.0, zTemp = 0.0, meanTemp = 20.0,
rawDataParser = "AxivityAx3Epochs", javaHeapSpace = None,
skipFiltering = False, sampleRate= 100, epochPeriod = 30,
useAbs = False, activityClassification = True,
rawOutput = False, rawOutputFile = None, fftOutput = False,
startTime = None, endTime = None,
verbose = False):
"""Process raw accelerometer file, writing summary epoch stats to file
This is usually achieved by
1) identify 10sec stationary epochs
2) record calibrated axes scale/offset/temp vals + static point stats
3) use calibration coefficients and then write filtered avgVm epochs
to <epochFile> from <rawFile>
:param str rawFile: Input <cwa/bin/gt3x> raw accelerometer file
:param str epochFile: Output csv.gz file of processed epoch data
:param str stationaryFile: Output/temporary file for calibration
:param dict summary: Output dictionary containing all summary metrics
:param bool skipCalibration: Perform software calibration (process data twice)
:param int stationaryStd: Gravity threshold (in mg units) for stationary vs not
:param float xIntercept: Calbiration offset x
:param float yIntercept: Calbiration offset y
:param float zIntercept: Calbiration offset z
:param float xSlope: Calbiration slope x
:param float ySlope: Calbiration slope y
:param float zSlope: Calbiration slope z
:param float xTemp: Calbiration temperature coefficient x
:param float yTemp: Calbiration temperature coefficient y
:param float zTemp: Calbiration temperature coefficient z
:param float meanTemp: Calibration mean temperature in file
:param str rawDataParser: External helper process to read raw acc file. If a
java class, it must omit .class ending.
:param str javaHeapSpace: Amount of heap space allocated to java subprocesses.
Useful for limiting RAM usage.
:param bool skipFiltering: Skip filtering stage
:param int sampleRate: Resample data to n Hz
:param int epochPeriod: Size of epoch time window (in seconds)
:param bool useAbs: Use abs(VM) instead of trunc(VM)
:param bool activityClassification: Extract features for machine learning
:param bool rawOutput: Output raw data to a .csv.gz file? requires ~70MB/day.
:param str rawOutputFile: Output raw data ".csv.gz" filename
:param bool fftOutput: Output FFT epochs to a .csv.gz file? requires ~0.1GB/day.
:param datetime startTime: Remove data before this time in analysis
:param datetime endTime: Remove data after this time in analysis
:param bool verbose: Print verbose output
:return: Raw processing summary values written to dict <summary>
:rtype: void
:Example:
>>> import device
>>> summary = {}
>>> device.processRawFileToEpoch('rawFile.cwa', 'epochFile.csv.gz',
'stationary.csv.gz', summary)
<epoch file written to "epochFile.csv.gz", and calibration points to
'stationary.csv.gz'>
"""
summary['file-size'] = os.path.getsize(rawFile)
summary['file-deviceID'] = getDeviceId(rawFile)
useJava = True
javaClassPath = "java:java/JTransforms-3.1-with-dependencies.jar"
staticStdG = stationaryStd / 1000.0 #java expects units of G (not mg)
if 'omconvert' in rawDataParser:
useJava = False
if useJava:
# calibrate axes scale/offset/temp values
if not skipCalibration:
# identify 10sec stationary epochs
accUtils.toScreen('calibrating to file: ' + stationaryFile)
commandArgs = ["java", "-classpath", javaClassPath,
"-XX:ParallelGCThreads=1", rawDataParser, rawFile,
"outputFile:" + stationaryFile,
"verbose:" + str(verbose), "filter:true",
"getStationaryBouts:true", "epochPeriod:10",
"stationaryStd:" + str(staticStdG)]
if javaHeapSpace:
commandArgs.insert(1, javaHeapSpace)
# call process to identify stationary epochs
exitCode = call(commandArgs)
if exitCode != 0:
print("Error: java calibration failed, exit ", exitCode)
sys.exit(-6)
# record calibrated axes scale/offset/temp vals + static point stats
getCalibrationCoefs(stationaryFile, summary)
xIntercept = summary['calibration-xOffset(g)']
yIntercept = summary['calibration-yOffset(g)']
zIntercept = summary['calibration-zOffset(g)']
xSlope = summary['calibration-xSlope(g)']
ySlope = summary['calibration-ySlope(g)']
zSlope = summary['calibration-zSlope(g)']
xTemp = summary['calibration-xTemp(C)']
yTemp = summary['calibration-yTemp(C)']
zTemp = summary['calibration-zTemp(C)']
meanTemp = summary['calibration-meanDeviceTemp(C)']
else:
storeCalibrationParams(summary, xIntercept, yIntercept, zIntercept,
xSlope, ySlope, zSlope, xTemp, yTemp, zTemp, meanTemp)
summary['quality-calibratedOnOwnData'] = 0
summary['quality-goodCalibration'] = 1
# calculate and write filtered avgVm epochs from raw file
commandArgs = ["java", "-classpath", javaClassPath,
"-XX:ParallelGCThreads=1", rawDataParser, rawFile,
"outputFile:" + epochFile, "verbose:" + str(verbose),
"filter:"+str(skipFiltering),
"sampleRate:" + str(sampleRate),
"xIntercept:" + str(xIntercept),
"yIntercept:" + str(yIntercept),
"zIntercept:" + str(zIntercept),
"xSlope:" + str(xSlope),
"ySlope:" + str(ySlope),
"zSlope:" + str(zSlope),
"xTemp:" + str(xTemp),
"yTemp:" + str(yTemp),
"zTemp:" + str(zTemp),
"meanTemp:" + str(meanTemp),
"epochPeriod:" + str(epochPeriod),
"rawOutput:" + str(rawOutput),
"rawOutputFile:" + str(rawOutputFile),
"fftOutput:" + str(fftOutput),
"getEpochCovariance:True",
"getSanDiegoFeatures:" + str(activityClassification),
"getMADFeatures:" + str(activityClassification),
"getAxisMeans:" + str(activityClassification),
"getUnileverFeatures:" + str(activityClassification),
"getEachAxis:" + str(activityClassification),
"get3DFourier:" + str(activityClassification),
"useAbs:" + str(useAbs)]
accUtils.toScreen('epoch generation')
if javaHeapSpace:
commandArgs.insert(1, javaHeapSpace)
if startTime:
commandArgs.append("startTime:" + startTime.strftime("%Y-%m-%dT%H:%M"))
if endTime:
commandArgs.append("endTime:" + endTime.strftime("%Y-%m-%dT%H:%M"))
exitCode = call(commandArgs)
if exitCode != 0:
print("Error: java epoch generation failed, exit ", exitCode)
sys.exit(-7)
else:
if not skipCalibration:
commandArgs = [rawDataParser, rawFile, "-svm-file", epochFile,
"-info", stationaryFile, "-svm-extended", "3",
"-calibrate", "1", "-interpolate-mode", "2",
"-svm-mode", "1", "-svm-epoch", str(epochPeriod),
"-svm-filter", "2"]
else:
calArgs = str(xSlope) + ','
calArgs += str(ySlope) + ','
calArgs += str(zSlope) + ','
calArgs += str(xIntercept) + ','
calArgs += str(yIntercept) + ','
calArgs += str(zIntercept) + ','
calArgs += str(xTemp) + ','
calArgs += str(yTemp) + ','
calArgs += str(zTemp) + ','
calArgs += str(meanTemp)
commandArgs = [rawDataParser, rawFile, "-svm-file",
epochFile, "-info", stationaryFile,
"-svm-extended", "3", "-calibrate", "0",
"-calibration", calArgs, "-interpolate-mode", "2",
"-svm-mode", "1", "-svm-epoch", str(epochPeriod),
"-svm-filter", "2"]
call(commandArgs)
getOmconvertInfo(stationaryFile, summary)
def getActivitySummary(epochFile, nonWearFile, summary,
activityClassification=True, timeZone='Europe/London',
startTime=None, endTime=None,
epochPeriod=30, stationaryStd=13, minNonWearDuration=60,
mgCutPointMVPA=100, mgCutPointVPA=425,
activityModel="activityModels/walmsley-nov20.tar",
intensityDistribution=False, useRecommendedImputation=True,
psd=False, fourierFrequency=False, fourierWithAcc=False, m10l5=False,
verbose=False):
"""Calculate overall activity summary from <epochFile> data
Get overall activity summary from input <epochFile>. This is achieved by
1) get interrupt and data error summary vals
2) check if data occurs at a daylight savings crossover
3) calculate wear-time statistics, and write nonWear episodes to file
4) predict activity from features, and add label column
5) calculate imputation values to replace nan PA metric values
6) calculate empirical cumulative distribution function of vector magnitudes
7) derive main movement summaries (overall, weekday/weekend, and hour)
:param str epochFile: Input csv.gz file of processed epoch data
:param str nonWearFile: Output filename for non wear .csv.gz episodes
:param dict summary: Output dictionary containing all summary metrics
:param bool activityClassification: Perform machine learning of activity states
:param str timeZone: timezone in country/city format to be used for daylight
savings crossover check
:param datetime startTime: Remove data before this time in analysis
:param datetime endTime: Remove data after this time in analysis
:param int epochPeriod: Size of epoch time window (in seconds)
:param int stationaryStd: Threshold (in mg units) for stationary vs not
:param int minNonWearDuration: Minimum duration of nonwear events (minutes)
:param int mgCutPointMVPA: Milli-gravity threshold for moderate intensity activity
:param int mgCutPointVPA: Milli-gravity threshold for vigorous intensity activity
:param str activityModel: Input tar model file which contains random forest
pickle model, HMM priors/transitions/emissions npy files, and npy file
of METS for each activity state
:param bool intensityDistribution: Add intensity outputs to dict <summary>
:param bool useRecommendedImputation: Highly recommended method to impute
missing data using data from other days around the same time
:param bool verbose: Print verbose output
:return: Pandas dataframe of activity epoch data
:rtype: pandas.DataFrame
:return: Activity prediction labels (empty if <activityClassification>==False)
:rtype: list(str)
:return: Write .csv.gz non wear episodes file to <nonWearFile>
:rtype: void
:return: Movement summary values written to dict <summary>
:rtype: void
:Example:
>>> import summariseEpoch
>>> summary = {}
>>> epochData, labels = summariseEpoch.getActivitySummary( "epoch.csv.gz",
"nonWear.csv.gz", summary)
<nonWear file written to "nonWear.csv.gz" and dict "summary" update with outcomes>
"""
accUtils.toScreen("=== Summarizing ===")
if isinstance(epochFile, pd.DataFrame):
e = epochFile
else:
# Use python PANDAS framework to read in and store epochs
e = pd.read_csv(
epochFile, index_col=['time'],
parse_dates=['time'], date_parser=accUtils.date_parser,
)
# Remove data before/after user specified start/end times
rows = e.shape[0]
tz = pytz.timezone(timeZone)
if startTime:
localStartTime = tz.localize(startTime)
e = e[e.index >= localStartTime]
if endTime:
localEndTime = tz.localize(endTime)
e = e[e.index <= localEndTime]
# Quit if no data left
if e.shape[0] == 0:
print("No rows remaining after start/end time removal")
print("Previously there were %d rows, now shape: %s" % (rows, str(e.shape)))
sys.exit(-9)
# Get start & end times
startTime = e.index[0]
endTime = e.index[-1]
summary['file-startTime'] = accUtils.date_strftime(startTime)
summary['file-endTime'] = accUtils.date_strftime(endTime)
summary['file-firstDay(0=mon,6=sun)'] = startTime.weekday()
# Get interrupt and data error summary vals
e = get_interrupts(e, epochPeriod, summary)
# Check daylight savings time crossover
check_daylight_savings_crossovers(e, summary)
# Calculate wear-time statistics, and write nonWear episodes to file
get_wear_time_stats(e, epochPeriod, stationaryStd, minNonWearDuration,
nonWearFile, summary)
# Calculate and include data quality statistics
get_total_reads(e, epochPeriod, summary)
get_clips(e, epochPeriod, summary)
# Predict activity from features, and add label column
if activityClassification:
e, labels = accClassification.activityClassification(e, activityModel)
else:
labels = []
# enmo : Euclidean Norm Minus One
# Trunc : negative values truncated to zero (i.e never negative)
# emmo = 1 - sqrt(x, y, z)
# enmoTrunc = max(enmo, 0)
e['acc'] = e['enmoTrunc'] * 1000 # convert enmoTrunc to milli-G units
# Calculate imputation values to replace nan PA metric values
e = perform_wearTime_imputation(e, verbose)
e['CutPointMVPA'] = e['accImputed'] >= mgCutPointMVPA
e['CutPointVPA'] = e['accImputed'] >= mgCutPointVPA
# Calculate empirical cumulative distribution function of vector magnitudes
if intensityDistribution:
calculateECDF(e, 'acc', summary, useRecommendedImputation)
# Calculate circadian metrics
if psd:
circadianRhythms.calculatePSD(e, epochPeriod, fourierWithAcc, labels, summary)
if fourierFrequency:
circadianRhythms.calculateFourierFreq(e, epochPeriod, fourierWithAcc, labels, summary)
if m10l5:
circadianRhythms.calculateM10L5(e, epochPeriod, summary)
# Main movement summaries
writeMovementSummaries(e, labels, summary, useRecommendedImputation)
# Return physical activity summary
return e, labels
def main(): # noqa: C901
"""
Application entry point responsible for parsing command line requests
"""
parser = argparse.ArgumentParser(
description="A script to plot acc time series data.", add_help=True)
# required
parser.add_argument('timeSeriesFile',
metavar='input file',
type=str,
help="input .csv.gz time series file to plot")
parser.add_argument('--plotFile',
metavar='output file',
type=str,
help="output .png file to plot to")
parser.add_argument('--activityModel',
type=str,
default="walmsley",
help="""trained activity model .tar file""")
parser.add_argument('--useRecommendedImputation',
metavar='True/False',
default=True,
type=str2bool,
help="""Highly recommended method to show imputed
missing data (default : %(default)s)""")
parser.add_argument('--imputedLabels',
metavar='True/False',
default=False,
type=str2bool,
help="""If activity classification during imputed
period will be displayed (default : %(default)s)"""
)
parser.add_argument('--imputedLabelsHeight',
metavar='Proportion i.e. 0-1.0',
default=0.9,
type=float,
help="""Proportion of plot labels take
if activity classification during imputed
period will be displayed (default : %(default)s)"""
)
parser.add_argument('--showFileName',
metavar='True/False',
default=False,
type=str2bool,
help="""Toggle showing filename as title in output
image (default : %(default)s)""")
parser.add_argument('--showFirstNDays',
metavar='days',
default=None,
type=int,
help="Show just first n days")
# check input is ok
if len(sys.argv) < 2:
msg = "\nInvalid input, please enter at least 1 parameter, e.g."
msg += "\npython accPlot.py timeSeries.csv.gz \n"
accUtils.toScreen(msg)
parser.print_help()
sys.exit(-1)
args = parser.parse_args()
# determine output file name
if args.plotFile is None:
inputFileFolder, inputFileName = os.path.split(args.timeSeriesFile)
inputFileName = inputFileName.split('.')[0] # remove any extension
args.plotFile = os.path.join(inputFileFolder,
inputFileName + "-plot.png")
# and then call plot function
plotTimeSeries(args.timeSeriesFile,
args.plotFile,
showFirstNDays=args.showFirstNDays,
activityModel=args.activityModel,
useRecommendedImputation=args.useRecommendedImputation,
imputedLabels=args.imputedLabels,
imputedLabelsHeight=args.imputedLabelsHeight,
showFileName=args.showFileName)
def main():
"""
Application entry point responsible for parsing command line requests
"""
parser = argparse.ArgumentParser(
description="""A tool to extract physical activity information from
raw accelerometer files.""", add_help=True
)
# required
parser.add_argument('rawFile', metavar='input file', type=str,
help="""the .cwa file to process (e.g. sample.cwa).
If the file path contains spaces,it must be enclosed
in quote marks (e.g. \"../My Documents/sample.cwa\")
""")
#optional inputs
parser.add_argument('--startTime',
metavar='e.g. 1991-01-01T23:59', default=None,
type=str2date, help="""removes data before this
time in the final analysis
(default : %(default)s)""")
parser.add_argument('--endTime',
metavar='e.g 1991-01-01T23:59', default=None,
type=str2date, help="""removes data after this
time in the final analysis
(default : %(default)s)""")
parser.add_argument('--timeSeriesDateColumn',
metavar='True/False', default=False, type=str2bool,
help="""adds a date/time column to the timeSeries
file, so acceleration and imputation values can be
compared easily. This increases output filesize
(default : %(default)s)""")
parser.add_argument('--processRawFile',
metavar='True/False', default=True, type=str2bool,
help="""False will skip processing of the .cwa file
(the epoch.csv file must already exist for this to
work) (default : %(default)s)""")
parser.add_argument('--epochPeriod',
metavar='length', default=30, type=int,
help="""length in seconds of a single epoch (default
: %(default)ss, must be an integer)""")
parser.add_argument('--sampleRate',
metavar='Hz, or samples/second', default=100,
type=int, help="""resample data to n Hz (default
: %(default)ss, must be an integer)""")
parser.add_argument('--useAbs',
metavar='useAbs', default=False, type=str2bool,
help="""use abs(VM) instead of trunc(VM)
(default : %(default)s)""")
parser.add_argument('--skipFiltering',
metavar='True/False', default=False, type=str2bool,
help="""Skip filtering stage
(default : %(default)s)""")
# calibration parameters
parser.add_argument('--skipCalibration',
metavar='True/False', default=False, type=str2bool,
help="""skip calibration? (default : %(default)s)""")
parser.add_argument('--calOffset',
metavar=('x', 'y', 'z'),default=[0.0, 0.0, 0.0],
type=float, nargs=3,
help="""accelerometer calibration offset (default :
%(default)s)""")
parser.add_argument('--calSlope',
metavar=('x', 'y', 'z'), default=[1.0, 1.0, 1.0],
type=float, nargs=3,
help="""accelerometer calibration slope linking
offset to temperature (default : %(default)s)""")
parser.add_argument('--calTemp',
metavar=('x', 'y', 'z'), default=[0.0, 0.0, 0.0],
type=float, nargs=3,
help="""mean temperature in degrees Celsius of
stationary data for calibration
(default : %(default)s)""")
parser.add_argument('--meanTemp',
metavar="temp", default=20.0, type=float,
help="""mean calibration temperature in degrees
Celsius (default : %(default)s)""")
parser.add_argument('--stationaryStd',
metavar='mg', default=13, type=int,
help="""stationary mg threshold (default
: %(default)s mg))""")
parser.add_argument('--calibrationSphereCriteria',
metavar='mg', default=0.3, type=float,
help="""calibration sphere threshold (default
: %(default)s mg))""")
# activity parameters
parser.add_argument('--mgMVPA',
metavar="mg", default=100, type=int,
help="""MVPA threshold (default : %(default)s)""")
parser.add_argument('--mgVPA',
metavar="mg", default=425, type=int,
help="""VPA threshold (default : %(default)s)""")
# calling helper processess and conducting multi-threadings
parser.add_argument('--rawDataParser',
metavar="rawDataParser", default="AxivityAx3Epochs",
type=str,
help="""file containing a java program to process
raw .cwa binary file, must end with .class (omitted)
(default : %(default)s)""")
parser.add_argument('--javaHeapSpace',
metavar="amount in MB", default="", type=str,
help="""amount of heap space allocated to the java
subprocesses,useful for limiting RAM usage (default
: unlimited)""")
# activity classification arguments
parser.add_argument('--activityClassification',
metavar='True/False', default=True, type=str2bool,
help="""Use pre-trained random forest to predict
activity type
(default : %(default)s)""")
parser.add_argument('--activityModel', type=str,
default="activityModels/doherty2018.tar",
help="""trained activity model .tar file""")
parser.add_argument('--rawOutput',
metavar='True/False', default=False, type=str2bool,
help="""output raw data to a .csv.gz file? NOTE:
requires ~70MB per day. (default : %(default)s)""")
parser.add_argument('--fftOutput',
metavar='True/False', default=False, type=str2bool,
help="""output FFT epochs to a .csv file? NOTE:
requires ~0.1GB per day. (default : %(default)s)""")
# optional outputs
parser.add_argument('--outputFolder', metavar='filename',default="",
help="""folder for all of the output files, \
unless specified using other options""")
parser.add_argument('--summaryFolder', metavar='filename',default="",
help="folder for -summary.json summary stats")
parser.add_argument('--epochFolder', metavar='filename', default="",
help="""folder -epoch.csv.gz - must be an existing
file if "-processRawFile" is set to False""")
parser.add_argument('--timeSeriesFolder', metavar='filename', default="",
help="folder for -timeSeries.csv.gz file")
parser.add_argument('--nonWearFolder', metavar='filename',default="",
help="folder for -nonWearBouts.csv.gz file")
parser.add_argument('--stationaryFolder', metavar='filename', default="",
help="folder -stationaryPoints.csv.gz file")
parser.add_argument('--rawFolder', metavar='filename', default="",
help="folder for raw .csv.gz file")
parser.add_argument('--verbose',
metavar='True/False', default=False, type=str2bool,
help="""enable verbose logging? (default :
%(default)s)""")
parser.add_argument('--deleteIntermediateFiles',
metavar='True/False', default=True, type=str2bool,
help="""True will remove extra "helper" files created
by the program (default : %(default)s)""")
parser.add_argument('--intensityDistribution',
metavar='True/False', default=False, type=str2bool,
help="""Save intensity distribution
(default : %(default)s)""")
#
# check that enough command line arguments are entered
#
if len(sys.argv) < 2:
msg = "\nInvalid input, please enter at least 1 parameter, e.g."
msg += "\npython ActivitySummary.py inputFile.CWA \n"
accUtils.toScreen(msg)
parser.print_help()
sys.exit(-1)
processingStartTime = datetime.datetime.now()
args = parser.parse_args()
##########################
# check input/output files/dirs exist and validate input args
##########################
if args.processRawFile is False:
if len(args.rawFile.split('.')) < 2:
args.rawFile += ".cwa" # TODO edge case since we still need a name?
elif not os.path.isfile(args.rawFile):
if args.rawFile:
print("error: specified file " + args.rawFile + " does not exist. Exiting..")
else:
print("error: no file specified. Exiting..")
sys.exit(-2)
# get file extension
rawFileNoExt = ('.').join(args.rawFile.split('.')[:-1])
(rawFilePath, rawFileName) = os.path.split(rawFileNoExt)
# check target output folders exist
for path in [args.summaryFolder, args.nonWearFolder, args.epochFolder,
args.stationaryFolder, args.timeSeriesFolder, args.outputFolder]:
if len(path) > 0 and not os.access(path, os.F_OK):
print("error: " + path + " is not a valid directory")
sys.exit(-3)
# assign output file names
if args.outputFolder == "" and rawFilePath != "":
args.outputFolder = rawFilePath + '/'
if args.summaryFolder == "":
args.summaryFolder = args.outputFolder
if args.nonWearFolder == "":
args.nonWearFolder = args.outputFolder
if args.epochFolder == "":
args.epochFolder = args.outputFolder
if args.stationaryFolder == "":
args.stationaryFolder = args.outputFolder
if args.timeSeriesFolder == "":
args.timeSeriesFolder = args.outputFolder
if args.rawFolder == "":
args.rawFolder = args.outputFolder
args.summaryFile = args.summaryFolder + rawFileName + "-summary.json"
args.nonWearFile = args.nonWearFolder + rawFileName + "-nonWearBouts.csv.gz"
args.epochFile = args.epochFolder + rawFileName + "-epoch.csv.gz"
args.stationaryFile = args.stationaryFolder + rawFileName + "-stationaryPoints.csv"
args.tsFile = args.timeSeriesFolder + rawFileName + "-timeSeries.csv.gz"
args.rawOutputFile = args.rawFolder + rawFileName + ".csv.gz"
# check user specified end time is not before start time
if args.startTime and args.endTime:
if args.startTime >= args.endTime:
print("start and end time arguments are invalid!")
print("startTime:", args.startTime.strftime("%Y-%m-%dT%H:%M"))
print("endTime:", args.endTime.strftime("%Y-%m-%dT%H:%M"))
sys.exit(-4)
# print processing options to screen
print("processing file " + args.rawFile + "' with these arguments:\n")
for key, value in sorted(vars(args).items()):
if not (isinstance(value, str) and len(value)==0):
print(key.ljust(15), ':', value)
print("\n")
##########################
# start processing file
##########################
summary = {}
# now process the .CWA file
if args.processRawFile:
summary['file-name'] = args.rawFile
accelerometer.device.processRawFileToEpoch(args.rawFile, args.epochFile,
args.stationaryFile, summary, skipCalibration = args.skipCalibration,
stationaryStd = args.stationaryStd, xIntercept = args.calOffset[0],
yIntercept = args.calOffset[1], zIntercept = args.calOffset[2],
xSlope = args.calSlope[0], ySlope = args.calSlope[1],
zSlope = args.calSlope[2], xTemp = args.calTemp[0],
yTemp = args.calTemp[1], zTemp = args.calTemp[2],
meanTemp = args.meanTemp, rawDataParser = args.rawDataParser,
javaHeapSpace = args.javaHeapSpace, skipFiltering = args.skipFiltering,
sampleRate= args.sampleRate, epochPeriod = args.epochPeriod,
useAbs = args.useAbs, activityClassification = args.activityClassification,
rawOutput = args.rawOutput, rawOutputFile = args.rawOutputFile,
fftOutput = args.fftOutput, startTime = args.startTime,
endTime = args.endTime, verbose = args.verbose)
else:
summary['file-name'] = args.epochFile
# summarise epoch
epochData, labels = accelerometer.summariseEpoch.getActivitySummary(
args.epochFile, args.nonWearFile, summary,
activityClassification = args.activityClassification, startTime = args.startTime,
endTime = args.endTime, epochPeriod = args.epochPeriod,
stationaryStd = args.stationaryStd, mgMVPA = args.mgMVPA,
mgVPA = args.mgVPA, activityModel = args.activityModel,
intensityDistribution = args.intensityDistribution,
verbose = args.verbose)
# generate time series file (note: this will also resample to epochData so do this last)
accelerometer.accUtils.generateTimeSeries(epochData, args.tsFile,
epochPeriod = args.epochPeriod,
timeSeriesDateColumn = args.timeSeriesDateColumn,
activityClassification = args.activityClassification, labels = labels)
# print basic output
summaryVals = ['file-name', 'file-startTime', 'file-endTime', \
'acc-overall-avg','wearTime-overall(days)', \
'nonWearTime-overall(days)', 'quality-goodWearTime']
summaryDict = collections.OrderedDict([(i, summary[i]) for i in summaryVals])
accelerometer.accUtils.toScreen(json.dumps(summaryDict, indent=4))
# write detailed output to file
f = open(args.summaryFile,'w')
json.dump(summary, f, indent=4)
f.close()
accelerometer.accUtils.toScreen('Summary file written to: ' + args.summaryFile)
##########################
# remove helper files and close program
##########################
if args.deleteIntermediateFiles:
try:
os.remove(args.stationaryFile)
os.remove(args.epochFile)
except:
accelerometer.accUtils.toScreen('could not delete helper file')
# finally, print out processing summary message
processingEndTime = datetime.datetime.now()
processingTime = (processingEndTime - processingStartTime).total_seconds()
accelerometer.accUtils.toScreen("in total, processing took " + \
str(processingTime) + " seconds")
