def main(): """ Application entry point responsible for parsing command line requests """ parser = argparse.ArgumentParser( description="A script to plot acc time series data.", add_help=True) # required parser.add_argument('timeSeriesFile', metavar='input file', type=str, help="input .csv.gz time series file to plot") parser.add_argument('plotFile', metavar='output file', type=str, help="output .png file to plot to") parser.add_argument('--activityModel', type=str, default="activityModels/doherty2018-apr20Update.tar", help="""trained activity model .tar file""") # check input is ok if len(sys.argv) < 3: msg = "\nInvalid input, please enter at least 2 parameters, e.g." msg += "\npython accPlot.py timeSeries.csv.gz plot.png \n" accUtils.toScreen(msg) parser.print_help() sys.exit(-1) args = parser.parse_args() # and then call plot function plotTimeSeries(args.timeSeriesFile, args.plotFile, activityModel=args.activityModel)
def main(): """ Application entry point responsible for parsing command line requests """ parser = argparse.ArgumentParser( description="A script to plot acc time series data.", add_help=True) # required parser.add_argument('timeSeriesFile', metavar='input file', type=str, help="input .csv.gz time series file to plot") parser.add_argument('plotFile', metavar='output file', type=str, help="output .png file to plot to") parser.add_argument('--activityModel', type=str, default="activityModels/walmsley-nov20.tar", help="""trained activity model .tar file""") parser.add_argument('--useRecommendedImputation', metavar='True/False', default=True, type=str2bool, help="""Highly recommended method to show imputed missing data (default : %(default)s)""") parser.add_argument('--imputedLabels', metavar='True/False', default=False, type=str2bool, help="""If activity classification during imputed period will be displayed (default : %(default)s)""" ) parser.add_argument('--imputedLabelsHeight', metavar='Proportion i.e. 0-1.0', default=0.9, type=float, help="""Proportion of plot labels take if activity classification during imputed period will be displayed (default : %(default)s)""" ) # check input is ok if len(sys.argv) < 3: msg = "\nInvalid input, please enter at least 2 parameters, e.g." msg += "\npython accPlot.py timeSeries.csv.gz plot.png \n" accUtils.toScreen(msg) parser.print_help() sys.exit(-1) args = parser.parse_args() # and then call plot function plotTimeSeries(args.timeSeriesFile, args.plotFile, activityModel=args.activityModel, useRecommendedImputation=args.useRecommendedImputation, imputedLabels=args.imputedLabels, imputedLabelsHeight=args.imputedLabelsHeight)
def processInputFileToEpoch(inputFile, epochFile, stationaryFile, summary, skipCalibration=False, stationaryStd=13, xyzIntercept=[0.0, 0.0, 0.0], xyzSlope=[1.0, 1.0, 1.0], xyzTemp=[0.0, 0.0, 0.0], meanTemp=20.0, rawDataParser="AccelerometerParser", javaHeapSpace=None, useFilter=True, sampleRate=100, epochPeriod=30, activityClassification=True, rawOutput=False, rawFile=None, npyOutput=False, npyFile=None, startTime=None, endTime=None, verbose=False, timeZoneOffset=0, csvStartTime=None, csvSampleRate=None, csvTimeFormat=None, csvStartRow=None, csvXYZTCols=None): """Process raw accelerometer file, writing summary epoch stats to file This is usually achieved by 1) identify 10sec stationary epochs 2) record calibrated axes scale/offset/temp vals + static point stats 3) use calibration coefficients and then write filtered avgVm epochs to <epochFile> from <inputFile> :param str inputFile: Input <cwa/cwa.gz/bin/gt3x> raw accelerometer file :param str epochFile: Output csv.gz file of processed epoch data :param str stationaryFile: Output/temporary file for calibration :param dict summary: Output dictionary containing all summary metrics :param bool skipCalibration: Perform software calibration (process data twice) :param int stationaryStd: Gravity threshold (in mg units) for stationary vs not :param list(float) xyzIntercept: Calbiration offset [x, y, z] :param list(float) xyzSlope: Calbiration slope [x, y, z] :param list(float) xyzTemp: Calbiration temperature coefficient [x, y, z] :param float meanTemp: Calibration mean temperature in file :param str rawDataParser: External helper process to read raw acc file. If a java class, it must omit .class ending. :param str javaHeapSpace: Amount of heap space allocated to java subprocesses. Useful for limiting RAM usage. :param bool useFilter: Filter ENMOtrunc signal :param int sampleRate: Resample data to n Hz :param int epochPeriod: Size of epoch time window (in seconds) :param bool activityClassification: Extract features for machine learning :param bool rawOutput: Output calibrated and resampled raw data to a .csv.gz file? requires ~50MB/day. :param str rawFile: Output raw data ".csv.gz" filename :param bool npyOutput: Output calibrated and resampled raw data to a .npy file? requires ~60MB/day. :param str npyFile: Output raw data ".npy" filename :param datetime startTime: Remove data before this time in analysis :param datetime endTime: Remove data after this time in analysis :param bool verbose: Print verbose output :param int timeZoneOffset: timezone difference between configure and deployment (in minutes) :param datetime csvStartTime: start time for csv file when time column is not available :param float csvSampleRate: sample rate for csv file when time column is not available :param str csvTimeFormat: time format for csv file when time column is available :param int csvStartRow: start row for accelerometer data in csv file :param str csvXYZTCols: index of column positions for XYZT columns, e.g. "0,1,2,3" :return: Raw processing summary values written to dict <summary> :rtype: void :Example: >>> import device >>> summary = {} >>> device.processInputFileToEpoch('inputFile.cwa', 'epochFile.csv.gz', 'stationary.csv.gz', summary) <epoch file written to "epochFile.csv.gz", and calibration points to 'stationary.csv.gz'> """ summary['file-size'] = os.path.getsize(inputFile) summary['file-deviceID'] = getDeviceId(inputFile) useJava = True javaClassPath = "java:java/JTransforms-3.1-with-dependencies.jar" staticStdG = stationaryStd / 1000.0 #java expects units of G (not mg) if xyzIntercept != [0, 0, 0] or xyzSlope != [1, 1, 1 ] or xyzTemp != [0, 0, 0]: skipCalibration = True print('\nSkipping calibration as input parameter supplied') if 'omconvert' in rawDataParser: useJava = False if useJava: if not skipCalibration: # identify 10sec stationary epochs accUtils.toScreen("=== Calibrating ===") commandArgs = [ "java", "-classpath", javaClassPath, "-XX:ParallelGCThreads=1", rawDataParser, inputFile, "outputFile:" + stationaryFile, "verbose:" + str(verbose), "filter:" + str(useFilter), "getStationaryBouts:true", "epochPeriod:10", "stationaryStd:" + str(staticStdG), "sampleRate:" + str(sampleRate) ] if javaHeapSpace: commandArgs.insert(1, javaHeapSpace) if timeZoneOffset: commandArgs.append("timeZoneOffset:" + str(timeZoneOffset)) if csvStartTime: commandArgs.append("csvStartTime:" + csvStartTime.strftime("%Y-%m-%dT%H:%M")) if csvSampleRate: commandArgs.append("csvSampleRate:" + str(csvSampleRate)) if csvTimeFormat: commandArgs.append("csvTimeFormat:" + str(csvTimeFormat)) if csvStartRow: commandArgs.append("csvStartRow:" + str(csvStartRow)) if csvXYZTCols: commandArgs.append("csvXYZTCols:" + str(csvXYZTCols)) # call process to identify stationary epochs exitCode = call(commandArgs) if exitCode != 0: print(commandArgs) print("Error: java calibration failed, exit ", exitCode) sys.exit(-6) # record calibrated axes scale/offset/temp vals + static point stats getCalibrationCoefs(stationaryFile, summary) xyzIntercept = [ summary['calibration-xOffset(g)'], summary['calibration-yOffset(g)'], summary['calibration-zOffset(g)'] ] xyzSlope = [ summary['calibration-xSlope(g)'], summary['calibration-ySlope(g)'], summary['calibration-zSlope(g)'] ] xyzTemp = [ summary['calibration-xTemp(C)'], summary['calibration-yTemp(C)'], summary['calibration-zTemp(C)'] ] meanTemp = summary['calibration-meanDeviceTemp(C)'] else: storeCalibrationParams(summary, xyzIntercept, xyzSlope, xyzTemp, meanTemp) summary['quality-calibratedOnOwnData'] = 0 summary['quality-goodCalibration'] = 1 accUtils.toScreen('=== Extracting features ===') commandArgs = [ "java", "-classpath", javaClassPath, "-XX:ParallelGCThreads=1", rawDataParser, inputFile, "outputFile:" + epochFile, "verbose:" + str(verbose), "filter:" + str(useFilter), "sampleRate:" + str(sampleRate), "xIntercept:" + str(xyzIntercept[0]), "yIntercept:" + str(xyzIntercept[1]), "zIntercept:" + str(xyzIntercept[2]), "xSlope:" + str(xyzSlope[0]), "ySlope:" + str(xyzSlope[1]), "zSlope:" + str(xyzSlope[2]), "xTemp:" + str(xyzTemp[0]), "yTemp:" + str(xyzTemp[1]), "zTemp:" + str(xyzTemp[2]), "meanTemp:" + str(meanTemp), "epochPeriod:" + str(epochPeriod), "rawOutput:" + str(rawOutput), "rawFile:" + str(rawFile), "npyOutput:" + str(npyOutput), "npyFile:" + str(npyFile), "getFeatures:" + str(activityClassification) ] if javaHeapSpace: commandArgs.insert(1, javaHeapSpace) if startTime: commandArgs.append("startTime:" + startTime.strftime("%Y-%m-%dT%H:%M")) if endTime: commandArgs.append("endTime:" + endTime.strftime("%Y-%m-%dT%H:%M")) if timeZoneOffset: commandArgs.append("timeZoneOffset:" + str(timeZoneOffset)) if csvStartTime: commandArgs.append("csvStartTime:" + csvStartTime.strftime("%Y-%m-%dT%H:%M")) if csvSampleRate: commandArgs.append("csvSampleRate:" + str(csvSampleRate)) if csvTimeFormat: commandArgs.append("csvTimeFormat:" + str(csvTimeFormat)) if csvStartRow: commandArgs.append("csvStartRow:" + str(csvStartRow)) if csvXYZTCols: commandArgs.append("csvXYZTCols:" + str(csvXYZTCols)) exitCode = call(commandArgs) if exitCode != 0: print(commandArgs) print("Error: Java epoch generation failed, exit ", exitCode) sys.exit(-7) else: if not skipCalibration: commandArgs = [ rawDataParser, inputFile, "-svm-file", epochFile, "-info", stationaryFile, "-svm-extended", "3", "-calibrate", "1", "-interpolate-mode", "2", "-svm-mode", "1", "-svm-epoch", str(epochPeriod), "-svm-filter", "2" ] else: calArgs = str(xSlope) + ',' calArgs += str(ySlope) + ',' calArgs += str(zSlope) + ',' calArgs += str(xyzIntercept[0]) + ',' calArgs += str(xyzIntercept[1]) + ',' calArgs += str(xyzIntercept[2]) + ',' calArgs += str(xyzTemp[0]) + ',' calArgs += str(xyzTemp[1]) + ',' calArgs += str(xyzTemp[2]) + ',' calArgs += str(meanTemp) commandArgs = [ rawDataParser, inputFile, "-svm-file", epochFile, "-info", stationaryFile, "-svm-extended", "3", "-calibrate", "0", "-calibration", calArgs, "-interpolate-mode", "2", "-svm-mode", "1", "-svm-epoch", str(epochPeriod), "-svm-filter", "2" ] call(commandArgs) getOmconvertInfo(stationaryFile, summary)
def processRawFileToEpoch(rawFile, epochFile, stationaryFile, summary, skipCalibration = False, stationaryStd = 13, xIntercept = 0.0, yIntercept = 0.0, zIntercept = 0.0, xSlope = 0.0, ySlope = 0.0, zSlope = 0.0, xTemp = 0.0, yTemp = 0.0, zTemp = 0.0, meanTemp = 20.0, rawDataParser = "AxivityAx3Epochs", javaHeapSpace = None, skipFiltering = False, sampleRate= 100, epochPeriod = 30, useAbs = False, activityClassification = True, rawOutput = False, rawOutputFile = None, fftOutput = False, startTime = None, endTime = None, verbose = False): """Process raw accelerometer file, writing summary epoch stats to file This is usually achieved by 1) identify 10sec stationary epochs 2) record calibrated axes scale/offset/temp vals + static point stats 3) use calibration coefficients and then write filtered avgVm epochs to <epochFile> from <rawFile> :param str rawFile: Input <cwa/bin/gt3x> raw accelerometer file :param str epochFile: Output csv.gz file of processed epoch data :param str stationaryFile: Output/temporary file for calibration :param dict summary: Output dictionary containing all summary metrics :param bool skipCalibration: Perform software calibration (process data twice) :param int stationaryStd: Gravity threshold (in mg units) for stationary vs not :param float xIntercept: Calbiration offset x :param float yIntercept: Calbiration offset y :param float zIntercept: Calbiration offset z :param float xSlope: Calbiration slope x :param float ySlope: Calbiration slope y :param float zSlope: Calbiration slope z :param float xTemp: Calbiration temperature coefficient x :param float yTemp: Calbiration temperature coefficient y :param float zTemp: Calbiration temperature coefficient z :param float meanTemp: Calibration mean temperature in file :param str rawDataParser: External helper process to read raw acc file. If a java class, it must omit .class ending. :param str javaHeapSpace: Amount of heap space allocated to java subprocesses. Useful for limiting RAM usage. :param bool skipFiltering: Skip filtering stage :param int sampleRate: Resample data to n Hz :param int epochPeriod: Size of epoch time window (in seconds) :param bool useAbs: Use abs(VM) instead of trunc(VM) :param bool activityClassification: Extract features for machine learning :param bool rawOutput: Output raw data to a .csv.gz file? requires ~70MB/day. :param str rawOutputFile: Output raw data ".csv.gz" filename :param bool fftOutput: Output FFT epochs to a .csv.gz file? requires ~0.1GB/day. :param datetime startTime: Remove data before this time in analysis :param datetime endTime: Remove data after this time in analysis :param bool verbose: Print verbose output :return: Raw processing summary values written to dict <summary> :rtype: void :Example: >>> import device >>> summary = {} >>> device.processRawFileToEpoch('rawFile.cwa', 'epochFile.csv.gz', 'stationary.csv.gz', summary) <epoch file written to "epochFile.csv.gz", and calibration points to 'stationary.csv.gz'> """ summary['file-size'] = os.path.getsize(rawFile) summary['file-deviceID'] = getDeviceId(rawFile) useJava = True javaClassPath = "java:java/JTransforms-3.1-with-dependencies.jar" staticStdG = stationaryStd / 1000.0 #java expects units of G (not mg) if 'omconvert' in rawDataParser: useJava = False if useJava: # calibrate axes scale/offset/temp values if not skipCalibration: # identify 10sec stationary epochs accUtils.toScreen('calibrating to file: ' + stationaryFile) commandArgs = ["java", "-classpath", javaClassPath, "-XX:ParallelGCThreads=1", rawDataParser, rawFile, "outputFile:" + stationaryFile, "verbose:" + str(verbose), "filter:true", "getStationaryBouts:true", "epochPeriod:10", "stationaryStd:" + str(staticStdG)] if javaHeapSpace: commandArgs.insert(1, javaHeapSpace) # call process to identify stationary epochs exitCode = call(commandArgs) if exitCode != 0: print("Error: java calibration failed, exit ", exitCode) sys.exit(-6) # record calibrated axes scale/offset/temp vals + static point stats getCalibrationCoefs(stationaryFile, summary) xIntercept = summary['calibration-xOffset(g)'] yIntercept = summary['calibration-yOffset(g)'] zIntercept = summary['calibration-zOffset(g)'] xSlope = summary['calibration-xSlope(g)'] ySlope = summary['calibration-ySlope(g)'] zSlope = summary['calibration-zSlope(g)'] xTemp = summary['calibration-xTemp(C)'] yTemp = summary['calibration-yTemp(C)'] zTemp = summary['calibration-zTemp(C)'] meanTemp = summary['calibration-meanDeviceTemp(C)'] else: storeCalibrationParams(summary, xIntercept, yIntercept, zIntercept, xSlope, ySlope, zSlope, xTemp, yTemp, zTemp, meanTemp) summary['quality-calibratedOnOwnData'] = 0 summary['quality-goodCalibration'] = 1 # calculate and write filtered avgVm epochs from raw file commandArgs = ["java", "-classpath", javaClassPath, "-XX:ParallelGCThreads=1", rawDataParser, rawFile, "outputFile:" + epochFile, "verbose:" + str(verbose), "filter:"+str(skipFiltering), "sampleRate:" + str(sampleRate), "xIntercept:" + str(xIntercept), "yIntercept:" + str(yIntercept), "zIntercept:" + str(zIntercept), "xSlope:" + str(xSlope), "ySlope:" + str(ySlope), "zSlope:" + str(zSlope), "xTemp:" + str(xTemp), "yTemp:" + str(yTemp), "zTemp:" + str(zTemp), "meanTemp:" + str(meanTemp), "epochPeriod:" + str(epochPeriod), "rawOutput:" + str(rawOutput), "rawOutputFile:" + str(rawOutputFile), "fftOutput:" + str(fftOutput), "getEpochCovariance:True", "getSanDiegoFeatures:" + str(activityClassification), "getMADFeatures:" + str(activityClassification), "getAxisMeans:" + str(activityClassification), "getUnileverFeatures:" + str(activityClassification), "getEachAxis:" + str(activityClassification), "get3DFourier:" + str(activityClassification), "useAbs:" + str(useAbs)] accUtils.toScreen('epoch generation') if javaHeapSpace: commandArgs.insert(1, javaHeapSpace) if startTime: commandArgs.append("startTime:" + startTime.strftime("%Y-%m-%dT%H:%M")) if endTime: commandArgs.append("endTime:" + endTime.strftime("%Y-%m-%dT%H:%M")) exitCode = call(commandArgs) if exitCode != 0: print("Error: java epoch generation failed, exit ", exitCode) sys.exit(-7) else: if not skipCalibration: commandArgs = [rawDataParser, rawFile, "-svm-file", epochFile, "-info", stationaryFile, "-svm-extended", "3", "-calibrate", "1", "-interpolate-mode", "2", "-svm-mode", "1", "-svm-epoch", str(epochPeriod), "-svm-filter", "2"] else: calArgs = str(xSlope) + ',' calArgs += str(ySlope) + ',' calArgs += str(zSlope) + ',' calArgs += str(xIntercept) + ',' calArgs += str(yIntercept) + ',' calArgs += str(zIntercept) + ',' calArgs += str(xTemp) + ',' calArgs += str(yTemp) + ',' calArgs += str(zTemp) + ',' calArgs += str(meanTemp) commandArgs = [rawDataParser, rawFile, "-svm-file", epochFile, "-info", stationaryFile, "-svm-extended", "3", "-calibrate", "0", "-calibration", calArgs, "-interpolate-mode", "2", "-svm-mode", "1", "-svm-epoch", str(epochPeriod), "-svm-filter", "2"] call(commandArgs) getOmconvertInfo(stationaryFile, summary)
def getActivitySummary(epochFile, nonWearFile, summary, activityClassification=True, timeZone='Europe/London', startTime=None, endTime=None, epochPeriod=30, stationaryStd=13, minNonWearDuration=60, mgCutPointMVPA=100, mgCutPointVPA=425, activityModel="activityModels/walmsley-nov20.tar", intensityDistribution=False, useRecommendedImputation=True, psd=False, fourierFrequency=False, fourierWithAcc=False, m10l5=False, verbose=False): """Calculate overall activity summary from <epochFile> data Get overall activity summary from input <epochFile>. This is achieved by 1) get interrupt and data error summary vals 2) check if data occurs at a daylight savings crossover 3) calculate wear-time statistics, and write nonWear episodes to file 4) predict activity from features, and add label column 5) calculate imputation values to replace nan PA metric values 6) calculate empirical cumulative distribution function of vector magnitudes 7) derive main movement summaries (overall, weekday/weekend, and hour) :param str epochFile: Input csv.gz file of processed epoch data :param str nonWearFile: Output filename for non wear .csv.gz episodes :param dict summary: Output dictionary containing all summary metrics :param bool activityClassification: Perform machine learning of activity states :param str timeZone: timezone in country/city format to be used for daylight savings crossover check :param datetime startTime: Remove data before this time in analysis :param datetime endTime: Remove data after this time in analysis :param int epochPeriod: Size of epoch time window (in seconds) :param int stationaryStd: Threshold (in mg units) for stationary vs not :param int minNonWearDuration: Minimum duration of nonwear events (minutes) :param int mgCutPointMVPA: Milli-gravity threshold for moderate intensity activity :param int mgCutPointVPA: Milli-gravity threshold for vigorous intensity activity :param str activityModel: Input tar model file which contains random forest pickle model, HMM priors/transitions/emissions npy files, and npy file of METS for each activity state :param bool intensityDistribution: Add intensity outputs to dict <summary> :param bool useRecommendedImputation: Highly recommended method to impute missing data using data from other days around the same time :param bool verbose: Print verbose output :return: Pandas dataframe of activity epoch data :rtype: pandas.DataFrame :return: Activity prediction labels (empty if <activityClassification>==False) :rtype: list(str) :return: Write .csv.gz non wear episodes file to <nonWearFile> :rtype: void :return: Movement summary values written to dict <summary> :rtype: void :Example: >>> import summariseEpoch >>> summary = {} >>> epochData, labels = summariseEpoch.getActivitySummary( "epoch.csv.gz", "nonWear.csv.gz", summary) <nonWear file written to "nonWear.csv.gz" and dict "summary" update with outcomes> """ accUtils.toScreen("=== Summarizing ===") if isinstance(epochFile, pd.DataFrame): e = epochFile else: # Use python PANDAS framework to read in and store epochs e = pd.read_csv( epochFile, index_col=['time'], parse_dates=['time'], date_parser=accUtils.date_parser, ) # Remove data before/after user specified start/end times rows = e.shape[0] tz = pytz.timezone(timeZone) if startTime: localStartTime = tz.localize(startTime) e = e[e.index >= localStartTime] if endTime: localEndTime = tz.localize(endTime) e = e[e.index <= localEndTime] # Quit if no data left if e.shape[0] == 0: print("No rows remaining after start/end time removal") print("Previously there were %d rows, now shape: %s" % (rows, str(e.shape))) sys.exit(-9) # Get start & end times startTime = e.index[0] endTime = e.index[-1] summary['file-startTime'] = accUtils.date_strftime(startTime) summary['file-endTime'] = accUtils.date_strftime(endTime) summary['file-firstDay(0=mon,6=sun)'] = startTime.weekday() # Get interrupt and data error summary vals e = get_interrupts(e, epochPeriod, summary) # Check daylight savings time crossover check_daylight_savings_crossovers(e, summary) # Calculate wear-time statistics, and write nonWear episodes to file get_wear_time_stats(e, epochPeriod, stationaryStd, minNonWearDuration, nonWearFile, summary) # Calculate and include data quality statistics get_total_reads(e, epochPeriod, summary) get_clips(e, epochPeriod, summary) # Predict activity from features, and add label column if activityClassification: e, labels = accClassification.activityClassification(e, activityModel) else: labels = [] # enmo : Euclidean Norm Minus One # Trunc : negative values truncated to zero (i.e never negative) # emmo = 1 - sqrt(x, y, z) # enmoTrunc = max(enmo, 0) e['acc'] = e['enmoTrunc'] * 1000 # convert enmoTrunc to milli-G units # Calculate imputation values to replace nan PA metric values e = perform_wearTime_imputation(e, verbose) e['CutPointMVPA'] = e['accImputed'] >= mgCutPointMVPA e['CutPointVPA'] = e['accImputed'] >= mgCutPointVPA # Calculate empirical cumulative distribution function of vector magnitudes if intensityDistribution: calculateECDF(e, 'acc', summary, useRecommendedImputation) # Calculate circadian metrics if psd: circadianRhythms.calculatePSD(e, epochPeriod, fourierWithAcc, labels, summary) if fourierFrequency: circadianRhythms.calculateFourierFreq(e, epochPeriod, fourierWithAcc, labels, summary) if m10l5: circadianRhythms.calculateM10L5(e, epochPeriod, summary) # Main movement summaries writeMovementSummaries(e, labels, summary, useRecommendedImputation) # Return physical activity summary return e, labels
def main(): # noqa: C901 """ Application entry point responsible for parsing command line requests """ parser = argparse.ArgumentParser( description="A script to plot acc time series data.", add_help=True) # required parser.add_argument('timeSeriesFile', metavar='input file', type=str, help="input .csv.gz time series file to plot") parser.add_argument('--plotFile', metavar='output file', type=str, help="output .png file to plot to") parser.add_argument('--activityModel', type=str, default="walmsley", help="""trained activity model .tar file""") parser.add_argument('--useRecommendedImputation', metavar='True/False', default=True, type=str2bool, help="""Highly recommended method to show imputed missing data (default : %(default)s)""") parser.add_argument('--imputedLabels', metavar='True/False', default=False, type=str2bool, help="""If activity classification during imputed period will be displayed (default : %(default)s)""" ) parser.add_argument('--imputedLabelsHeight', metavar='Proportion i.e. 0-1.0', default=0.9, type=float, help="""Proportion of plot labels take if activity classification during imputed period will be displayed (default : %(default)s)""" ) parser.add_argument('--showFileName', metavar='True/False', default=False, type=str2bool, help="""Toggle showing filename as title in output image (default : %(default)s)""") parser.add_argument('--showFirstNDays', metavar='days', default=None, type=int, help="Show just first n days") # check input is ok if len(sys.argv) < 2: msg = "\nInvalid input, please enter at least 1 parameter, e.g." msg += "\npython accPlot.py timeSeries.csv.gz \n" accUtils.toScreen(msg) parser.print_help() sys.exit(-1) args = parser.parse_args() # determine output file name if args.plotFile is None: inputFileFolder, inputFileName = os.path.split(args.timeSeriesFile) inputFileName = inputFileName.split('.')[0] # remove any extension args.plotFile = os.path.join(inputFileFolder, inputFileName + "-plot.png") # and then call plot function plotTimeSeries(args.timeSeriesFile, args.plotFile, showFirstNDays=args.showFirstNDays, activityModel=args.activityModel, useRecommendedImputation=args.useRecommendedImputation, imputedLabels=args.imputedLabels, imputedLabelsHeight=args.imputedLabelsHeight, showFileName=args.showFileName)
def main(): """ Application entry point responsible for parsing command line requests """ parser = argparse.ArgumentParser( description="""A tool to extract physical activity information from raw accelerometer files.""", add_help=True ) # required parser.add_argument('rawFile', metavar='input file', type=str, help="""the .cwa file to process (e.g. sample.cwa). If the file path contains spaces,it must be enclosed in quote marks (e.g. \"../My Documents/sample.cwa\") """) #optional inputs parser.add_argument('--startTime', metavar='e.g. 1991-01-01T23:59', default=None, type=str2date, help="""removes data before this time in the final analysis (default : %(default)s)""") parser.add_argument('--endTime', metavar='e.g 1991-01-01T23:59', default=None, type=str2date, help="""removes data after this time in the final analysis (default : %(default)s)""") parser.add_argument('--timeSeriesDateColumn', metavar='True/False', default=False, type=str2bool, help="""adds a date/time column to the timeSeries file, so acceleration and imputation values can be compared easily. This increases output filesize (default : %(default)s)""") parser.add_argument('--processRawFile', metavar='True/False', default=True, type=str2bool, help="""False will skip processing of the .cwa file (the epoch.csv file must already exist for this to work) (default : %(default)s)""") parser.add_argument('--epochPeriod', metavar='length', default=30, type=int, help="""length in seconds of a single epoch (default : %(default)ss, must be an integer)""") parser.add_argument('--sampleRate', metavar='Hz, or samples/second', default=100, type=int, help="""resample data to n Hz (default : %(default)ss, must be an integer)""") parser.add_argument('--useAbs', metavar='useAbs', default=False, type=str2bool, help="""use abs(VM) instead of trunc(VM) (default : %(default)s)""") parser.add_argument('--skipFiltering', metavar='True/False', default=False, type=str2bool, help="""Skip filtering stage (default : %(default)s)""") # calibration parameters parser.add_argument('--skipCalibration', metavar='True/False', default=False, type=str2bool, help="""skip calibration? (default : %(default)s)""") parser.add_argument('--calOffset', metavar=('x', 'y', 'z'),default=[0.0, 0.0, 0.0], type=float, nargs=3, help="""accelerometer calibration offset (default : %(default)s)""") parser.add_argument('--calSlope', metavar=('x', 'y', 'z'), default=[1.0, 1.0, 1.0], type=float, nargs=3, help="""accelerometer calibration slope linking offset to temperature (default : %(default)s)""") parser.add_argument('--calTemp', metavar=('x', 'y', 'z'), default=[0.0, 0.0, 0.0], type=float, nargs=3, help="""mean temperature in degrees Celsius of stationary data for calibration (default : %(default)s)""") parser.add_argument('--meanTemp', metavar="temp", default=20.0, type=float, help="""mean calibration temperature in degrees Celsius (default : %(default)s)""") parser.add_argument('--stationaryStd', metavar='mg', default=13, type=int, help="""stationary mg threshold (default : %(default)s mg))""") parser.add_argument('--calibrationSphereCriteria', metavar='mg', default=0.3, type=float, help="""calibration sphere threshold (default : %(default)s mg))""") # activity parameters parser.add_argument('--mgMVPA', metavar="mg", default=100, type=int, help="""MVPA threshold (default : %(default)s)""") parser.add_argument('--mgVPA', metavar="mg", default=425, type=int, help="""VPA threshold (default : %(default)s)""") # calling helper processess and conducting multi-threadings parser.add_argument('--rawDataParser', metavar="rawDataParser", default="AxivityAx3Epochs", type=str, help="""file containing a java program to process raw .cwa binary file, must end with .class (omitted) (default : %(default)s)""") parser.add_argument('--javaHeapSpace', metavar="amount in MB", default="", type=str, help="""amount of heap space allocated to the java subprocesses,useful for limiting RAM usage (default : unlimited)""") # activity classification arguments parser.add_argument('--activityClassification', metavar='True/False', default=True, type=str2bool, help="""Use pre-trained random forest to predict activity type (default : %(default)s)""") parser.add_argument('--activityModel', type=str, default="activityModels/doherty2018.tar", help="""trained activity model .tar file""") parser.add_argument('--rawOutput', metavar='True/False', default=False, type=str2bool, help="""output raw data to a .csv.gz file? NOTE: requires ~70MB per day. (default : %(default)s)""") parser.add_argument('--fftOutput', metavar='True/False', default=False, type=str2bool, help="""output FFT epochs to a .csv file? NOTE: requires ~0.1GB per day. (default : %(default)s)""") # optional outputs parser.add_argument('--outputFolder', metavar='filename',default="", help="""folder for all of the output files, \ unless specified using other options""") parser.add_argument('--summaryFolder', metavar='filename',default="", help="folder for -summary.json summary stats") parser.add_argument('--epochFolder', metavar='filename', default="", help="""folder -epoch.csv.gz - must be an existing file if "-processRawFile" is set to False""") parser.add_argument('--timeSeriesFolder', metavar='filename', default="", help="folder for -timeSeries.csv.gz file") parser.add_argument('--nonWearFolder', metavar='filename',default="", help="folder for -nonWearBouts.csv.gz file") parser.add_argument('--stationaryFolder', metavar='filename', default="", help="folder -stationaryPoints.csv.gz file") parser.add_argument('--rawFolder', metavar='filename', default="", help="folder for raw .csv.gz file") parser.add_argument('--verbose', metavar='True/False', default=False, type=str2bool, help="""enable verbose logging? (default : %(default)s)""") parser.add_argument('--deleteIntermediateFiles', metavar='True/False', default=True, type=str2bool, help="""True will remove extra "helper" files created by the program (default : %(default)s)""") parser.add_argument('--intensityDistribution', metavar='True/False', default=False, type=str2bool, help="""Save intensity distribution (default : %(default)s)""") # # check that enough command line arguments are entered # if len(sys.argv) < 2: msg = "\nInvalid input, please enter at least 1 parameter, e.g." msg += "\npython ActivitySummary.py inputFile.CWA \n" accUtils.toScreen(msg) parser.print_help() sys.exit(-1) processingStartTime = datetime.datetime.now() args = parser.parse_args() ########################## # check input/output files/dirs exist and validate input args ########################## if args.processRawFile is False: if len(args.rawFile.split('.')) < 2: args.rawFile += ".cwa" # TODO edge case since we still need a name? elif not os.path.isfile(args.rawFile): if args.rawFile: print("error: specified file " + args.rawFile + " does not exist. Exiting..") else: print("error: no file specified. Exiting..") sys.exit(-2) # get file extension rawFileNoExt = ('.').join(args.rawFile.split('.')[:-1]) (rawFilePath, rawFileName) = os.path.split(rawFileNoExt) # check target output folders exist for path in [args.summaryFolder, args.nonWearFolder, args.epochFolder, args.stationaryFolder, args.timeSeriesFolder, args.outputFolder]: if len(path) > 0 and not os.access(path, os.F_OK): print("error: " + path + " is not a valid directory") sys.exit(-3) # assign output file names if args.outputFolder == "" and rawFilePath != "": args.outputFolder = rawFilePath + '/' if args.summaryFolder == "": args.summaryFolder = args.outputFolder if args.nonWearFolder == "": args.nonWearFolder = args.outputFolder if args.epochFolder == "": args.epochFolder = args.outputFolder if args.stationaryFolder == "": args.stationaryFolder = args.outputFolder if args.timeSeriesFolder == "": args.timeSeriesFolder = args.outputFolder if args.rawFolder == "": args.rawFolder = args.outputFolder args.summaryFile = args.summaryFolder + rawFileName + "-summary.json" args.nonWearFile = args.nonWearFolder + rawFileName + "-nonWearBouts.csv.gz" args.epochFile = args.epochFolder + rawFileName + "-epoch.csv.gz" args.stationaryFile = args.stationaryFolder + rawFileName + "-stationaryPoints.csv" args.tsFile = args.timeSeriesFolder + rawFileName + "-timeSeries.csv.gz" args.rawOutputFile = args.rawFolder + rawFileName + ".csv.gz" # check user specified end time is not before start time if args.startTime and args.endTime: if args.startTime >= args.endTime: print("start and end time arguments are invalid!") print("startTime:", args.startTime.strftime("%Y-%m-%dT%H:%M")) print("endTime:", args.endTime.strftime("%Y-%m-%dT%H:%M")) sys.exit(-4) # print processing options to screen print("processing file " + args.rawFile + "' with these arguments:\n") for key, value in sorted(vars(args).items()): if not (isinstance(value, str) and len(value)==0): print(key.ljust(15), ':', value) print("\n") ########################## # start processing file ########################## summary = {} # now process the .CWA file if args.processRawFile: summary['file-name'] = args.rawFile accelerometer.device.processRawFileToEpoch(args.rawFile, args.epochFile, args.stationaryFile, summary, skipCalibration = args.skipCalibration, stationaryStd = args.stationaryStd, xIntercept = args.calOffset[0], yIntercept = args.calOffset[1], zIntercept = args.calOffset[2], xSlope = args.calSlope[0], ySlope = args.calSlope[1], zSlope = args.calSlope[2], xTemp = args.calTemp[0], yTemp = args.calTemp[1], zTemp = args.calTemp[2], meanTemp = args.meanTemp, rawDataParser = args.rawDataParser, javaHeapSpace = args.javaHeapSpace, skipFiltering = args.skipFiltering, sampleRate= args.sampleRate, epochPeriod = args.epochPeriod, useAbs = args.useAbs, activityClassification = args.activityClassification, rawOutput = args.rawOutput, rawOutputFile = args.rawOutputFile, fftOutput = args.fftOutput, startTime = args.startTime, endTime = args.endTime, verbose = args.verbose) else: summary['file-name'] = args.epochFile # summarise epoch epochData, labels = accelerometer.summariseEpoch.getActivitySummary( args.epochFile, args.nonWearFile, summary, activityClassification = args.activityClassification, startTime = args.startTime, endTime = args.endTime, epochPeriod = args.epochPeriod, stationaryStd = args.stationaryStd, mgMVPA = args.mgMVPA, mgVPA = args.mgVPA, activityModel = args.activityModel, intensityDistribution = args.intensityDistribution, verbose = args.verbose) # generate time series file (note: this will also resample to epochData so do this last) accelerometer.accUtils.generateTimeSeries(epochData, args.tsFile, epochPeriod = args.epochPeriod, timeSeriesDateColumn = args.timeSeriesDateColumn, activityClassification = args.activityClassification, labels = labels) # print basic output summaryVals = ['file-name', 'file-startTime', 'file-endTime', \ 'acc-overall-avg','wearTime-overall(days)', \ 'nonWearTime-overall(days)', 'quality-goodWearTime'] summaryDict = collections.OrderedDict([(i, summary[i]) for i in summaryVals]) accelerometer.accUtils.toScreen(json.dumps(summaryDict, indent=4)) # write detailed output to file f = open(args.summaryFile,'w') json.dump(summary, f, indent=4) f.close() accelerometer.accUtils.toScreen('Summary file written to: ' + args.summaryFile) ########################## # remove helper files and close program ########################## if args.deleteIntermediateFiles: try: os.remove(args.stationaryFile) os.remove(args.epochFile) except: accelerometer.accUtils.toScreen('could not delete helper file') # finally, print out processing summary message processingEndTime = datetime.datetime.now() processingTime = (processingEndTime - processingStartTime).total_seconds() accelerometer.accUtils.toScreen("in total, processing took " + \ str(processingTime) + " seconds")