def main(argv): # parse input arguments parser = argparse.ArgumentParser() parser.add_argument('-i', '--input', help="the input YAML configuration file") parser.add_argument('-s', '--serial', help="run in serial mode", action="store_true") args = parser.parse_args() batchStart = time.time() Logger.logPrint("Starting batchprocess: "+args.input) Logger.logPrint("Start time:"+timeStr(batchStart)) # load WormVideo to YAML configuration file with open(args.input, 'r') as f: wvs = wtc.loadWormVideos(f) # run analysis on regions if args.serial: for wv in wvs: wv.processRegions() else: wtp.batchProcessVideos(wvs) batchStop = time.time() Logger.logPrint("Start time:"+timeStr(batchStart)) Logger.logPrint("End time :"+timeStr(batchStop)) Logger.logPrint("Total time:"+timeStr(batchStop-batchStart)) return 'Success'
def parallelProcessRegions(wormVideo): wormVideo.saveConfiguration() pool = multiprocessing.Pool() result = pool.map_async(processRegion, wormVideo.regions) Logger.logPrint(','.join([str(r) for r in result.get()])) pool.close() pool.join() Logger.logPrint('Finished analyzing all regions') cleanUpProcess(wormVideo)
def batchProcessVideos(wormVideos): pool = multiprocessing.Pool() results = [] for video in wormVideos: video.saveConfiguration() results.append(pool.map_async(processRegion, video.regions)) for result in results: for regionResult in result.get(): Logger.logPrint(regionResult) pool.close() pool.join() Logger.logPrint('Finished analyzing all regions') for video in wormVideos: cleanUpProcess(video)
def processRegion(region): Logger.logPrint('Starting analysis of {0} {1}'.format( region.strainName, region.wormName)) try: # split output to a different file path, name = os.path.split(region.resultsStoreFile) newName = (region.strainName + '_' + region.wormName + '_' + name) region.resultsStoreFile = os.path.join(path, newName) tStart = time.clock() region.saveConfiguration() region.process() tFinish = time.clock() tDuration = (tFinish - tStart) / 60 Logger.logPrint('Analysis of {0} {1} took {2} min.'.format( region.strainName, region.wormName, str(tDuration))) return 'Success' except (Exception) as e: Logger.logPrint('Error during analysis of {0}{1}: {2}'.format( region.strainName, region.wormName, str(e))) return 'Failed'
def processRegion(region): Logger.logPrint('Starting analysis of {0} {1}'.format(region.strainName, region.wormName)) try: # split output to a different file path, name = os.path.split(region.resultsStoreFile) newName = (region.strainName+'_'+region.wormName+'_'+name) region.resultsStoreFile = os.path.join(path, newName) tStart = time.clock() region.saveConfiguration() region.process() tFinish = time.clock() tDuration = (tFinish - tStart) / 60 Logger.logPrint('Analysis of {0} {1} took {2} min.'.format(region.strainName, region.wormName, str(tDuration))) return 'Success' except(Exception) as e: Logger.logPrint('Error during analysis of {0}{1}: {2}'.format(region.strainName, region.wormName, str(e))) return 'Failed'
def cleanUpProcess(wormVideo): # merge results into original output file # should work path, name = os.path.split(wormVideo.storeFile) outputFile = os.path.join(path, 'merge_' + name) try: # copy video info Logger.logPrint('Merging HDF5 output files...') obj = '/video' # Linux: create command, do no escape argument values and keep arguments as separated argument list: # have to copy because the merge will fail if there are any duplicates cmd = [ hdf5path + 'h5copy', '-i', os.path.join(path, name), '-o', outputFile, '-s', obj, '-d', obj, '-p' ] Logger.logPrint('Executing:' + ' '.join(cmd)) Logger.logPrint(check_output(cmd)) # remove premerge file os.remove(wormVideo.storeFile) for region in wormVideo.regions: try: args = [ hdf5path + 'h5copy', '-i', os.path.join(path, '{1}_{2}_{0}'), '-o', os.path.join(path, 'merge_{0}'), '-s', '/worms/{1}/{2}', '-d', '/worms/{1}/{2}', '-p' ] #update cmd = [ arg.format(name, region.strainName, region.wormName) for arg in args ] Logger.logPrint('Executing:' + ' '.join(cmd)) Logger.logPrint('Output:' + check_output(cmd, stderr=STDOUT)) # remove premerge file os.remove( os.path.join(path, '{1}_{2}_{0}').format(name, region.strainName, region.wormName)) except (Exception) as e: Logger.logPrint('Error cleaning up:') Logger.logPrint('Exception:' + str(e)) # rename merge file os.rename(outputFile, wormVideo.storeFile) except (Exception) as e: Logger.logPrint('Error cleaning up:') Logger.logPrint('Exception:' + str(e))
def parallelPostProcessRegions(storeFile): path, name = os.path.split(storeFile) outputFile = os.path.join(path, 'merge_' + name) try: # copy video info Logger.logPrint('Splitting HDF5 output files...') obj = '/video' # Linux: create command, do no escape argument values and keep arguments as separated argument list: # have to copy because the merge will fail if there are any duplicates cmd = [ hdf5path + 'h5copy', '-i', os.path.join(path, name), '-o', outputFile, '-s', obj, '-d', obj, '-p' ] Logger.logPrint('Executing:' + ' '.join(cmd)) Logger.logPrint(check_output(cmd)) postProcessList = [] # split worms into separate files to avoid IO issues with h5py.File(storeFile, 'r') as f: strains = f['worms'].keys() for strain in strains: worms = f['worms'][strain].keys() for worm in worms: # split each worm into a separate hdf5 store wormFileName = '{1}_{2}_{0}'.format(name, strain, worm) wormFile = os.path.join(path, wormFileName) # first copy video info over cmd = [ hdf5path + 'h5copy', '-i', storeFile, '-o', wormFile, '-s', obj, '-d', obj, '-p' ] Logger.logPrint('Executing:' + ' '.join(cmd)) Logger.logPrint(check_output(cmd)) # then copy worm data over wormObj = '/worms/{0}/{1}'.format(strain, worm) cmd = [ hdf5path + 'h5copy', '-i', storeFile, '-o', wormFile, '-s', wormObj, '-d', wormObj, '-p' ] Logger.logPrint('Executing:' + ' '.join(cmd)) Logger.logPrint(check_output(cmd)) # add worm to list to postprocess postProcessList.append((wormFile, strain, worm)) # parallel postprocess each region pool = multiprocessing.Pool() result = pool.map_async(postProcessRegion, postProcessList) Logger.logPrint(','.join([str(r) for r in result.get()])) pool.close() pool.join() Logger.logPrint('Finished analyzing all regions') # merge files back together for (wormFile, strain, worm) in postProcessList: try: args = [ hdf5path + 'h5copy', '-i', wormFile, '-o', outputFile, '-s', '/worms/{0}/{1}', '-d', '/worms/{0}/{1}', '-p' ] #update cmd = [arg.format(strain, worm) for arg in args] Logger.logPrint('Executing:' + ' '.join(cmd)) Logger.logPrint('Output:' + check_output(cmd, stderr=STDOUT)) # remove premerge file os.remove(wormFile) except (Exception) as e: Logger.logPrint('Error cleaning up:') Logger.logPrint('Exception:' + str(e)) # remove original file os.remove(storeFile) # rename merge file os.rename(outputFile, storeFile) except (Exception) as e: Logger.logPrint('Error cleaning up:') Logger.logPrint('Exception:' + str(e))
def cleanUpProcess(wormVideo): # merge results into original output file # should work path, name = os.path.split(wormVideo.storeFile) outputFile = os.path.join(path, 'merge_' + name) try: # copy video info Logger.logPrint('Merging HDF5 output files...') obj = '/video' # Linux: create command, do no escape argument values and keep arguments as separated argument list: # have to copy because the merge will fail if there are any duplicates cmd = [hdf5path + 'h5copy', '-i', os.path.join(path, name), '-o', outputFile, '-s', obj, '-d', obj, '-p'] Logger.logPrint('Executing:'+' '.join(cmd)) Logger.logPrint(check_output(cmd)) # remove premerge file os.remove(wormVideo.storeFile) for region in wormVideo.regions: try: args = [hdf5path + 'h5copy', '-i', os.path.join(path,'{1}_{2}_{0}'), '-o', os.path.join(path, 'merge_{0}'), '-s', '/worms/{1}/{2}','-d', '/worms/{1}/{2}', '-p'] #update cmd=[arg.format(name,region.strainName,region.wormName) for arg in args] Logger.logPrint('Executing:'+' '.join(cmd)) Logger.logPrint('Output:'+check_output(cmd, stderr=STDOUT)) # remove premerge file os.remove(os.path.join(path, '{1}_{2}_{0}').format(name, region.strainName, region.wormName)) except(Exception) as e: Logger.logPrint('Error cleaning up:') Logger.logPrint('Exception:'+str(e)) # rename merge file os.rename(outputFile, wormVideo.storeFile) except(Exception) as e: Logger.logPrint('Error cleaning up:') Logger.logPrint('Exception:'+str(e))
def parallelPostProcessRegions(storeFile): path, name = os.path.split(storeFile) outputFile = os.path.join(path, 'merge_' + name) try: # copy video info Logger.logPrint('Splitting HDF5 output files...') obj = '/video' # Linux: create command, do no escape argument values and keep arguments as separated argument list: # have to copy because the merge will fail if there are any duplicates cmd = [hdf5path + 'h5copy', '-i', os.path.join(path, name), '-o', outputFile, '-s', obj, '-d', obj, '-p'] Logger.logPrint('Executing:'+' '.join(cmd)) Logger.logPrint(check_output(cmd)) postProcessList = [] # split worms into separate files to avoid IO issues with h5py.File(storeFile, 'r') as f: strains = f['worms'].keys() for strain in strains: worms = f['worms'][strain].keys() for worm in worms: # split each worm into a separate hdf5 store wormFileName = '{1}_{2}_{0}'.format(name, strain, worm) wormFile = os.path.join(path, wormFileName) # first copy video info over cmd = [hdf5path + 'h5copy', '-i', storeFile, '-o', wormFile, '-s', obj, '-d', obj, '-p'] Logger.logPrint('Executing:'+' '.join(cmd)) Logger.logPrint(check_output(cmd)) # then copy worm data over wormObj = '/worms/{0}/{1}'.format(strain, worm) cmd = [hdf5path + 'h5copy', '-i', storeFile, '-o', wormFile, '-s', wormObj, '-d', wormObj, '-p'] Logger.logPrint('Executing:'+' '.join(cmd)) Logger.logPrint(check_output(cmd)) # add worm to list to postprocess postProcessList.append((wormFile, strain, worm)) # parallel postprocess each region pool = multiprocessing.Pool() result = pool.map_async(postProcessRegion, postProcessList) Logger.logPrint(','.join([str(r) for r in result.get()])) pool.close() pool.join() Logger.logPrint('Finished analyzing all regions') # merge files back together for (wormFile, strain, worm) in postProcessList: try: args = [hdf5path + 'h5copy', '-i', wormFile, '-o', outputFile, '-s', '/worms/{0}/{1}','-d', '/worms/{0}/{1}', '-p'] #update cmd=[arg.format(strain,worm) for arg in args] Logger.logPrint('Executing:'+' '.join(cmd)) Logger.logPrint('Output:'+check_output(cmd, stderr=STDOUT)) # remove premerge file os.remove(wormFile) except(Exception) as e: Logger.logPrint('Error cleaning up:') Logger.logPrint('Exception:'+str(e)) # remove original file os.remove(storeFile) # rename merge file os.rename(outputFile, storeFile) except(Exception) as e: Logger.logPrint('Error cleaning up:') Logger.logPrint('Exception:'+str(e))
def postProcess(self): Logger.logPrint('Identifying bad frames...') self.identifyBadFrames() Logger.logPrint('Extracting postural data...') self.extractPosturalData() Logger.logPrint('Fixing order of postural data...') self.fixPosturalOrdering() Logger.logPrint('Segmenting trajectory...') self.segment() Logger.logPrint('Assigning head...') if self.usePosturalHeadAssignment: self.assignHeadTail() else: self.assignHeadTailCentroidOnly() Logger.logPrint('Ordering postural data head to tail...') self.orderHeadTail() Logger.logPrint('Calculating centroid motion variables...') self.calculateCentroidMeasurements() Logger.logPrint('Calculating postural measurements...') self.calculatePosturalMeasurements()
def main(argv): # parse input arguments parser = argparse.ArgumentParser() parser.add_argument('-i', '--input', help="the input YAML configuration file or h5 store file") parser.add_argument('-s', '--serial', help="run in serial mode", action="store_true") args = parser.parse_args() batchStart = time.time() Logger.logPrint("Starting batchprocess: "+args.input) Logger.logPrint("Start time:"+timeStr(batchStart)) fileName, fileExtension = os.path.splitext(args.input) # If given a store file (.h5), just use that if fileExtension == '.h5': storeFiles = [args.input] else: with open(args.input, 'r') as f: storeFiles = wtc.extractStoreFileList(f) # postprocessing if args.serial: for storeFile in storeFiles: with h5py.File(storeFile, 'r+') as f: strains = f['worms'].keys() for strain in strains: worms = f['worms'][strain].keys() for worm in worms: pp = wtpp.WormTrajectoryPostProcessor(f, strain, worm) pp.postProcess() pp.store() else: for storeFile in storeFiles: Logger.logPrint('Post-processing: ' + storeFile) wtp.parallelPostProcessRegions(storeFile) batchStop = time.time() Logger.logPrint("Start time:"+timeStr(batchStart)) Logger.logPrint("End time :"+timeStr(batchStop)) Logger.logPrint("Total time:"+timeStr(batchStop-batchStart)) return 'Success'