def process(self): for fn in self.dataFiles: EDVerbose.screen("Processing file %s" % fn) edj = EDJob(self.EDNAPluginName) edj.dataInput = self.fileName2xml(fn) edj.connectSUCCESS(self.XMLsuccess) edj.connectFAILURE(self.XMLerr) self.queue.put(edj) if self.process_sem._Semaphore__value > 0 : t = threading.Thread(target=self.startProcessing) t.start() EDVerbose.screen("Back in main") while self.cpu_sem._Semaphore__value < self.nbcpu: time.sleep(0.1) EDJob.synchronizeAll() EDJob.stats()
else: ffx = FullFieldXas() if os.path.isfile(".XSDataInputFullFieldXAS.xml"): ffx.load(".XSDataInputFullFieldXAS.xml") ffx.dontAlign = dontAlign if dontAlign: print ("*" * 80) print ("*" + "Skipping image alignement part".center(78) + "*") print ("*" * 80) ffx.setup(_listInput=paths, _mode=mode) if not isinstance(ffx.reference, int): ffx.uploadReferenceFrame(ffx.reference) ffx.save(".XSDataInputFullFieldXAS.xml") ffx.dump("analysis-%s.json" % time.strftime("%Y%m%d-%Hh%Mm%Ss")) edna = EDParallelExecute( ffx.pluginName, ffx.makeXML, _functXMLerr=ffx.error, _bVerbose=True, _bDebug=debug, _iNbThreads=iNbCPU ) edna.runEDNA(ffx.listInput, ffx.strMode, ffx.bNewerOnly) EDVerbose.WARNING("Back to main !") EDJob.synchronizeAll() ffx.updateAttrRefImage() EDPluginControlAlignStackv1_0.showData() if keepShare: EDShare.close() elif (ffx.getNrErrors() == 0) and (not EDVerbose.isVerboseDebug()): EDVerbose.WARNING("All processing finished successfully: Remove EDShare's big HDF5 file") EDShare.close(remove=True) else: EDShare.close()
if replay: ffx = FullFieldXas.replay_from(replay) else: ffx = FullFieldXas() if os.path.isfile(".XSDataInputFullFieldXAS.xml"): ffx.load(".XSDataInputFullFieldXAS.xml") ffx.dontAlign = dontAlign if dontAlign: print("*"*80) print("*" + "Skipping image alignement part".center(78) + "*") print("*"*80) ffx.setup(_listInput=paths, _mode=mode) if not isinstance(ffx.reference, int): ffx.uploadReferenceFrame(ffx.reference) ffx.save(".XSDataInputFullFieldXAS.xml") ffx.dump("analysis-%s.json" % time.strftime("%Y%m%d-%Hh%Mm%Ss")) edna = EDParallelExecute(ffx.pluginName, ffx.makeXML, _functXMLerr=ffx.error, _bVerbose=True, _bDebug=debug, _iNbThreads=iNbCPU) edna.runEDNA(ffx.listInput, ffx.strMode , ffx.bNewerOnly) EDVerbose.WARNING("Back to main !") EDJob.synchronizeAll() ffx.updateAttrRefImage() EDPluginControlAlignStackv1_0.showData() if keepShare: EDShare.close() elif (ffx.getNrErrors() == 0) and (not EDVerbose.isVerboseDebug()): EDVerbose.WARNING("All processing finished successfully: Remove EDShare's big HDF5 file") EDShare.close(remove=True) else: EDShare.close()
print("All %i jobs queued after %.3fs" % (len(args), time.time() - reprocess.startTime)) reprocess.join() if yappi: yappi.stop() print("All %i jobs processed after %.3fs" % (len(args), time.time() - reprocess.startTime)) print reprocess.statistics() if yappi: stat = yappi.get_stats(sort_type=yappi.SORTTYPE_TTOT) res = {} for i in stat.func_stats: if i[0] in res: res[i[0]][0] += i[1] res[i[0]][1] += i[2] else: res[i[0]] = [i[1], i[2]] keys = res.keys() keys.sort(sortn) with open("yappi.out", "w") as f: f.write("ncall\t\ttotal\t\tpercall\t\tfunction%s" % (os.linesep)) for i in keys: f.write("%8s\t%16s\t%16s\t%s%s" % (res[i][0], res[i][1], res[i][1] / res[i][0], i, os.linesep)) print("Profiling information written in yappi.out") edJob = EDJob(options.plugin.replace("EDPluginBioSaxsHPLC", "EDPluginBioSaxsFlushHPLC")) edJob.setDataInput(open(fullargs[-1], "r").read()) edJob.execute() edJob.synchronizeAll() if options.profile: for i in EDObject.analyze_profiling(): print(i)
print("All %i jobs processed after %.3fs" % (len(args), time.time() - reprocess.startTime)) print reprocess.statistics() if yappi: stat = yappi.get_stats(sort_type=yappi.SORTTYPE_TTOT) res = {} for i in stat.func_stats: if i[0] in res: res[i[0]][0] += i[1] res[i[0]][1] += i[2] else: res[i[0]] = [i[1], i[2]] keys = res.keys() keys.sort(sortn) with open("yappi.out", "w") as f: f.write("ncall\t\ttotal\t\tpercall\t\tfunction%s" % (os.linesep)) for i in keys: f.write("%8s\t%16s\t%16s\t%s%s" % (res[i][0], res[i][1], res[i][1] / res[i][0], i, os.linesep)) print("Profiling information written in yappi.out") edJob = EDJob( options.plugin.replace("EDPluginBioSaxsHPLC", "EDPluginBioSaxsFlushHPLC")) edJob.setDataInput(open(fullargs[-1], "r").read()) edJob.execute() edJob.synchronizeAll() if options.profile: for i in EDObject.analyze_profiling(): print(i)