def process(self):
for fn in self.dataFiles:
EDVerbose.screen("Processing file %s" % fn)
edj = EDJob(self.EDNAPluginName)
edj.dataInput = self.fileName2xml(fn)
edj.connectSUCCESS(self.XMLsuccess)
edj.connectFAILURE(self.XMLerr)
self.queue.put(edj)
if self.process_sem._Semaphore__value > 0 :
t = threading.Thread(target=self.startProcessing)
t.start()
EDVerbose.screen("Back in main")
while self.cpu_sem._Semaphore__value < self.nbcpu:
time.sleep(0.1)
EDJob.synchronizeAll()
EDJob.stats()
else:
ffx = FullFieldXas()
if os.path.isfile(".XSDataInputFullFieldXAS.xml"):
ffx.load(".XSDataInputFullFieldXAS.xml")
ffx.dontAlign = dontAlign
if dontAlign:
print ("*" * 80)
print ("*" + "Skipping image alignement part".center(78) + "*")
print ("*" * 80)
ffx.setup(_listInput=paths, _mode=mode)
if not isinstance(ffx.reference, int):
ffx.uploadReferenceFrame(ffx.reference)
ffx.save(".XSDataInputFullFieldXAS.xml")
ffx.dump("analysis-%s.json" % time.strftime("%Y%m%d-%Hh%Mm%Ss"))
edna = EDParallelExecute(
ffx.pluginName, ffx.makeXML, _functXMLerr=ffx.error, _bVerbose=True, _bDebug=debug, _iNbThreads=iNbCPU
)
edna.runEDNA(ffx.listInput, ffx.strMode, ffx.bNewerOnly)
EDVerbose.WARNING("Back to main !")
EDJob.synchronizeAll()
ffx.updateAttrRefImage()
EDPluginControlAlignStackv1_0.showData()
if keepShare:
EDShare.close()
elif (ffx.getNrErrors() == 0) and (not EDVerbose.isVerboseDebug()):
EDVerbose.WARNING("All processing finished successfully: Remove EDShare's big HDF5 file")
EDShare.close(remove=True)
else:
EDShare.close()
if replay:
ffx = FullFieldXas.replay_from(replay)
else:
ffx = FullFieldXas()
if os.path.isfile(".XSDataInputFullFieldXAS.xml"):
ffx.load(".XSDataInputFullFieldXAS.xml")
ffx.dontAlign = dontAlign
if dontAlign:
print("*"*80)
print("*" + "Skipping image alignement part".center(78) + "*")
print("*"*80)
ffx.setup(_listInput=paths, _mode=mode)
if not isinstance(ffx.reference, int):
ffx.uploadReferenceFrame(ffx.reference)
ffx.save(".XSDataInputFullFieldXAS.xml")
ffx.dump("analysis-%s.json" % time.strftime("%Y%m%d-%Hh%Mm%Ss"))
edna = EDParallelExecute(ffx.pluginName, ffx.makeXML, _functXMLerr=ffx.error, _bVerbose=True, _bDebug=debug, _iNbThreads=iNbCPU)
edna.runEDNA(ffx.listInput, ffx.strMode , ffx.bNewerOnly)
EDVerbose.WARNING("Back to main !")
EDJob.synchronizeAll()
ffx.updateAttrRefImage()
EDPluginControlAlignStackv1_0.showData()
if keepShare:
EDShare.close()
elif (ffx.getNrErrors() == 0) and (not EDVerbose.isVerboseDebug()):
EDVerbose.WARNING("All processing finished successfully: Remove EDShare's big HDF5 file")
EDShare.close(remove=True)
else:
EDShare.close()
print("All %i jobs queued after %.3fs" % (len(args), time.time() - reprocess.startTime))
reprocess.join()
if yappi: yappi.stop()
print("All %i jobs processed after %.3fs" % (len(args), time.time() - reprocess.startTime))
print reprocess.statistics()
if yappi:
stat = yappi.get_stats(sort_type=yappi.SORTTYPE_TTOT)
res = {}
for i in stat.func_stats:
if i[0] in res:
res[i[0]][0] += i[1]
res[i[0]][1] += i[2]
else:
res[i[0]] = [i[1], i[2]]
keys = res.keys()
keys.sort(sortn)
with open("yappi.out", "w") as f:
f.write("ncall\t\ttotal\t\tpercall\t\tfunction%s" % (os.linesep))
for i in keys:
f.write("%8s\t%16s\t%16s\t%s%s" % (res[i][0], res[i][1], res[i][1] / res[i][0], i, os.linesep))
print("Profiling information written in yappi.out")
edJob = EDJob(options.plugin.replace("EDPluginBioSaxsHPLC", "EDPluginBioSaxsFlushHPLC"))
edJob.setDataInput(open(fullargs[-1], "r").read())
edJob.execute()
edJob.synchronizeAll()
if options.profile:
for i in EDObject.analyze_profiling():
print(i)
print("All %i jobs processed after %.3fs" %
(len(args), time.time() - reprocess.startTime))
print reprocess.statistics()
if yappi:
stat = yappi.get_stats(sort_type=yappi.SORTTYPE_TTOT)
res = {}
for i in stat.func_stats:
if i[0] in res:
res[i[0]][0] += i[1]
res[i[0]][1] += i[2]
else:
res[i[0]] = [i[1], i[2]]
keys = res.keys()
keys.sort(sortn)
with open("yappi.out", "w") as f:
f.write("ncall\t\ttotal\t\tpercall\t\tfunction%s" % (os.linesep))
for i in keys:
f.write("%8s\t%16s\t%16s\t%s%s" %
(res[i][0], res[i][1], res[i][1] / res[i][0], i,
os.linesep))
print("Profiling information written in yappi.out")
edJob = EDJob(
options.plugin.replace("EDPluginBioSaxsHPLC",
"EDPluginBioSaxsFlushHPLC"))
edJob.setDataInput(open(fullargs[-1], "r").read())
edJob.execute()
edJob.synchronizeAll()
if options.profile:
for i in EDObject.analyze_profiling():
print(i)
