def successJobExecution(self, jobId):
self.DEBUG("In %s.successJobExecution(%s)" % (self.__class__.__name__, jobId))
with self.locked():
self.__semaphoreNbThreads.release()
EDJob.cleanJobfromID(jobId, False)
self.lastSuccess = jobId
gc.collect()
def process(_listInputFile, _output, dummy=0, autoscale=False, center=None, width=None, blending=None, mask=None):
"""
call EDNA with this options:
@param _listInputFile: list of input files as strings
@param _output: output file name
@param dummy: value for dummy pixels
@param autoscale: shall image intensity be scaled (boolean)
@param center: 2-list of int representing the center of the ROI
@param width: 2-list of int representing the width of the ROI
@param blending: blending method: max, mean or min
@param mask: name of the file containing the mask
"""
xsd = XSDataInputStitchImage()
xsd.dummyValue = XSDataDouble(dummy)
xsd.autoscale = XSDataBoolean(autoscale)
if blending:
xsd.blending = XSDataString(blending)
if mask:
xsd.mask = XSDataImage(XSDataString(mask))
xsd.outputImage = XSDataImage(XSDataString(_output))
xsd.inputImages = [XSDataImage(XSDataString(i)) for i in _listInputFile]
if isinstance(width, list):
xsd.widthROI = [XSDataInteger(i) for i in width]
if isinstance(center, list):
xsd.centerROI = [XSDataInteger(i) for i in center]
job = EDJob(EDNAPluginName)
job.setDataInput(xsd)
job.execute()
def successJobExecution(self, jobId):
self.DEBUG("In %s.successJobExecution(%s)" % (self.get_name(), jobId))
with self.locked():
self.__semaphoreNbThreads.release()
EDJob.cleanJobfromID(jobId, False)
self.lastSuccess = jobId
self.push_change_event("jobSuccess", jobId)
gc.collect()
def successJobExecution(self, jobId):
self.DEBUG("In %s.successJobExecution(%s)" % (self.get_name(), jobId))
with self.locked():
self.__semaphoreNbThreads.release()
EDJob.cleanJobfromID(jobId, False)
self.lastSuccess = jobId
self.push_change_event("jobSuccess", jobId)
gc.collect()
def unitTestSetGetData(self):
"""
check the status after a job creation
"""
EDVerbose.DEBUG("EDTestCaseEDJob.unitTestSetGetData")
edJob = EDJob(self.strPluginName)
edJob.setDataInput(self.strXmlInput)
EDAssert.equal(self.strXmlInput, edJob.getDataInput().strip(), "Data Input is correctly set")
EDAssert.equal("uninitialized", edJob.getStatus(), "Job %s is still ''uninitialized''" % edJob.getJobId())
def failureJobExecution(self, jobId):
self.DEBUG("In %s.failureJobExecution(%s)" % (self.__class__.__name__, jobId))
with self.locked():
self.__semaphoreNbThreads.release()
EDJob.cleanJobfromID(jobId, False)
self.lastFailure = jobId
sys.stdout.flush()
sys.stderr.flush()
gc.collect()
def waitForAllProcessToFinish(self):
"""
as it names says, this method waits for all plug-ins which are currently running to finish before returning.
"""
self.screen("Waiting for launched jobs to finish .")
while (self.getNbRunning() > 0):
time.sleep(1)
sys.stderr.write(".")
sys.stderr.write("Done.\n")
EDJob.stats()
def failureJobExecution(self, jobId):
self.DEBUG("In %s.failureJobExecution(%s)" % (self.get_name(), jobId))
with self.locked():
self.__semaphoreNbThreads.release()
EDJob.cleanJobfromID(jobId, False)
self.lastFailure = jobId
self.push_change_event("jobFailure", jobId)
sys.stdout.flush()
sys.stderr.flush()
gc.collect()
def failureJobExecution(self, jobId):
self.DEBUG("In %s.failureJobExecution(%s)" % (self.get_name(), jobId))
with self.locked():
self.__semaphoreNbThreads.release()
EDJob.cleanJobfromID(jobId, False)
self.lastFailure = jobId
self.push_change_event("jobFailure", jobId)
sys.stdout.flush()
sys.stderr.flush()
gc.collect()
def waitForAllProcessToFinish(self):
"""
as it names says, this method waits for all plug-ins which are currently running to finish before returning.
"""
self.screen("Waiting for launched jobs to finish .")
while (self.getNbRunning() > 0):
time.sleep(1)
sys.stderr.write(".")
sys.stderr.write("Done.\n")
EDJob.stats()
def unitTestInitialState(self):
"""
check the status after a job creation
"""
EDVerbose.DEBUG("EDTestCaseEDJob.unitTestInitialState")
edJob = EDJob(self.strPluginName)
strJobId = edJob.getJobId()
EDVerbose.DEBUG("EDJobId is: %s" % strJobId)
EDAssert.equal(2, len(strJobId.split("-")), "JobID is composed of 2 parts")
EDAssert.equal(True, strJobId.split("-")[1].isdigit(), "JobID's second part is an integer")
EDAssert.equal("uninitialized", edJob.getStatus(), "Initial stat is ''uninitialized''")
def unitTestInitialState(self):
"""
check the status after a job creation
"""
EDVerbose.DEBUG("EDTestCaseEDJob.unitTestInitialState")
edJob = EDJob(self.strPluginName)
strJobId = edJob.getJobId()
EDVerbose.DEBUG("EDJobId is: %s" % strJobId)
EDAssert.equal(2, len(strJobId.split("-")), "JobID is composed of 2 parts")
EDAssert.equal(True, strJobId.split("-")[1].isdigit(), "JobID's second part is an integer")
EDAssert.equal("uninitialized", edJob.getStatus(), "Initial stat is ''uninitialized''")
def startJob(self, argin):
"""
@param argin: 2-list [ "EDPluginName", "<xml/><XSDataInputPluginName>...."]
@return: jobID which is a sting: Plugin-000001
"""
self.DEBUG("In %s.startJob()" % self.get_name())
name, xsd = argin[:2]
if xsd.strip() == "":
return
edJob = EDJob(name)
if edJob is None:
return "Error in load Plugin"
jobId = edJob.getJobId()
edJob.setDataInput(xsd)
self.jobQueue.put(edJob)
return jobId
def getJobOutput(self, jobId):
"""
Retrieve XML output form a job
@param jobId: name of the job
@return: output from a job
"""
return EDJob.getDataOutputFromId(jobId)
def getJobInput(self, jobId):
"""
Retrieve XML input from a job
@param jobId: name of the job
@return: xml input from a job
"""
return EDJob.getDataInputFromId(jobId)
def callBack(self, _strJobId):
"""
Example of Call Back function ...
"""
myJob = EDJob.getJobFromID(_strJobId)
strOutput = myJob.getDataOutput().strip()
EDAssert.equal(strOutput, self.strXmlInput, "From Callback: Output is OK")
EDAssert.equal("success", myJob.getStatus(), "From Callback: Job %s is finished with ''success''" % _strJobId)
def callBack(self, _strJobId):
"""
Example of Call Back function ...
"""
myJob = EDJob.getJobFromID(_strJobId)
strOutput = myJob.getDataOutput().strip()
EDAssert.equal(strOutput, self.strXmlInput, "From Callback: Output is OK")
EDAssert.equal("success", myJob.getStatus(), "From Callback: Job %s is finished with ''success''" % _strJobId)
def statistics(self):
"""
retrieve some statistics about past jobs.
"""
with self.statLock:
fStartStat = time.time()
self.lastStatistics = EDJob.stats()
self.lastStatistics += os.linesep + "Statistics collected on %s, the collect took: %.3fs" % (time.asctime(), time.time() - fStartStat)
return self.lastStatistics
def unitTestExecuteCallbackSuccess(self):
"""
check the execution of a job (without callback)
"""
EDVerbose.DEBUG("EDTestCaseEDJob.unitTestExecuteCallbackSuccess")
edJob = EDJob(self.strPluginName)
edJob.connectSUCCESS(self.callBack)
edJob.setDataInput(self.strXmlInput)
strJobId = edJob.execute()
strStatus = edJob.getStatus()
EDVerbose.DEBUG("Job %s in State ''%s''" % (strJobId, strStatus))
def startJob(self, argin):
"""
@param argin: 2-list [ "EDPluginName", "<xml/><XSDataInputPluginName>...."]
@return: jobID which is a sting: Plugin-000001
"""
self.DEBUG("In %s.startJob()" % self.get_name())
name, xsd = argin[:2]
if xsd.strip() == "":
return
edJob = EDJob(name)
if edJob is None:
return "Error in load Plugin"
jobId = edJob.getJobId()
edJob.setDataInput(xsd)
self.jobQueue.put(edJob)
if self.processingSem._Semaphore__value > 0 :
t = threading.Thread(target=self.startProcessing)
t.start()
return jobId
def startJob(self, argin):
"""
@param argin: 2-list [ "EDPluginName", "<xml/><XSDataInputPluginName>...."]
@return: jobID which is a sting: Plugin-000001
"""
self.DEBUG("In %s.startJob()" % self.get_name())
name, xsd = argin[:2]
if xsd.strip() == "":
return
edJob = EDJob(name)
if edJob is None:
return "Error in load Plugin"
jobId = edJob.getJobId()
edJob.setDataInput(xsd)
self.jobQueue.put(edJob)
if self.processingSem._Semaphore__value > 0:
t = threading.Thread(target=self.startProcessing)
t.start()
return jobId
def start(self, _strXmlInput):
"""
Launch EDNA with the given XML stream
@param _strXmlInput: XML to be passed to the plugin
@type _strXmlInput: python string representing the XML data structure
"""
jobid = None
if _strXmlInput not in ["", None]:
job = EDJob(self.__strPluginName)
job.setDataInput(_strXmlInput)
job.connectFAILURE(self.failureJobExecution)
job.connectSUCCESS(self.successJobExecution)
job.connectCallBack(self.unregisterJob)
self.semaphoreNbThreadsAcquire()
jobid = job.execute()
self.DEBUG("Running Job id %s" % jobid)
if jobid is None:
self.semaphoreNbThreadsRelease()
return jobid
def startJob(self, xsd):
"""
@param xsd: XML data structure as a string or path to a string
@return: jobID which is a sting: Plugin-000001
"""
self.DEBUG("In %s.startJob()" % self.__class__.__name__)
if xsd.strip() == "":
return
if os.path.isfile(xsd):
xsd = open(xsd, "rb").read()
edJob = EDJob(self.pluginName)
if edJob is None:
return "Error in load Plugin"
jobId = edJob.getJobId()
edJob.setDataInput(xsd)
self.jobQueue.put(edJob)
if self.processingSem._Semaphore__value > 0:
t = threading.Thread(target=self.startProcessing)
t.start()
return jobId
def failureJobExecution(self, _jobId):
"""
Method called when the execution of the plugin finishes with failure
@param _jobId: string of type EDPluginName-number
"""
self.DEBUG("EDParallelExcecute.failureJobExecution for %s" % _jobId)
self.semaphoreNbThreadsRelease()
with self.locked():
if self.__functXMLerr is not None:
self.__functXMLerr(EDJob.getJobFromID(_jobId).getPlugin().getDataInput())
def startJob(self, xsd):
"""
@param xsd: XML data structure as a string or path to a string
@return: jobID which is a sting: Plugin-000001
"""
self.DEBUG("In %s.startJob()" % self.__class__.__name__)
if xsd.strip() == "":
return
if os.path.isfile(xsd):
xsd = open(xsd, "rb").read()
edJob = EDJob(self.pluginName)
if edJob is None:
return "Error in load Plugin"
jobId = edJob.getJobId()
edJob.setDataInput(xsd)
self.jobQueue.put(edJob)
if self.processingSem._Semaphore__value > 0 :
t = threading.Thread(target=self.startProcessing)
t.start()
return jobId
def quitEdna(self, delay=10):
"""
Try to quit properly
@param: delay for finishing jobs
"""
self.quit = True
self.DEBUG("In %s.quitEdna()" % self.get_name())
self.screen("QuitEdna: Close input pipe" % delay)
self.processingThread.join(delay)
self.screen("QuitEdna: Finish the processing (timeout = 10s)")
t0 = time.time()
remaining = EDJob.countRunning()
while (remaining > 0) and (time.time() < t0 + delay):
self.screen("%i remaining jobs" % remaining)
time.sleep(1)
remaining = EDJob.countRunning()
self.screen("QuitEdna: Close output pipe (timeout = 10s)")
self.finishingThread.join(delay)
self.screen("Quitting tango-EdnaDS")
sys.exit()
def process_event(self):
"""
process finished jobs on the tango side (issue with tango locks)
"""
while not self.quit:
jobid = self.eventQueue.get()
status = EDJob.getStatusFromID(jobid)
if status == EDJob.PLUGIN_STATE_SUCCESS:
self.push_change_event("jobSuccess", jobid)
else:
self.push_change_event("jobFailure", jobid)
def failureJobExecution(self, _jobId):
"""
Method called when the execution of the plugin finishes with failure
@param _jobId: string of type EDPluginName-number
"""
self.DEBUG("EDParallelExcecute.failureJobExecution for %s" % _jobId)
self.semaphoreNbThreadsRelease()
with self.locked():
if self.__functXMLerr is not None:
self.__functXMLerr(
EDJob.getJobFromID(_jobId).getPlugin().getDataInput())
def process(_listInputFile,
_output,
dummy=0,
autoscale=False,
center=None,
width=None,
blending=None,
mask=None):
"""
call EDNA with this options:
@param _listInputFile: list of input files as strings
@param _output: output file name
@param dummy: value for dummy pixels
@param autoscale: shall image intensity be scaled (boolean)
@param center: 2-list of int representing the center of the ROI
@param width: 2-list of int representing the width of the ROI
@param blending: blending method: max, mean or min
@param mask: name of the file containing the mask
"""
xsd = XSDataInputStitchImage()
xsd.dummyValue = XSDataDouble(dummy)
xsd.autoscale = XSDataBoolean(autoscale)
if blending:
xsd.blending = XSDataString(blending)
if mask:
xsd.mask = XSDataImage(XSDataString(mask))
xsd.outputImage = XSDataImage(XSDataString(_output))
xsd.inputImages = [XSDataImage(XSDataString(i)) for i in _listInputFile]
if isinstance(width, list):
xsd.widthROI = [XSDataInteger(i) for i in width]
if isinstance(center, list):
xsd.centerROI = [XSDataInteger(i) for i in center]
job = EDJob(EDNAPluginName)
job.setDataInput(xsd)
job.execute()
def unitTestExecuteCallbackSuccess(self):
"""
check the execution of a job (without callback)
"""
EDVerbose.DEBUG("EDTestCaseEDJob.unitTestExecuteCallbackSuccess")
edJob = EDJob(self.strPluginName)
edJob.connectSUCCESS(self.callBack)
edJob.setDataInput(self.strXmlInput)
strJobId = edJob.execute()
strStatus = edJob.getStatus()
EDVerbose.DEBUG("Job %s in State ''%s''" % (strJobId, strStatus))
def unitTestSetGetData(self):
"""
check the status after a job creation
"""
EDVerbose.DEBUG("EDTestCaseEDJob.unitTestSetGetData")
edJob = EDJob(self.strPluginName)
edJob.setDataInput(self.strXmlInput)
EDAssert.equal(self.strXmlInput,
edJob.getDataInput().strip(),
"Data Input is correctly set")
EDAssert.equal("uninitialized", edJob.getStatus(),
"Job %s is still ''uninitialized''" % edJob.getJobId())
def cleanJob(self, jobId):
return EDJob.cleanJobFromID(jobId)
def unitTestExecute(self):
"""
check the execution of a job (without callback)
"""
EDVerbose.DEBUG("EDTestCaseEDJob.unitTestExecute")
edJob = EDJob(self.strPluginName)
strJobId = edJob.getJobId()
edJob.setDataInput(self.strXmlInput)
ref = edJob.execute()
EDAssert.equal(strJobId, ref, "JobId has not changed")
strStatus = edJob.getStatus()
EDVerbose.WARNING("Job %s in State %s" % (strJobId, strStatus))
while strStatus in ["running", "uninitialized"]:
EDVerbose.WARNING("Job %s in state %s" % (strJobId, strStatus))
time.sleep(0.01)
strStatus = edJob.getStatus()
xsdOut = edJob.getDataOutput()
while xsdOut is None:
EDVerbose.WARNING("No Output data, still waiting for output data to arrive, %s" % edJob.getStatus())
time.sleep(0.01)
xsdOut = edJob.getDataOutput()
strOutput = xsdOut.strip()
strStatus = edJob.getStatus()
while strStatus == "running":
EDVerbose.WARNING("Job %s is still in state %s" % (strJobId, strStatus))
time.sleep(0.01)
strStatus = edJob.getStatus()
EDAssert.equal(strOutput, self.strXmlInput, "Output is OK")
EDAssert.equal("success", edJob.getStatus(), "Job %s is finished with ''success''" % edJob.getJobId())
print("All %i jobs processed after %.3fs" %
(len(args), time.time() - reprocess.startTime))
print reprocess.statistics()
if yappi:
stat = yappi.get_stats(sort_type=yappi.SORTTYPE_TTOT)
res = {}
for i in stat.func_stats:
if i[0] in res:
res[i[0]][0] += i[1]
res[i[0]][1] += i[2]
else:
res[i[0]] = [i[1], i[2]]
keys = res.keys()
keys.sort(sortn)
with open("yappi.out", "w") as f:
f.write("ncall\t\ttotal\t\tpercall\t\tfunction%s" % (os.linesep))
for i in keys:
f.write("%8s\t%16s\t%16s\t%s%s" %
(res[i][0], res[i][1], res[i][1] / res[i][0], i,
os.linesep))
print("Profiling information written in yappi.out")
edJob = EDJob(
options.plugin.replace("EDPluginBioSaxsHPLC",
"EDPluginBioSaxsFlushHPLC"))
edJob.setDataInput(open(fullargs[-1], "r").read())
edJob.execute()
edJob.synchronizeAll()
if options.profile:
for i in EDObject.analyze_profiling():
print(i)
def runEdnaPlugin(execPath, pluginName, isDebug, xml, additionalPaths=None):
'''
execPath - path to run plugin in
pluginName - plugin name
isDebug - True if should run edna in debug mode
xml - xml input to edna
additionalPaths - list of other python path locations
You must set EDNA_HOME to use this method
This method blocks until the EDJob has reached a final status
'''
if (not 'EDNA_HOME' in os.environ):
raise Exception(
"Cannot locate EDNA_HOME. Please set before running Edna plugins.")
if (not 'EDNA_SITE' in os.environ):
raise Exception(" Please set EDNA_SITE before running Edna plugins.")
'''
Add edna to path
'''
ednaKernelPath = os.environ['EDNA_HOME'] + "/kernel/src"
sys.path.insert(0, ednaKernelPath)
'''
If there are any additional paths such as fabio, add these
'''
if (not additionalPaths is None and len(additionalPaths) > 0):
for path in additionalPaths:
sys.path.append(path)
os.chdir(execPath)
from EDVerbose import EDVerbose
if (isDebug):
EDVerbose.setVerboseDebugOn()
else:
EDVerbose.setVerboseOn()
EDVerbose.setVerboseDebugOff()
from EDJob import EDJob
EDVerbose.setLogFileName(execPath + "/" + pluginName + ".log")
edJob = EDJob(pluginName)
edJob.setDataInput(xml)
edJob.execute()
edJob.synchronize(
) # In theory should mean that the following loop is not needed
# Unhelpful way of waiting for EDJob to be finished
# TODO Fix this in EDJob some time
while (True):
status = edJob.getStatus()
if (status is None):
time.sleep(0.2) # 200 ms
continue
if ("failure" == status):
raise Exception("EDJob failed! ")
if ("success" == status):
break
ret = edJob.getDataOutput()
return str(ret)
else:
ffx = FullFieldXas()
if os.path.isfile(".XSDataInputFullFieldXAS.xml"):
ffx.load(".XSDataInputFullFieldXAS.xml")
ffx.dontAlign = dontAlign
if dontAlign:
print ("*" * 80)
print ("*" + "Skipping image alignement part".center(78) + "*")
print ("*" * 80)
ffx.setup(_listInput=paths, _mode=mode)
if not isinstance(ffx.reference, int):
ffx.uploadReferenceFrame(ffx.reference)
ffx.save(".XSDataInputFullFieldXAS.xml")
ffx.dump("analysis-%s.json" % time.strftime("%Y%m%d-%Hh%Mm%Ss"))
edna = EDParallelExecute(
ffx.pluginName, ffx.makeXML, _functXMLerr=ffx.error, _bVerbose=True, _bDebug=debug, _iNbThreads=iNbCPU
)
edna.runEDNA(ffx.listInput, ffx.strMode, ffx.bNewerOnly)
EDVerbose.WARNING("Back to main !")
EDJob.synchronizeAll()
ffx.updateAttrRefImage()
EDPluginControlAlignStackv1_0.showData()
if keepShare:
EDShare.close()
elif (ffx.getNrErrors() == 0) and (not EDVerbose.isVerboseDebug()):
EDVerbose.WARNING("All processing finished successfully: Remove EDShare's big HDF5 file")
EDShare.close(remove=True)
else:
EDShare.close()
if replay:
ffx = FullFieldXas.replay_from(replay)
else:
ffx = FullFieldXas()
if os.path.isfile(".XSDataInputFullFieldXAS.xml"):
ffx.load(".XSDataInputFullFieldXAS.xml")
ffx.dontAlign = dontAlign
if dontAlign:
print("*"*80)
print("*" + "Skipping image alignement part".center(78) + "*")
print("*"*80)
ffx.setup(_listInput=paths, _mode=mode)
if not isinstance(ffx.reference, int):
ffx.uploadReferenceFrame(ffx.reference)
ffx.save(".XSDataInputFullFieldXAS.xml")
ffx.dump("analysis-%s.json" % time.strftime("%Y%m%d-%Hh%Mm%Ss"))
edna = EDParallelExecute(ffx.pluginName, ffx.makeXML, _functXMLerr=ffx.error, _bVerbose=True, _bDebug=debug, _iNbThreads=iNbCPU)
edna.runEDNA(ffx.listInput, ffx.strMode , ffx.bNewerOnly)
EDVerbose.WARNING("Back to main !")
EDJob.synchronizeAll()
ffx.updateAttrRefImage()
EDPluginControlAlignStackv1_0.showData()
if keepShare:
EDShare.close()
elif (ffx.getNrErrors() == 0) and (not EDVerbose.isVerboseDebug()):
EDVerbose.WARNING("All processing finished successfully: Remove EDShare's big HDF5 file")
EDShare.close(remove=True)
else:
EDShare.close()
for i in fullargs:
reprocess.startJob(i)
print("All %i jobs queued after %.3fs" %
(len(args), time.time() - reprocess.startTime))
reprocess.join()
if yappi: yappi.stop()
print("All %i jobs processed after %.3fs" %
(len(args), time.time() - reprocess.startTime))
print reprocess.statistics()
if yappi:
stat = yappi.get_stats(sort_type=yappi.SORTTYPE_TTOT)
res = {}
for i in stat.func_stats:
if i[0] in res:
res[i[0]][0] += i[1]
res[i[0]][1] += i[2]
else:
res[i[0]] = [i[1], i[2]]
keys = res.keys()
keys.sort(sortn)
with open("yappi.out", "w") as f:
f.write("ncall\t\ttotal\t\tpercall\t\tfunction%s" % (os.linesep))
for i in keys:
f.write("%8s\t%16s\t%16s\t%s%s" %
(res[i][0], res[i][1], res[i][1] / res[i][0], i,
os.linesep))
print("Profiling information written in yappi.out")
edJob = EDJob("EDPluginBioSaxsFlushHPLCv1_0")
edJob.setDataInput(open(fullargs[-1], "r").read())
edJob.execute()
def unitTestExecute(self):
"""
check the execution of a job (without callback)
"""
EDVerbose.DEBUG("EDTestCaseEDJob.unitTestExecute")
edJob = EDJob(self.strPluginName)
strJobId = edJob.getJobId()
edJob.setDataInput(self.strXmlInput)
ref = edJob.execute()
EDAssert.equal(strJobId, ref, "JobId has not changed")
strStatus = edJob.getStatus()
EDVerbose.WARNING("Job %s in State %s" % (strJobId, strStatus))
while strStatus in ["running", "uninitialized"]:
EDVerbose.WARNING("Job %s in state %s" % (strJobId, strStatus))
time.sleep(0.01)
strStatus = edJob.getStatus()
xsdOut = edJob.getDataOutput()
while xsdOut is None:
EDVerbose.WARNING("No Output data, still waiting for output data to arrive, %s" % edJob.getStatus())
time.sleep(0.01)
xsdOut = edJob.getDataOutput()
strOutput = xsdOut.strip()
strStatus = edJob.getStatus()
while strStatus == "running":
EDVerbose.WARNING("Job %s is still in state %s" % (strJobId, strStatus))
time.sleep(0.01)
strStatus = edJob.getStatus()
EDAssert.equal(strOutput, self.strXmlInput, "Output is OK")
EDAssert.equal("success", edJob.getStatus(), "Job %s is finished with ''success''" % edJob.getJobId())
def start(self, _strXmlInput):
"""
Launch EDNA with the given XML stream
@param _strXmlInput: XML to be passed to the plugin
@type _strXmlInput: python string representing the XML data structure
"""
jobid = None
if _strXmlInput not in ["", None]:
job = EDJob(self.__strPluginName)
job.setDataInput(_strXmlInput)
job.connectFAILURE(self.failureJobExecution)
job.connectSUCCESS(self.successJobExecution)
job.connectCallBack(self.unregisterJob)
self.semaphoreNbThreadsAcquire()
jobid = job.execute()
self.DEBUG("Running Job id %s" % jobid)
if jobid is None:
self.semaphoreNbThreadsRelease()
return jobid
def getJobState(self, jobId):
return EDJob.getStatusFromID(jobId)
os.makedirs(working_dir)
os.chdir(working_dir)
if yappi: yappi.start()
for i in fullargs:
reprocess.startJob(i)
print("All %i jobs queued after %.3fs" % (len(args), time.time() - reprocess.startTime))
reprocess.join()
if yappi: yappi.stop()
print("All %i jobs processed after %.3fs" % (len(args), time.time() - reprocess.startTime))
print reprocess.statistics()
if yappi:
stat = yappi.get_stats(sort_type=yappi.SORTTYPE_TTOT)
res = {}
for i in stat.func_stats:
if i[0] in res:
res[i[0]][0] += i[1]
res[i[0]][1] += i[2]
else:
res[i[0]] = [i[1], i[2]]
keys = res.keys()
keys.sort(sortn)
with open("yappi.out", "w") as f:
f.write("ncall\t\ttotal\t\tpercall\t\tfunction%s" % (os.linesep))
for i in keys:
f.write("%8s\t%16s\t%16s\t%s%s" % (res[i][0], res[i][1], res[i][1] / res[i][0], i, os.linesep))
print("Profiling information written in yappi.out")
edJob = EDJob(options.plugin.replace("EDPluginBioSaxsHPLC", "EDPluginBioSaxsFlushHPLC"))
edJob.setDataInput(open(fullargs[-1], "r").read())
edJob.execute()
def process(self):
for fn in self.dataFiles:
EDVerbose.screen("Processing file %s" % fn)
edj = EDJob(self.EDNAPluginName)
edj.dataInput = self.fileName2xml(fn)
edj.connectSUCCESS(self.XMLsuccess)
edj.connectFAILURE(self.XMLerr)
self.queue.put(edj)
if self.process_sem._Semaphore__value > 0 :
t = threading.Thread(target=self.startProcessing)
t.start()
EDVerbose.screen("Back in main")
while self.cpu_sem._Semaphore__value < self.nbcpu:
time.sleep(0.1)
EDJob.synchronizeAll()
EDJob.stats()
def runEdnaPlugin(execPath, pluginName, isDebug, xml, additionalPaths=None):
'''
execPath - path to run plugin in
pluginName - plugin name
isDebug - True if should run edna in debug mode
xml - xml input to edna
additionalPaths - list of other python path locations
You must set EDNA_HOME to use this method
This method blocks until the EDJob has reached a final status
'''
if (not 'EDNA_HOME' in os.environ):
raise Exception("Cannot locate EDNA_HOME. Please set before running Edna plugins.")
if (not 'EDNA_SITE' in os.environ):
raise Exception(" Please set EDNA_SITE before running Edna plugins.")
'''
Add edna to path
'''
ednaKernelPath = os.environ['EDNA_HOME']+"/kernel/src"
sys.path.insert(0, ednaKernelPath)
'''
If there are any additional paths such as fabio, add these
'''
if (not additionalPaths is None and len(additionalPaths)>0):
for path in additionalPaths:
sys.path.append(path)
os.chdir(execPath)
from EDVerbose import EDVerbose
if (isDebug):
EDVerbose.setVerboseDebugOn()
else:
EDVerbose.setVerboseOn()
EDVerbose.setVerboseDebugOff()
from EDJob import EDJob
EDVerbose.setLogFileName(execPath+"/"+pluginName+".log")
edJob = EDJob(pluginName)
edJob.setDataInput(xml)
edJob.execute()
edJob.synchronize() # In theory should mean that the following loop is not needed
# Unhelpful way of waiting for EDJob to be finished
# TODO Fix this in EDJob some time
while(True):
status = edJob.getStatus()
if (status is None):
time.sleep(0.2) # 200 ms
continue
if ("failure" == status):
raise Exception("EDJob failed! ")
if ("success" == status):
break
ret = edJob.getDataOutput()
return str(ret)
elif i == "-v":
bVerbose = True
elif os.path.isdir(i):
pathWebDoc = i
if WebSite and pathWebDoc:
print "Generating web documentation in ", pathWebDoc
rmdir(pathWebDoc, bypass=True)
for oneproject in findProjects(pyStrEdnaHomePath):
print ("Generating HTML pages for " + oneproject)
docPath = os.path.join(pathWebDoc, oneproject)
os.makedirs(docPath, int("755", 8))
listOfPythonFiles = findFile(os.path.join(pyStrEdnaHomePath, oneproject))
listOfPythonFiles.sort()
if len(listOfPythonFiles) > 0:
epydocJob = EDJob("EDPluginExecEpydocv1_0")
dictJobs[oneproject] = epydocJob
xsd = XSDataInputEpydoc()
xsd.setDocPath(XSDataFile(XSDataString(docPath)))
xsd.setProjectName(XSDataString(oneproject))
xsd.setDocType(XSDataString(docFormat))
if bVerbose:
xsd.setVerbosity(XSDataInteger(1))
else:
xsd.setVerbosity(XSDataInteger(-1))
xsd.setSources([XSDataFile(XSDataString(oneFile)) for oneFile in listOfPythonFiles])
epydocJob.setDataInput(xsd)
epydocJob.execute()
else:
print ("Error: No python files for project %s" % oneproject)
def getJobState(self, jobId):
return EDJob.getStatusFromID(jobId)
bVerbose = True
elif os.path.isdir(i):
pathWebDoc = i
if WebSite and pathWebDoc:
print "Generating web documentation in ", pathWebDoc
rmdir(pathWebDoc, bypass=True)
for oneproject in findProjects(pyStrEdnaHomePath):
print("Generating HTML pages for " + oneproject)
docPath = os.path.join(pathWebDoc, oneproject)
os.makedirs(docPath, int("755", 8))
listOfPythonFiles = findFile(
os.path.join(pyStrEdnaHomePath, oneproject))
listOfPythonFiles.sort()
if len(listOfPythonFiles) > 0:
epydocJob = EDJob("EDPluginExecEpydocv1_0")
dictJobs[oneproject] = epydocJob
xsd = XSDataInputEpydoc()
xsd.setDocPath(XSDataFile(XSDataString(docPath)))
xsd.setProjectName(XSDataString(oneproject))
xsd.setDocType(XSDataString(docFormat))
if bVerbose:
xsd.setVerbosity(XSDataInteger(1))
else:
xsd.setVerbosity(XSDataInteger(-1))
xsd.setSources([
XSDataFile(XSDataString(oneFile))
for oneFile in listOfPythonFiles
])
epydocJob.setDataInput(xsd)
epydocJob.execute()
def cleanJob(self, jobId):
return EDJob.cleanJobFromID(jobId)
print("All %i jobs queued after %.3fs" % (len(args), time.time() - reprocess.startTime))
reprocess.join()
if yappi: yappi.stop()
print("All %i jobs processed after %.3fs" % (len(args), time.time() - reprocess.startTime))
print reprocess.statistics()
if yappi:
stat = yappi.get_stats(sort_type=yappi.SORTTYPE_TTOT)
res = {}
for i in stat.func_stats:
if i[0] in res:
res[i[0]][0] += i[1]
res[i[0]][1] += i[2]
else:
res[i[0]] = [i[1], i[2]]
keys = res.keys()
keys.sort(sortn)
with open("yappi.out", "w") as f:
f.write("ncall\t\ttotal\t\tpercall\t\tfunction%s" % (os.linesep))
for i in keys:
f.write("%8s\t%16s\t%16s\t%s%s" % (res[i][0], res[i][1], res[i][1] / res[i][0], i, os.linesep))
print("Profiling information written in yappi.out")
edJob = EDJob(options.plugin.replace("EDPluginBioSaxsHPLC", "EDPluginBioSaxsFlushHPLC"))
edJob.setDataInput(open(fullargs[-1], "r").read())
edJob.execute()
edJob.synchronizeAll()
if options.profile:
for i in EDObject.analyze_profiling():
print(i)