def unitTestExecute(self):
"""
check the execution of a job (without callback)
"""
EDVerbose.DEBUG("EDTestCaseEDJob.unitTestExecute")
edJob = EDJob(self.strPluginName)
strJobId = edJob.getJobId()
edJob.setDataInput(self.strXmlInput)
ref = edJob.execute()
EDAssert.equal(strJobId, ref, "JobId has not changed")
strStatus = edJob.getStatus()
EDVerbose.WARNING("Job %s in State %s" % (strJobId, strStatus))
while strStatus in ["running", "uninitialized"]:
EDVerbose.WARNING("Job %s in state %s" % (strJobId, strStatus))
time.sleep(0.01)
strStatus = edJob.getStatus()
xsdOut = edJob.getDataOutput()
while xsdOut is None:
EDVerbose.WARNING("No Output data, still waiting for output data to arrive, %s" % edJob.getStatus())
time.sleep(0.01)
xsdOut = edJob.getDataOutput()
strOutput = xsdOut.strip()
strStatus = edJob.getStatus()
while strStatus == "running":
EDVerbose.WARNING("Job %s is still in state %s" % (strJobId, strStatus))
time.sleep(0.01)
strStatus = edJob.getStatus()
EDAssert.equal(strOutput, self.strXmlInput, "Output is OK")
EDAssert.equal("success", edJob.getStatus(), "Job %s is finished with ''success''" % edJob.getJobId())
def process(_listInputFile,
_output,
dummy=0,
autoscale=False,
center=None,
width=None,
blending=None,
mask=None):
"""
call EDNA with this options:
@param _listInputFile: list of input files as strings
@param _output: output file name
@param dummy: value for dummy pixels
@param autoscale: shall image intensity be scaled (boolean)
@param center: 2-list of int representing the center of the ROI
@param width: 2-list of int representing the width of the ROI
@param blending: blending method: max, mean or min
@param mask: name of the file containing the mask
"""
xsd = XSDataInputStitchImage()
xsd.dummyValue = XSDataDouble(dummy)
xsd.autoscale = XSDataBoolean(autoscale)
if blending:
xsd.blending = XSDataString(blending)
if mask:
xsd.mask = XSDataImage(XSDataString(mask))
xsd.outputImage = XSDataImage(XSDataString(_output))
xsd.inputImages = [XSDataImage(XSDataString(i)) for i in _listInputFile]
if isinstance(width, list):
xsd.widthROI = [XSDataInteger(i) for i in width]
if isinstance(center, list):
xsd.centerROI = [XSDataInteger(i) for i in center]
job = EDJob(EDNAPluginName)
job.setDataInput(xsd)
job.execute()
def unitTestSetGetData(self):
"""
check the status after a job creation
"""
EDVerbose.DEBUG("EDTestCaseEDJob.unitTestSetGetData")
edJob = EDJob(self.strPluginName)
edJob.setDataInput(self.strXmlInput)
EDAssert.equal(self.strXmlInput, edJob.getDataInput().strip(), "Data Input is correctly set")
EDAssert.equal("uninitialized", edJob.getStatus(), "Job %s is still ''uninitialized''" % edJob.getJobId())
def unitTestInitialState(self):
"""
check the status after a job creation
"""
EDVerbose.DEBUG("EDTestCaseEDJob.unitTestInitialState")
edJob = EDJob(self.strPluginName)
strJobId = edJob.getJobId()
EDVerbose.DEBUG("EDJobId is: %s" % strJobId)
EDAssert.equal(2, len(strJobId.split("-")), "JobID is composed of 2 parts")
EDAssert.equal(True, strJobId.split("-")[1].isdigit(), "JobID's second part is an integer")
EDAssert.equal("uninitialized", edJob.getStatus(), "Initial stat is ''uninitialized''")
def unitTestExecuteCallbackSuccess(self):
"""
check the execution of a job (without callback)
"""
EDVerbose.DEBUG("EDTestCaseEDJob.unitTestExecuteCallbackSuccess")
edJob = EDJob(self.strPluginName)
edJob.connectSUCCESS(self.callBack)
edJob.setDataInput(self.strXmlInput)
strJobId = edJob.execute()
strStatus = edJob.getStatus()
EDVerbose.DEBUG("Job %s in State ''%s''" % (strJobId, strStatus))
def process(self):
for fn in self.dataFiles:
EDVerbose.screen("Processing file %s" % fn)
edj = EDJob(self.EDNAPluginName)
edj.dataInput = self.fileName2xml(fn)
edj.connectSUCCESS(self.XMLsuccess)
edj.connectFAILURE(self.XMLerr)
self.queue.put(edj)
if self.process_sem._Semaphore__value > 0 :
t = threading.Thread(target=self.startProcessing)
t.start()
EDVerbose.screen("Back in main")
while self.cpu_sem._Semaphore__value < self.nbcpu:
time.sleep(0.1)
EDJob.synchronizeAll()
EDJob.stats()
def startJob(self, argin):
"""
@param argin: 2-list [ "EDPluginName", "<xml/><XSDataInputPluginName>...."]
@return: jobID which is a sting: Plugin-000001
"""
self.DEBUG("In %s.startJob()" % self.get_name())
name, xsd = argin[:2]
if xsd.strip() == "":
return
edJob = EDJob(name)
if edJob is None:
return "Error in load Plugin"
jobId = edJob.getJobId()
edJob.setDataInput(xsd)
self.jobQueue.put(edJob)
return jobId
def start(self, _strXmlInput):
"""
Launch EDNA with the given XML stream
@param _strXmlInput: XML to be passed to the plugin
@type _strXmlInput: python string representing the XML data structure
"""
jobid = None
if _strXmlInput not in ["", None]:
job = EDJob(self.__strPluginName)
job.setDataInput(_strXmlInput)
job.connectFAILURE(self.failureJobExecution)
job.connectSUCCESS(self.successJobExecution)
job.connectCallBack(self.unregisterJob)
self.semaphoreNbThreadsAcquire()
jobid = job.execute()
self.DEBUG("Running Job id %s" % jobid)
if jobid is None:
self.semaphoreNbThreadsRelease()
return jobid
def startJob(self, argin):
"""
@param argin: 2-list [ "EDPluginName", "<xml/><XSDataInputPluginName>...."]
@return: jobID which is a sting: Plugin-000001
"""
self.DEBUG("In %s.startJob()" % self.get_name())
name, xsd = argin[:2]
if xsd.strip() == "":
return
edJob = EDJob(name)
if edJob is None:
return "Error in load Plugin"
jobId = edJob.getJobId()
edJob.setDataInput(xsd)
self.jobQueue.put(edJob)
if self.processingSem._Semaphore__value > 0:
t = threading.Thread(target=self.startProcessing)
t.start()
return jobId
def startJob(self, xsd):
"""
@param xsd: XML data structure as a string or path to a string
@return: jobID which is a sting: Plugin-000001
"""
self.DEBUG("In %s.startJob()" % self.__class__.__name__)
if xsd.strip() == "":
return
if os.path.isfile(xsd):
xsd = open(xsd, "rb").read()
edJob = EDJob(self.pluginName)
if edJob is None:
return "Error in load Plugin"
jobId = edJob.getJobId()
edJob.setDataInput(xsd)
self.jobQueue.put(edJob)
if self.processingSem._Semaphore__value > 0:
t = threading.Thread(target=self.startProcessing)
t.start()
return jobId
bVerbose = True
elif os.path.isdir(i):
pathWebDoc = i
if WebSite and pathWebDoc:
print "Generating web documentation in ", pathWebDoc
rmdir(pathWebDoc, bypass=True)
for oneproject in findProjects(pyStrEdnaHomePath):
print("Generating HTML pages for " + oneproject)
docPath = os.path.join(pathWebDoc, oneproject)
os.makedirs(docPath, int("755", 8))
listOfPythonFiles = findFile(
os.path.join(pyStrEdnaHomePath, oneproject))
listOfPythonFiles.sort()
if len(listOfPythonFiles) > 0:
epydocJob = EDJob("EDPluginExecEpydocv1_0")
dictJobs[oneproject] = epydocJob
xsd = XSDataInputEpydoc()
xsd.setDocPath(XSDataFile(XSDataString(docPath)))
xsd.setProjectName(XSDataString(oneproject))
xsd.setDocType(XSDataString(docFormat))
if bVerbose:
xsd.setVerbosity(XSDataInteger(1))
else:
xsd.setVerbosity(XSDataInteger(-1))
xsd.setSources([
XSDataFile(XSDataString(oneFile))
for oneFile in listOfPythonFiles
])
epydocJob.setDataInput(xsd)
epydocJob.execute()
for i in fullargs:
reprocess.startJob(i)
print("All %i jobs queued after %.3fs" %
(len(args), time.time() - reprocess.startTime))
reprocess.join()
if yappi: yappi.stop()
print("All %i jobs processed after %.3fs" %
(len(args), time.time() - reprocess.startTime))
print reprocess.statistics()
if yappi:
stat = yappi.get_stats(sort_type=yappi.SORTTYPE_TTOT)
res = {}
for i in stat.func_stats:
if i[0] in res:
res[i[0]][0] += i[1]
res[i[0]][1] += i[2]
else:
res[i[0]] = [i[1], i[2]]
keys = res.keys()
keys.sort(sortn)
with open("yappi.out", "w") as f:
f.write("ncall\t\ttotal\t\tpercall\t\tfunction%s" % (os.linesep))
for i in keys:
f.write("%8s\t%16s\t%16s\t%s%s" %
(res[i][0], res[i][1], res[i][1] / res[i][0], i,
os.linesep))
print("Profiling information written in yappi.out")
edJob = EDJob("EDPluginBioSaxsFlushHPLCv1_0")
edJob.setDataInput(open(fullargs[-1], "r").read())
edJob.execute()
def runEdnaPlugin(execPath, pluginName, isDebug, xml, additionalPaths=None):
'''
execPath - path to run plugin in
pluginName - plugin name
isDebug - True if should run edna in debug mode
xml - xml input to edna
additionalPaths - list of other python path locations
You must set EDNA_HOME to use this method
This method blocks until the EDJob has reached a final status
'''
if (not 'EDNA_HOME' in os.environ):
raise Exception(
"Cannot locate EDNA_HOME. Please set before running Edna plugins.")
if (not 'EDNA_SITE' in os.environ):
raise Exception(" Please set EDNA_SITE before running Edna plugins.")
'''
Add edna to path
'''
ednaKernelPath = os.environ['EDNA_HOME'] + "/kernel/src"
sys.path.insert(0, ednaKernelPath)
'''
If there are any additional paths such as fabio, add these
'''
if (not additionalPaths is None and len(additionalPaths) > 0):
for path in additionalPaths:
sys.path.append(path)
os.chdir(execPath)
from EDVerbose import EDVerbose
if (isDebug):
EDVerbose.setVerboseDebugOn()
else:
EDVerbose.setVerboseOn()
EDVerbose.setVerboseDebugOff()
from EDJob import EDJob
EDVerbose.setLogFileName(execPath + "/" + pluginName + ".log")
edJob = EDJob(pluginName)
edJob.setDataInput(xml)
edJob.execute()
edJob.synchronize(
) # In theory should mean that the following loop is not needed
# Unhelpful way of waiting for EDJob to be finished
# TODO Fix this in EDJob some time
while (True):
status = edJob.getStatus()
if (status is None):
time.sleep(0.2) # 200 ms
continue
if ("failure" == status):
raise Exception("EDJob failed! ")
if ("success" == status):
break
ret = edJob.getDataOutput()
return str(ret)
print("All %i jobs processed after %.3fs" %
(len(args), time.time() - reprocess.startTime))
print reprocess.statistics()
if yappi:
stat = yappi.get_stats(sort_type=yappi.SORTTYPE_TTOT)
res = {}
for i in stat.func_stats:
if i[0] in res:
res[i[0]][0] += i[1]
res[i[0]][1] += i[2]
else:
res[i[0]] = [i[1], i[2]]
keys = res.keys()
keys.sort(sortn)
with open("yappi.out", "w") as f:
f.write("ncall\t\ttotal\t\tpercall\t\tfunction%s" % (os.linesep))
for i in keys:
f.write("%8s\t%16s\t%16s\t%s%s" %
(res[i][0], res[i][1], res[i][1] / res[i][0], i,
os.linesep))
print("Profiling information written in yappi.out")
edJob = EDJob(
options.plugin.replace("EDPluginBioSaxsHPLC",
"EDPluginBioSaxsFlushHPLC"))
edJob.setDataInput(open(fullargs[-1], "r").read())
edJob.execute()
edJob.synchronizeAll()
if options.profile:
for i in EDObject.analyze_profiling():
print(i)
