def unitTestExecute(self): """ check the execution of a job (without callback) """ EDVerbose.DEBUG("EDTestCaseEDJob.unitTestExecute") edJob = EDJob(self.strPluginName) strJobId = edJob.getJobId() edJob.setDataInput(self.strXmlInput) ref = edJob.execute() EDAssert.equal(strJobId, ref, "JobId has not changed") strStatus = edJob.getStatus() EDVerbose.WARNING("Job %s in State %s" % (strJobId, strStatus)) while strStatus in ["running", "uninitialized"]: EDVerbose.WARNING("Job %s in state %s" % (strJobId, strStatus)) time.sleep(0.01) strStatus = edJob.getStatus() xsdOut = edJob.getDataOutput() while xsdOut is None: EDVerbose.WARNING("No Output data, still waiting for output data to arrive, %s" % edJob.getStatus()) time.sleep(0.01) xsdOut = edJob.getDataOutput() strOutput = xsdOut.strip() strStatus = edJob.getStatus() while strStatus == "running": EDVerbose.WARNING("Job %s is still in state %s" % (strJobId, strStatus)) time.sleep(0.01) strStatus = edJob.getStatus() EDAssert.equal(strOutput, self.strXmlInput, "Output is OK") EDAssert.equal("success", edJob.getStatus(), "Job %s is finished with ''success''" % edJob.getJobId())
def process(_listInputFile, _output, dummy=0, autoscale=False, center=None, width=None, blending=None, mask=None): """ call EDNA with this options: @param _listInputFile: list of input files as strings @param _output: output file name @param dummy: value for dummy pixels @param autoscale: shall image intensity be scaled (boolean) @param center: 2-list of int representing the center of the ROI @param width: 2-list of int representing the width of the ROI @param blending: blending method: max, mean or min @param mask: name of the file containing the mask """ xsd = XSDataInputStitchImage() xsd.dummyValue = XSDataDouble(dummy) xsd.autoscale = XSDataBoolean(autoscale) if blending: xsd.blending = XSDataString(blending) if mask: xsd.mask = XSDataImage(XSDataString(mask)) xsd.outputImage = XSDataImage(XSDataString(_output)) xsd.inputImages = [XSDataImage(XSDataString(i)) for i in _listInputFile] if isinstance(width, list): xsd.widthROI = [XSDataInteger(i) for i in width] if isinstance(center, list): xsd.centerROI = [XSDataInteger(i) for i in center] job = EDJob(EDNAPluginName) job.setDataInput(xsd) job.execute()
def unitTestSetGetData(self): """ check the status after a job creation """ EDVerbose.DEBUG("EDTestCaseEDJob.unitTestSetGetData") edJob = EDJob(self.strPluginName) edJob.setDataInput(self.strXmlInput) EDAssert.equal(self.strXmlInput, edJob.getDataInput().strip(), "Data Input is correctly set") EDAssert.equal("uninitialized", edJob.getStatus(), "Job %s is still ''uninitialized''" % edJob.getJobId())
def unitTestInitialState(self): """ check the status after a job creation """ EDVerbose.DEBUG("EDTestCaseEDJob.unitTestInitialState") edJob = EDJob(self.strPluginName) strJobId = edJob.getJobId() EDVerbose.DEBUG("EDJobId is: %s" % strJobId) EDAssert.equal(2, len(strJobId.split("-")), "JobID is composed of 2 parts") EDAssert.equal(True, strJobId.split("-")[1].isdigit(), "JobID's second part is an integer") EDAssert.equal("uninitialized", edJob.getStatus(), "Initial stat is ''uninitialized''")
def unitTestExecuteCallbackSuccess(self): """ check the execution of a job (without callback) """ EDVerbose.DEBUG("EDTestCaseEDJob.unitTestExecuteCallbackSuccess") edJob = EDJob(self.strPluginName) edJob.connectSUCCESS(self.callBack) edJob.setDataInput(self.strXmlInput) strJobId = edJob.execute() strStatus = edJob.getStatus() EDVerbose.DEBUG("Job %s in State ''%s''" % (strJobId, strStatus))
def process(self): for fn in self.dataFiles: EDVerbose.screen("Processing file %s" % fn) edj = EDJob(self.EDNAPluginName) edj.dataInput = self.fileName2xml(fn) edj.connectSUCCESS(self.XMLsuccess) edj.connectFAILURE(self.XMLerr) self.queue.put(edj) if self.process_sem._Semaphore__value > 0 : t = threading.Thread(target=self.startProcessing) t.start() EDVerbose.screen("Back in main") while self.cpu_sem._Semaphore__value < self.nbcpu: time.sleep(0.1) EDJob.synchronizeAll() EDJob.stats()
def startJob(self, argin): """ @param argin: 2-list [ "EDPluginName", "<xml/><XSDataInputPluginName>...."] @return: jobID which is a sting: Plugin-000001 """ self.DEBUG("In %s.startJob()" % self.get_name()) name, xsd = argin[:2] if xsd.strip() == "": return edJob = EDJob(name) if edJob is None: return "Error in load Plugin" jobId = edJob.getJobId() edJob.setDataInput(xsd) self.jobQueue.put(edJob) return jobId
def start(self, _strXmlInput): """ Launch EDNA with the given XML stream @param _strXmlInput: XML to be passed to the plugin @type _strXmlInput: python string representing the XML data structure """ jobid = None if _strXmlInput not in ["", None]: job = EDJob(self.__strPluginName) job.setDataInput(_strXmlInput) job.connectFAILURE(self.failureJobExecution) job.connectSUCCESS(self.successJobExecution) job.connectCallBack(self.unregisterJob) self.semaphoreNbThreadsAcquire() jobid = job.execute() self.DEBUG("Running Job id %s" % jobid) if jobid is None: self.semaphoreNbThreadsRelease() return jobid
def startJob(self, argin): """ @param argin: 2-list [ "EDPluginName", "<xml/><XSDataInputPluginName>...."] @return: jobID which is a sting: Plugin-000001 """ self.DEBUG("In %s.startJob()" % self.get_name()) name, xsd = argin[:2] if xsd.strip() == "": return edJob = EDJob(name) if edJob is None: return "Error in load Plugin" jobId = edJob.getJobId() edJob.setDataInput(xsd) self.jobQueue.put(edJob) if self.processingSem._Semaphore__value > 0: t = threading.Thread(target=self.startProcessing) t.start() return jobId
def startJob(self, xsd): """ @param xsd: XML data structure as a string or path to a string @return: jobID which is a sting: Plugin-000001 """ self.DEBUG("In %s.startJob()" % self.__class__.__name__) if xsd.strip() == "": return if os.path.isfile(xsd): xsd = open(xsd, "rb").read() edJob = EDJob(self.pluginName) if edJob is None: return "Error in load Plugin" jobId = edJob.getJobId() edJob.setDataInput(xsd) self.jobQueue.put(edJob) if self.processingSem._Semaphore__value > 0: t = threading.Thread(target=self.startProcessing) t.start() return jobId
bVerbose = True elif os.path.isdir(i): pathWebDoc = i if WebSite and pathWebDoc: print "Generating web documentation in ", pathWebDoc rmdir(pathWebDoc, bypass=True) for oneproject in findProjects(pyStrEdnaHomePath): print("Generating HTML pages for " + oneproject) docPath = os.path.join(pathWebDoc, oneproject) os.makedirs(docPath, int("755", 8)) listOfPythonFiles = findFile( os.path.join(pyStrEdnaHomePath, oneproject)) listOfPythonFiles.sort() if len(listOfPythonFiles) > 0: epydocJob = EDJob("EDPluginExecEpydocv1_0") dictJobs[oneproject] = epydocJob xsd = XSDataInputEpydoc() xsd.setDocPath(XSDataFile(XSDataString(docPath))) xsd.setProjectName(XSDataString(oneproject)) xsd.setDocType(XSDataString(docFormat)) if bVerbose: xsd.setVerbosity(XSDataInteger(1)) else: xsd.setVerbosity(XSDataInteger(-1)) xsd.setSources([ XSDataFile(XSDataString(oneFile)) for oneFile in listOfPythonFiles ]) epydocJob.setDataInput(xsd) epydocJob.execute()
for i in fullargs: reprocess.startJob(i) print("All %i jobs queued after %.3fs" % (len(args), time.time() - reprocess.startTime)) reprocess.join() if yappi: yappi.stop() print("All %i jobs processed after %.3fs" % (len(args), time.time() - reprocess.startTime)) print reprocess.statistics() if yappi: stat = yappi.get_stats(sort_type=yappi.SORTTYPE_TTOT) res = {} for i in stat.func_stats: if i[0] in res: res[i[0]][0] += i[1] res[i[0]][1] += i[2] else: res[i[0]] = [i[1], i[2]] keys = res.keys() keys.sort(sortn) with open("yappi.out", "w") as f: f.write("ncall\t\ttotal\t\tpercall\t\tfunction%s" % (os.linesep)) for i in keys: f.write("%8s\t%16s\t%16s\t%s%s" % (res[i][0], res[i][1], res[i][1] / res[i][0], i, os.linesep)) print("Profiling information written in yappi.out") edJob = EDJob("EDPluginBioSaxsFlushHPLCv1_0") edJob.setDataInput(open(fullargs[-1], "r").read()) edJob.execute()
def runEdnaPlugin(execPath, pluginName, isDebug, xml, additionalPaths=None): ''' execPath - path to run plugin in pluginName - plugin name isDebug - True if should run edna in debug mode xml - xml input to edna additionalPaths - list of other python path locations You must set EDNA_HOME to use this method This method blocks until the EDJob has reached a final status ''' if (not 'EDNA_HOME' in os.environ): raise Exception( "Cannot locate EDNA_HOME. Please set before running Edna plugins.") if (not 'EDNA_SITE' in os.environ): raise Exception(" Please set EDNA_SITE before running Edna plugins.") ''' Add edna to path ''' ednaKernelPath = os.environ['EDNA_HOME'] + "/kernel/src" sys.path.insert(0, ednaKernelPath) ''' If there are any additional paths such as fabio, add these ''' if (not additionalPaths is None and len(additionalPaths) > 0): for path in additionalPaths: sys.path.append(path) os.chdir(execPath) from EDVerbose import EDVerbose if (isDebug): EDVerbose.setVerboseDebugOn() else: EDVerbose.setVerboseOn() EDVerbose.setVerboseDebugOff() from EDJob import EDJob EDVerbose.setLogFileName(execPath + "/" + pluginName + ".log") edJob = EDJob(pluginName) edJob.setDataInput(xml) edJob.execute() edJob.synchronize( ) # In theory should mean that the following loop is not needed # Unhelpful way of waiting for EDJob to be finished # TODO Fix this in EDJob some time while (True): status = edJob.getStatus() if (status is None): time.sleep(0.2) # 200 ms continue if ("failure" == status): raise Exception("EDJob failed! ") if ("success" == status): break ret = edJob.getDataOutput() return str(ret)
print("All %i jobs processed after %.3fs" % (len(args), time.time() - reprocess.startTime)) print reprocess.statistics() if yappi: stat = yappi.get_stats(sort_type=yappi.SORTTYPE_TTOT) res = {} for i in stat.func_stats: if i[0] in res: res[i[0]][0] += i[1] res[i[0]][1] += i[2] else: res[i[0]] = [i[1], i[2]] keys = res.keys() keys.sort(sortn) with open("yappi.out", "w") as f: f.write("ncall\t\ttotal\t\tpercall\t\tfunction%s" % (os.linesep)) for i in keys: f.write("%8s\t%16s\t%16s\t%s%s" % (res[i][0], res[i][1], res[i][1] / res[i][0], i, os.linesep)) print("Profiling information written in yappi.out") edJob = EDJob( options.plugin.replace("EDPluginBioSaxsHPLC", "EDPluginBioSaxsFlushHPLC")) edJob.setDataInput(open(fullargs[-1], "r").read()) edJob.execute() edJob.synchronizeAll() if options.profile: for i in EDObject.analyze_profiling(): print(i)