class EDTestCaseParallelExecute(EDTestCase): """ Unit & execution test for the EDParallelExecute class """ def __init__(self, _strTestName=None): EDTestCase.__init__(self, "EDTestCaseParallelExecute") self.__edpluginPE = EDParallelExecute("EDPluginTestPluginFactory", fakeXML) def unitTestMoveToTempDir(self): """ test the execution of self.movetoTempDir """ initcwd = os.getcwd() self.__edpluginPE.moveToTempDir() EDAssert.equal(os.path.abspath(os.path.dirname(os.getcwd())), os.path.abspath(tempfile.gettempdir()), "Test Move to Temporary Directory...") os.chdir(initcwd) def unitTestDetectNumberOfCPUs(self): """ test the execution of detectNumberOfCPUs """ iNbCPU = EDUtilsParallel.detectNumberOfCPUs() EDVerbose.unitTest("Detection of the number of Cores: %s" % iNbCPU) EDAssert.equal(types.IntType, type(iNbCPU), "Number of CPU is an integer") iNbCPU = EDUtilsParallel.detectNumberOfCPUs(1) #limited to 1 EDAssert.equal(1, iNbCPU, "Limit number of CPU") iNbCPU = EDUtilsParallel.detectNumberOfCPUs(100, True) #forced to 100 EDAssert.equal(100, iNbCPU, "Force number of CPU") def process(self): self.addTestMethod(self.unitTestMoveToTempDir) self.addTestMethod(self.unitTestDetectNumberOfCPUs) EDUtilsParallel.uninitializeNbThread()
mode = "OffLine" newerOnly = True debug = False for i in sys.argv[1:]: if i.lower().find("-online") in [0, 1]: mode = "dirwatch" elif i.lower().find("-all") in [0, 1]: newerOnly = False elif i.lower().find("-debug") in [0, 1]: debug = True if os.path.exists(i): paths.append(os.path.abspath(i)) if len(paths) == 0: if mode == "OffLine": print "This is the Normalize application of EDNA-BioSaxs, \nplease give a path to process offline or the option:\n\ --online to process online incoming data in the given directory.\n\ --all to process all existing files (unless they will be excluded)\n\ --debug to turn on debugging mode in EDNA" sys.exit() else: paths = [os.getcwd()] edna = EDParallelExecute(EDNAPluginName, fileName2xml, _functXMLerr=XMLerr, _bVerbose=True, _bDebug=debug) edna.runEDNA(paths, mode, newerOnly) EDVerbose.screen("Back in main")
mode = "dirwatch" elif i.lower().find("-all") in [0, 1]: newerOnly = False elif i.lower().find("-debug") in [0, 1]: debug = True elif i.lower().find("-ncpu") in [0, 1]: iNbCPU = int(i.split("=", 1)[1]) elif os.path.exists(i): paths.append(os.path.abspath(i)) if len(paths) == 0: if mode == "OffLine": print "This is the SPD-Correct application of %s, \nplease give a path to process offline or the option:\n\ --online to process online incoming data in the given directory.\n\ --all to process all existing files (unless they will be excluded)\n\ --debug to turn on debugging mode in EDNA\n\ --nCPU=xxx to specify the number of CPUs to use. Usually it auto-detects the number of processors." % EDNAPluginName sys.exit() else: paths = [os.getcwd()] listKeys = dictConfig.keys() listKeys.sort() for key in listKeys: val = raw_input("%s [%s] ?" % (key, dictConfig[key])) if len(val) > 0: dictConfig[key] = val edna = EDParallelExecute(EDNAPluginName, fileName2xml, _functXMLerr=XMLerr, _bVerbose=True, _bDebug=debug, _iNbThreads=iNbCPU) edna.runEDNA(paths, mode , newerOnly) edna.cleanUp(["killAllWorkers()"])
elif one_arg.find("-p=") in [0, 1]: strParamFile = oneArg.split("=", 1)[1] if os.path.isfile(strParamFile): xsdParam.setPath(XSDataString(os.path.abspath(strParamFile))) else: EDVerbose.ERROR("The parameter file you provided %s is not valid !!!" % strParamFile) elif one_arg.find("-nocorrectmask") in [0, 1]: xsdCorrectMask = XSDataBoolean(0) elif os.path.exists(oneArg): paths.append(os.path.abspath(oneArg)) if len(paths) == 0: if mode == "OffLine": print "This is the ID11 application of EDNA, \nplease give a path to process offline or the option:\n\ --online to process online incoming data in the given directory.\n\ --all to process all existing files (unless they will be excluded)\n\ --debug to turn on debugging mod in EDNA\n\ -p=ParameterFile.param (MANDATORY)\n\ -o=outputDir" sys.exit() else: paths = [os.getcwd()] if strParamFile is None: EDVerbose.ERROR("No parameter file: Use ''-p=file.param''") sys.exit(1) edna = EDParallelExecute(EDNAPluginName, fileName2xml, _functXMLerr=XMLerr, _bVerbose=True, _bDebug=debug, _iNbThreads=iNbCPU) edna.runEDNA(paths, mode , newerOnly) EDVerbose.screen("Back in main .... close to finish") edPluginSPD.killAllWorkers()
mode = "dirwatch" elif i.lower().find("-all") in [0, 1]: mode = "all" newerOnly = False elif i.lower().find("-debug") in [0, 1]: debug = True elif i.lower().find("-ncpu") in [0, 1]: iNbCPU = int(i.split("=", 1)[1]) elif i.lower().find("-h") in [0, 1]: print "This is the DiffractionCTv1 application of EDNA %s, \nplease give a path to process offline or the option:\n\ --online to process online incoming data in the given directory.\n\ --all to process all existing files (unless they will be excluded)\n\ --debug to turn on debugging mode in EDNA\n\ --nCPU=xxx to specify the number of CPUs to use. Usually EDNA autodetects the number of processors." % EDNAPluginName sys.exit() elif os.path.exists(i): paths.append(os.path.abspath(i)) dct = DiffractionCT() dct.setup(_listInput=paths, _mode=mode) edna = EDParallelExecute(EDNAPluginName, dct.makeXML, _functXMLerr=dct.error, _bVerbose=True, _bDebug=debug, _iNbThreads=iNbCPU) edna.runEDNA(dct.listInput, dct.strMode, dct.bNewerOnly) EDPluginSPDCakev1_5.killAllWorkers()
else: ffx = FullFieldXas() if os.path.isfile(".XSDataInputFullFieldXAS.xml"): ffx.load(".XSDataInputFullFieldXAS.xml") ffx.dontAlign = dontAlign if dontAlign: print ("*" * 80) print ("*" + "Skipping image alignement part".center(78) + "*") print ("*" * 80) ffx.setup(_listInput=paths, _mode=mode) if not isinstance(ffx.reference, int): ffx.uploadReferenceFrame(ffx.reference) ffx.save(".XSDataInputFullFieldXAS.xml") ffx.dump("analysis-%s.json" % time.strftime("%Y%m%d-%Hh%Mm%Ss")) edna = EDParallelExecute( ffx.pluginName, ffx.makeXML, _functXMLerr=ffx.error, _bVerbose=True, _bDebug=debug, _iNbThreads=iNbCPU ) edna.runEDNA(ffx.listInput, ffx.strMode, ffx.bNewerOnly) EDVerbose.WARNING("Back to main !") EDJob.synchronizeAll() ffx.updateAttrRefImage() EDPluginControlAlignStackv1_0.showData() if keepShare: EDShare.close() elif (ffx.getNrErrors() == 0) and (not EDVerbose.isVerboseDebug()): EDVerbose.WARNING("All processing finished successfully: Remove EDShare's big HDF5 file") EDShare.close(remove=True) else: EDShare.close()
def __init__(self, _strTestName=None): EDTestCase.__init__(self, "EDTestCaseParallelExecute") self.__edpluginPE = EDParallelExecute("EDPluginTestPluginFactory", fakeXML)
iNbCPU = None for i in sys.argv[1:]: if i.lower().find("-online") in [0, 1]: mode = "dirwatch" elif i.lower().find("-all") in [0, 1]: mode = "all" newerOnly = False elif i.lower().find("-debug") in [0, 1]: debug = True elif i.lower().find("-ncpu") in [0, 1]: iNbCPU = int(i.split("=", 1)[1]) elif i.lower().find("-h") in [0, 1]: print "This is the DiffractionCTv1 application of EDNA %s, \nplease give a path to process offline or the option:\n\ --online to process online incoming data in the given directory.\n\ --all to process all existing files (unless they will be excluded)\n\ --debug to turn on debugging mode in EDNA\n\ --nCPU=xxx to specify the number of CPUs to use. Usually EDNA autodetects the number of processors." % EDNAPluginName sys.exit() elif os.path.exists(i): paths.append(os.path.abspath(i)) dct = DiffractionCT() dct.setup(_listInput=paths, _mode=mode) edna = EDParallelExecute(EDNAPluginName, dct.makeXML, _functXMLerr=dct.error, _bVerbose=True, _bDebug=debug, _iNbThreads=iNbCPU) edna.runEDNA(dct.listInput, dct.strMode , dct.bNewerOnly) EDPluginSPDCakev1_5.killAllWorkers()
if replay: ffx = FullFieldXas.replay_from(replay) else: ffx = FullFieldXas() if os.path.isfile(".XSDataInputFullFieldXAS.xml"): ffx.load(".XSDataInputFullFieldXAS.xml") ffx.dontAlign = dontAlign if dontAlign: print("*"*80) print("*" + "Skipping image alignement part".center(78) + "*") print("*"*80) ffx.setup(_listInput=paths, _mode=mode) if not isinstance(ffx.reference, int): ffx.uploadReferenceFrame(ffx.reference) ffx.save(".XSDataInputFullFieldXAS.xml") ffx.dump("analysis-%s.json" % time.strftime("%Y%m%d-%Hh%Mm%Ss")) edna = EDParallelExecute(ffx.pluginName, ffx.makeXML, _functXMLerr=ffx.error, _bVerbose=True, _bDebug=debug, _iNbThreads=iNbCPU) edna.runEDNA(ffx.listInput, ffx.strMode , ffx.bNewerOnly) EDVerbose.WARNING("Back to main !") EDJob.synchronizeAll() ffx.updateAttrRefImage() EDPluginControlAlignStackv1_0.showData() if keepShare: EDShare.close() elif (ffx.getNrErrors() == 0) and (not EDVerbose.isVerboseDebug()): EDVerbose.WARNING("All processing finished successfully: Remove EDShare's big HDF5 file") EDShare.close(remove=True) else: EDShare.close()
hdf5 = None for i in sys.argv[1:]: if i.lower().find("-online") in [0, 1]: mode = "dirwatch" elif i.lower().find("-all") in [0, 1]: newerOnly = False elif i.lower().find("-debug") in [0, 1]: debug = True elif i.lower().find("-ncpu") in [0, 1]: iNbCPU = int(i.split("=", 1)[1]) elif i.lower().find("-h") in [0, 1]: hdf5 = i.split("=", 1)[1] elif os.path.exists(i): paths.append(os.path.abspath(i)) print hdf5 mkXml = MakeXML(hdf5) if len(paths) == 0: if mode == "OffLine": print "This is the HDF5 3D image stacker application of EDNA, \nplease give a path to process offline or the option:\n\ --online to process online incoming data in the given directory.\n\ --all to process all existing files (unless they will be excluded)\n\ --debug to turn on debugging mod in EDNA" sys.exit() else: paths = [os.getcwd()] edna = EDParallelExecute(EDNAPluginName, mkXml.fileName2xml, _functXMLerr=mkXml.errBack, _bVerbose=True, _bDebug=debug, _iNbThreads=iNbCPU) edna.runEDNA(paths, mode , newerOnly) print "Back in main" EDPluginHDF5StackImagesv10.closeAll()
mode = "dirwatch" elif i.lower().find("-all") in [0, 1]: newerOnly = False elif i.lower().find("-debug") in [0, 1]: debug = True if os.path.exists(i): paths.append(os.path.abspath(i)) if len(paths) == 0: if mode == "OffLine": print "This is the Azimuthal Integration application of EDNA-BioSaxs, \nplease give a path to process offline or the option:\n\ --online to process online incoming data in the given directory.\n\ --all to process all existing files (unless they will be excluded)\n\ --debug to turn on debugging mode in EDNA" sys.exit() else: paths = [os.getcwd()] cli = BioSaxsReprocess() print("#"*80 + os.linesep + "# Default configuration" + os.linesep + "#" * 80) print cli cli.cliReadParam() print("#"*80 + os.linesep + "# New configuration" + os.linesep + "#" * 80) print cli edna = EDParallelExecute(EDNAPluginName, _functXMLin=cli.toXml, _functXMLerr=cli.functError, _bVerbose=True, _bDebug=debug) edna.runEDNA(paths, mode , newerOnly)
elif i.lower().find("-ncpu") in [0, 1]: iNbCPU = int(i.split("=", 1)[1]) elif os.path.exists(i): paths.append(os.path.abspath(i)) if len(paths) == 0: if mode == "OffLine": print "This is the SPD-Correct application of %s, \nplease give a path to process offline or the option:\n\ --online to process online incoming data in the given directory.\n\ --all to process all existing files (unless they will be excluded)\n\ --debug to turn on debugging mode in EDNA\n\ --nCPU=xxx to specify the number of CPUs to use. Usually it auto-detects the number of processors." % EDNAPluginName sys.exit() else: paths = [os.getcwd()] listKeys = dictConfig.keys() listKeys.sort() for key in listKeys: val = raw_input("%s [%s] ?" % (key, dictConfig[key])) if len(val) > 0: dictConfig[key] = val edna = EDParallelExecute(EDNAPluginName, fileName2xml, _functXMLerr=XMLerr, _bVerbose=True, _bDebug=debug, _iNbThreads=iNbCPU) edna.runEDNA(paths, mode, newerOnly) edna.cleanUp(["killAllWorkers()"])