class EDTestCaseParallelExecute(EDTestCase):
"""
Unit & execution test for the EDParallelExecute class
"""
def __init__(self, _strTestName=None):
EDTestCase.__init__(self, "EDTestCaseParallelExecute")
self.__edpluginPE = EDParallelExecute("EDPluginTestPluginFactory",
fakeXML)
def unitTestMoveToTempDir(self):
"""
test the execution of self.movetoTempDir
"""
initcwd = os.getcwd()
self.__edpluginPE.moveToTempDir()
EDAssert.equal(os.path.abspath(os.path.dirname(os.getcwd())),
os.path.abspath(tempfile.gettempdir()),
"Test Move to Temporary Directory...")
os.chdir(initcwd)
def unitTestDetectNumberOfCPUs(self):
"""
test the execution of detectNumberOfCPUs
"""
iNbCPU = EDUtilsParallel.detectNumberOfCPUs()
EDVerbose.unitTest("Detection of the number of Cores: %s" % iNbCPU)
EDAssert.equal(types.IntType, type(iNbCPU),
"Number of CPU is an integer")
iNbCPU = EDUtilsParallel.detectNumberOfCPUs(1) #limited to 1
EDAssert.equal(1, iNbCPU, "Limit number of CPU")
iNbCPU = EDUtilsParallel.detectNumberOfCPUs(100, True) #forced to 100
EDAssert.equal(100, iNbCPU, "Force number of CPU")
def process(self):
self.addTestMethod(self.unitTestMoveToTempDir)
self.addTestMethod(self.unitTestDetectNumberOfCPUs)
EDUtilsParallel.uninitializeNbThread()
class EDTestCaseParallelExecute(EDTestCase):
"""
Unit & execution test for the EDParallelExecute class
"""
def __init__(self, _strTestName=None):
EDTestCase.__init__(self, "EDTestCaseParallelExecute")
self.__edpluginPE = EDParallelExecute("EDPluginTestPluginFactory", fakeXML)
def unitTestMoveToTempDir(self):
"""
test the execution of self.movetoTempDir
"""
initcwd = os.getcwd()
self.__edpluginPE.moveToTempDir()
EDAssert.equal(os.path.abspath(os.path.dirname(os.getcwd())), os.path.abspath(tempfile.gettempdir()), "Test Move to Temporary Directory...")
os.chdir(initcwd)
def unitTestDetectNumberOfCPUs(self):
"""
test the execution of detectNumberOfCPUs
"""
iNbCPU = EDUtilsParallel.detectNumberOfCPUs()
EDVerbose.unitTest("Detection of the number of Cores: %s" % iNbCPU)
EDAssert.equal(types.IntType, type(iNbCPU), "Number of CPU is an integer")
iNbCPU = EDUtilsParallel.detectNumberOfCPUs(1) #limited to 1
EDAssert.equal(1, iNbCPU, "Limit number of CPU")
iNbCPU = EDUtilsParallel.detectNumberOfCPUs(100, True) #forced to 100
EDAssert.equal(100, iNbCPU, "Force number of CPU")
def process(self):
self.addTestMethod(self.unitTestMoveToTempDir)
self.addTestMethod(self.unitTestDetectNumberOfCPUs)
EDUtilsParallel.uninitializeNbThread()
mode = "OffLine"
newerOnly = True
debug = False
for i in sys.argv[1:]:
if i.lower().find("-online") in [0, 1]:
mode = "dirwatch"
elif i.lower().find("-all") in [0, 1]:
newerOnly = False
elif i.lower().find("-debug") in [0, 1]:
debug = True
if os.path.exists(i):
paths.append(os.path.abspath(i))
if len(paths) == 0:
if mode == "OffLine":
print "This is the Normalize application of EDNA-BioSaxs, \nplease give a path to process offline or the option:\n\
--online to process online incoming data in the given directory.\n\
--all to process all existing files (unless they will be excluded)\n\
--debug to turn on debugging mode in EDNA"
sys.exit()
else:
paths = [os.getcwd()]
edna = EDParallelExecute(EDNAPluginName,
fileName2xml,
_functXMLerr=XMLerr,
_bVerbose=True,
_bDebug=debug)
edna.runEDNA(paths, mode, newerOnly)
EDVerbose.screen("Back in main")
mode = "dirwatch"
elif i.lower().find("-all") in [0, 1]:
newerOnly = False
elif i.lower().find("-debug") in [0, 1]:
debug = True
elif i.lower().find("-ncpu") in [0, 1]:
iNbCPU = int(i.split("=", 1)[1])
elif os.path.exists(i):
paths.append(os.path.abspath(i))
if len(paths) == 0:
if mode == "OffLine":
print "This is the SPD-Correct application of %s, \nplease give a path to process offline or the option:\n\
--online to process online incoming data in the given directory.\n\
--all to process all existing files (unless they will be excluded)\n\
--debug to turn on debugging mode in EDNA\n\
--nCPU=xxx to specify the number of CPUs to use. Usually it auto-detects the number of processors." % EDNAPluginName
sys.exit()
else:
paths = [os.getcwd()]
listKeys = dictConfig.keys()
listKeys.sort()
for key in listKeys:
val = raw_input("%s [%s] ?" % (key, dictConfig[key]))
if len(val) > 0:
dictConfig[key] = val
edna = EDParallelExecute(EDNAPluginName, fileName2xml, _functXMLerr=XMLerr, _bVerbose=True, _bDebug=debug, _iNbThreads=iNbCPU)
edna.runEDNA(paths, mode , newerOnly)
edna.cleanUp(["killAllWorkers()"])
elif one_arg.find("-p=") in [0, 1]:
strParamFile = oneArg.split("=", 1)[1]
if os.path.isfile(strParamFile):
xsdParam.setPath(XSDataString(os.path.abspath(strParamFile)))
else:
EDVerbose.ERROR("The parameter file you provided %s is not valid !!!" % strParamFile)
elif one_arg.find("-nocorrectmask") in [0, 1]:
xsdCorrectMask = XSDataBoolean(0)
elif os.path.exists(oneArg):
paths.append(os.path.abspath(oneArg))
if len(paths) == 0:
if mode == "OffLine":
print "This is the ID11 application of EDNA, \nplease give a path to process offline or the option:\n\
--online to process online incoming data in the given directory.\n\
--all to process all existing files (unless they will be excluded)\n\
--debug to turn on debugging mod in EDNA\n\
-p=ParameterFile.param (MANDATORY)\n\
-o=outputDir"
sys.exit()
else:
paths = [os.getcwd()]
if strParamFile is None:
EDVerbose.ERROR("No parameter file: Use ''-p=file.param''")
sys.exit(1)
edna = EDParallelExecute(EDNAPluginName, fileName2xml, _functXMLerr=XMLerr, _bVerbose=True, _bDebug=debug, _iNbThreads=iNbCPU)
edna.runEDNA(paths, mode , newerOnly)
EDVerbose.screen("Back in main .... close to finish")
edPluginSPD.killAllWorkers()
mode = "dirwatch"
elif i.lower().find("-all") in [0, 1]:
mode = "all"
newerOnly = False
elif i.lower().find("-debug") in [0, 1]:
debug = True
elif i.lower().find("-ncpu") in [0, 1]:
iNbCPU = int(i.split("=", 1)[1])
elif i.lower().find("-h") in [0, 1]:
print "This is the DiffractionCTv1 application of EDNA %s, \nplease give a path to process offline or the option:\n\
--online to process online incoming data in the given directory.\n\
--all to process all existing files (unless they will be excluded)\n\
--debug to turn on debugging mode in EDNA\n\
--nCPU=xxx to specify the number of CPUs to use. Usually EDNA autodetects the number of processors." % EDNAPluginName
sys.exit()
elif os.path.exists(i):
paths.append(os.path.abspath(i))
dct = DiffractionCT()
dct.setup(_listInput=paths, _mode=mode)
edna = EDParallelExecute(EDNAPluginName,
dct.makeXML,
_functXMLerr=dct.error,
_bVerbose=True,
_bDebug=debug,
_iNbThreads=iNbCPU)
edna.runEDNA(dct.listInput, dct.strMode, dct.bNewerOnly)
EDPluginSPDCakev1_5.killAllWorkers()
else:
ffx = FullFieldXas()
if os.path.isfile(".XSDataInputFullFieldXAS.xml"):
ffx.load(".XSDataInputFullFieldXAS.xml")
ffx.dontAlign = dontAlign
if dontAlign:
print ("*" * 80)
print ("*" + "Skipping image alignement part".center(78) + "*")
print ("*" * 80)
ffx.setup(_listInput=paths, _mode=mode)
if not isinstance(ffx.reference, int):
ffx.uploadReferenceFrame(ffx.reference)
ffx.save(".XSDataInputFullFieldXAS.xml")
ffx.dump("analysis-%s.json" % time.strftime("%Y%m%d-%Hh%Mm%Ss"))
edna = EDParallelExecute(
ffx.pluginName, ffx.makeXML, _functXMLerr=ffx.error, _bVerbose=True, _bDebug=debug, _iNbThreads=iNbCPU
)
edna.runEDNA(ffx.listInput, ffx.strMode, ffx.bNewerOnly)
EDVerbose.WARNING("Back to main !")
EDJob.synchronizeAll()
ffx.updateAttrRefImage()
EDPluginControlAlignStackv1_0.showData()
if keepShare:
EDShare.close()
elif (ffx.getNrErrors() == 0) and (not EDVerbose.isVerboseDebug()):
EDVerbose.WARNING("All processing finished successfully: Remove EDShare's big HDF5 file")
EDShare.close(remove=True)
else:
EDShare.close()
def __init__(self, _strTestName=None):
EDTestCase.__init__(self, "EDTestCaseParallelExecute")
self.__edpluginPE = EDParallelExecute("EDPluginTestPluginFactory",
fakeXML)
def __init__(self, _strTestName=None):
EDTestCase.__init__(self, "EDTestCaseParallelExecute")
self.__edpluginPE = EDParallelExecute("EDPluginTestPluginFactory", fakeXML)
iNbCPU = None
for i in sys.argv[1:]:
if i.lower().find("-online") in [0, 1]:
mode = "dirwatch"
elif i.lower().find("-all") in [0, 1]:
mode = "all"
newerOnly = False
elif i.lower().find("-debug") in [0, 1]:
debug = True
elif i.lower().find("-ncpu") in [0, 1]:
iNbCPU = int(i.split("=", 1)[1])
elif i.lower().find("-h") in [0, 1]:
print "This is the DiffractionCTv1 application of EDNA %s, \nplease give a path to process offline or the option:\n\
--online to process online incoming data in the given directory.\n\
--all to process all existing files (unless they will be excluded)\n\
--debug to turn on debugging mode in EDNA\n\
--nCPU=xxx to specify the number of CPUs to use. Usually EDNA autodetects the number of processors." % EDNAPluginName
sys.exit()
elif os.path.exists(i):
paths.append(os.path.abspath(i))
dct = DiffractionCT()
dct.setup(_listInput=paths, _mode=mode)
edna = EDParallelExecute(EDNAPluginName, dct.makeXML, _functXMLerr=dct.error, _bVerbose=True, _bDebug=debug, _iNbThreads=iNbCPU)
edna.runEDNA(dct.listInput, dct.strMode , dct.bNewerOnly)
EDPluginSPDCakev1_5.killAllWorkers()
if replay:
ffx = FullFieldXas.replay_from(replay)
else:
ffx = FullFieldXas()
if os.path.isfile(".XSDataInputFullFieldXAS.xml"):
ffx.load(".XSDataInputFullFieldXAS.xml")
ffx.dontAlign = dontAlign
if dontAlign:
print("*"*80)
print("*" + "Skipping image alignement part".center(78) + "*")
print("*"*80)
ffx.setup(_listInput=paths, _mode=mode)
if not isinstance(ffx.reference, int):
ffx.uploadReferenceFrame(ffx.reference)
ffx.save(".XSDataInputFullFieldXAS.xml")
ffx.dump("analysis-%s.json" % time.strftime("%Y%m%d-%Hh%Mm%Ss"))
edna = EDParallelExecute(ffx.pluginName, ffx.makeXML, _functXMLerr=ffx.error, _bVerbose=True, _bDebug=debug, _iNbThreads=iNbCPU)
edna.runEDNA(ffx.listInput, ffx.strMode , ffx.bNewerOnly)
EDVerbose.WARNING("Back to main !")
EDJob.synchronizeAll()
ffx.updateAttrRefImage()
EDPluginControlAlignStackv1_0.showData()
if keepShare:
EDShare.close()
elif (ffx.getNrErrors() == 0) and (not EDVerbose.isVerboseDebug()):
EDVerbose.WARNING("All processing finished successfully: Remove EDShare's big HDF5 file")
EDShare.close(remove=True)
else:
EDShare.close()
hdf5 = None
for i in sys.argv[1:]:
if i.lower().find("-online") in [0, 1]:
mode = "dirwatch"
elif i.lower().find("-all") in [0, 1]:
newerOnly = False
elif i.lower().find("-debug") in [0, 1]:
debug = True
elif i.lower().find("-ncpu") in [0, 1]:
iNbCPU = int(i.split("=", 1)[1])
elif i.lower().find("-h") in [0, 1]:
hdf5 = i.split("=", 1)[1]
elif os.path.exists(i):
paths.append(os.path.abspath(i))
print hdf5
mkXml = MakeXML(hdf5)
if len(paths) == 0:
if mode == "OffLine":
print "This is the HDF5 3D image stacker application of EDNA, \nplease give a path to process offline or the option:\n\
--online to process online incoming data in the given directory.\n\
--all to process all existing files (unless they will be excluded)\n\
--debug to turn on debugging mod in EDNA"
sys.exit()
else:
paths = [os.getcwd()]
edna = EDParallelExecute(EDNAPluginName, mkXml.fileName2xml, _functXMLerr=mkXml.errBack, _bVerbose=True, _bDebug=debug, _iNbThreads=iNbCPU)
edna.runEDNA(paths, mode , newerOnly)
print "Back in main"
EDPluginHDF5StackImagesv10.closeAll()
mode = "dirwatch"
elif i.lower().find("-all") in [0, 1]:
newerOnly = False
elif i.lower().find("-debug") in [0, 1]:
debug = True
if os.path.exists(i):
paths.append(os.path.abspath(i))
if len(paths) == 0:
if mode == "OffLine":
print "This is the Azimuthal Integration application of EDNA-BioSaxs, \nplease give a path to process offline or the option:\n\
--online to process online incoming data in the given directory.\n\
--all to process all existing files (unless they will be excluded)\n\
--debug to turn on debugging mode in EDNA"
sys.exit()
else:
paths = [os.getcwd()]
cli = BioSaxsReprocess()
print("#"*80 + os.linesep + "# Default configuration" + os.linesep + "#" * 80)
print cli
cli.cliReadParam()
print("#"*80 + os.linesep + "# New configuration" + os.linesep + "#" * 80)
print cli
edna = EDParallelExecute(EDNAPluginName, _functXMLin=cli.toXml, _functXMLerr=cli.functError, _bVerbose=True, _bDebug=debug)
edna.runEDNA(paths, mode , newerOnly)
elif i.lower().find("-ncpu") in [0, 1]:
iNbCPU = int(i.split("=", 1)[1])
elif os.path.exists(i):
paths.append(os.path.abspath(i))
if len(paths) == 0:
if mode == "OffLine":
print "This is the SPD-Correct application of %s, \nplease give a path to process offline or the option:\n\
--online to process online incoming data in the given directory.\n\
--all to process all existing files (unless they will be excluded)\n\
--debug to turn on debugging mode in EDNA\n\
--nCPU=xxx to specify the number of CPUs to use. Usually it auto-detects the number of processors." % EDNAPluginName
sys.exit()
else:
paths = [os.getcwd()]
listKeys = dictConfig.keys()
listKeys.sort()
for key in listKeys:
val = raw_input("%s [%s] ?" % (key, dictConfig[key]))
if len(val) > 0:
dictConfig[key] = val
edna = EDParallelExecute(EDNAPluginName,
fileName2xml,
_functXMLerr=XMLerr,
_bVerbose=True,
_bDebug=debug,
_iNbThreads=iNbCPU)
edna.runEDNA(paths, mode, newerOnly)
edna.cleanUp(["killAllWorkers()"])
