def main(options, args, batchManager):
batchManager.cfgFileName = args[0]
handle = open(batchManager.cfgFileName, 'r')
cfo = imp.load_source("pycfg", batchManager.cfgFileName, handle)
config = cfo.config
handle.close()
batchManager.components = split( [comp for comp in config.components \
if len(comp.files)>0] )
listOfValues = range(0, len(batchManager.components))
listOfNames = [comp.name for comp in batchManager.components]
batchManager.PrepareJobs( listOfValues, listOfNames )
waitingTime = 0.1
batchManager.SubmitJobs( waitingTime )
def main(options, args, batchManager):
batchManager.cfgFileName = args[0]
handle = open(batchManager.cfgFileName, 'r')
cfo = imp.load_source("pycfg", batchManager.cfgFileName, handle)
config = cfo.config
handle.close()
batchManager.components = split( [comp for comp in config.components \
if len(comp.files)>0] )
listOfValues = range(0, len(batchManager.components))
listOfNames = [comp.name for comp in batchManager.components]
batchManager.PrepareJobs(listOfValues, listOfNames)
waitingTime = 0.1
batchManager.SubmitJobs(waitingTime)
def main(options, args, parser):
if len(args) != 2:
parser.print_help()
print 'ERROR: please provide the processing name and the component list'
sys.exit(1)
outDir = args[0]
if os.path.exists(outDir) and not os.path.isdir(outDir):
parser.print_help()
print 'ERROR: when it exists, first argument must be a directory.'
sys.exit(2)
cfgFileName = args[1]
if not os.path.isfile(cfgFileName):
parser.print_help()
print 'ERROR: second argument must be an existing file (your input cfg).'
sys.exit(3)
file = open(cfgFileName, 'r')
sys.path.append(os.path.dirname(cfgFileName))
cfg = imp.load_source('cfg', cfgFileName, file)
selComps = [comp for comp in cfg.config.components if len(comp.files) > 0]
selComps = split(selComps)
# for comp in selComps:
# print comp
# if len(selComps)>14:
# raise ValueError('too many threads: {tnum}'.format(tnum=len(selComps)))
if not createOutputDir(outDir, selComps, options.force):
print 'exiting'
sys.exit(0)
if len(selComps) > 1:
shutil.copy(cfgFileName, outDir)
pool = Pool(processes=len(selComps))
for comp in selComps:
pool.apply_async(runLoopAsync, [comp, outDir, cfg.config, options],
callback=callBack)
pool.close()
pool.join()
else:
# when running only one loop, do not use multiprocessor module.
# then, the exceptions are visible -> use only one sample for testing
global loop
loop = runLoop(comp, outDir, cfg.config, options)
def main( options, args, parser ):
if len(args) != 2:
parser.print_help()
print 'ERROR: please provide the processing name and the component list'
sys.exit(1)
outDir = args[0]
if os.path.exists(outDir) and not os.path.isdir( outDir ):
parser.print_help()
print 'ERROR: when it exists, first argument must be a directory.'
sys.exit(2)
cfgFileName = args[1]
if not os.path.isfile( cfgFileName ):
parser.print_help()
print 'ERROR: second argument must be an existing file (your input cfg).'
sys.exit(3)
if options.verbose:
import logging
logging.basicConfig(level=logging.INFO)
# Propagate global options to _heppyGlobalOptions within this module
# I have to import it explicitly, 'global' does not work since the
# module is not set when executing the main
from heppy.framework.heppy_loop import _heppyGlobalOptions
for opt in options.extraOptions:
if "=" in opt:
(key,val) = opt.split("=",1)
_heppyGlobalOptions[key] = val
else:
_heppyGlobalOptions[opt] = True
file = open( cfgFileName, 'r' )
sys.path.append( os.path.dirname(cfgFileName) )
cfg = imp.load_source( 'heppy.__cfg_to_run__',
cfgFileName, file)
selComps = [comp for comp in cfg.config.components if len(comp.files)>0]
selComps = split(selComps)
# for comp in selComps:
# print comp
if len(selComps)>options.ntasks:
print "WARNING: too many threads {tnum}, will just use a maximum of {jnum}.".format(tnum=len(selComps),jnum=options.ntasks)
if not createOutputDir(outDir, selComps, options.force):
print 'exiting'
sys.exit(0)
if len(selComps)>1:
shutil.copy( cfgFileName, outDir )
pool = multiprocessing.Pool(processes=min(len(selComps),options.ntasks))
## workaround for a scoping problem in ipython+multiprocessing
import heppy.framework.heppy_loop as ML
for comp in selComps:
pool.apply_async( ML.runLoopAsync, [comp, outDir, 'heppy.__cfg_to_run__', options],
callback=ML.callBack)
pool.close()
pool.join()
else:
# when running only one loop, do not use multiprocessor module.
# then, the exceptions are visible -> use only one sample for testing
global loop
loop = runLoop( comp, outDir, cfg.config, options )
return loop
