def new_tasks(self, extra):
inputs = self._search_for_input_files(self.params.args)
# Copy base dir
localBaseDir = os.path.join(os.getcwd(), 'localBaseDir')
# gc3libs.utils.copytree(self.params.initial, '/mnt/shareOffice/ForwardPremium/Results/sensitivity/wGridSize/dfs')
gc3libs.utils.copytree(self.params.initial, localBaseDir)
for path in inputs:
para_loop = path
path_to_base_dir = os.path.dirname(para_loop)
# self.log.debug("Processing loop file '%s' ...", para_loop)
for jobname, substs in self.process_para_file(para_loop):
## self.log.debug("Job '%s' defined by substitutions: %s.",
## jobname, substs)
executable = os.path.basename(self.params.executable)
inputs = {self.params.executable: executable}
# make a "stage" directory where input files are collected
path_to_stage_dir = self.make_directory_path(
self.params.output, jobname)
input_dir = path_to_stage_dir #os.path.join(path_to_stage_dir, 'input')
gc3libs.utils.mkdir(input_dir)
prefix_len = len(input_dir) + 1
# 2. apply substitutions to parameter files
for (path, changes) in substs.iteritems():
for (var, val, index, regex) in changes:
update_parameter_in_file(
os.path.join(localBaseDir, path), var, index, val,
regex)
fillInputDir(localBaseDir, input_dir)
# 3. build input file list
for dirpath, dirnames, filenames in os.walk(input_dir):
for filename in filenames:
# cut the leading part, which is == to path_to_stage_dir
relpath = dirpath[prefix_len:]
# ignore output directory contents in resubmission
if relpath.startswith('output'):
continue
remote_path = os.path.join(relpath, filename)
inputs[os.path.join(dirpath, filename)] = remote_path
# all contents of the `output` directory are to be fetched
outputs = {'output/': ''}
kwargs = extra.copy()
kwargs['stdout'] = 'forwardPremiumOut.log'
kwargs['join'] = True
kwargs['output_dir'] = os.path.join(path_to_stage_dir,
'output')
kwargs['requested_architecture'] = self.params.architecture
print 'kwargs = %s' % kwargs
print 'inputs = %s' % inputs
print 'outputs = %s' % outputs
# hand over job to create
yield (jobname, gIdRisk.idRiskApplication,
['./' + executable, [], inputs, outputs], kwargs)
def new_tasks(self, extra):
inputs = self._search_for_input_files(self.params.args)
# Copy base dir
localBaseDir = os.path.join(os.getcwd(), 'localBaseDir')
# gc3libs.utils.copytree(self.params.initial, '/mnt/shareOffice/ForwardPremium/Results/sensitivity/wGridSize/dfs')
gc3libs.utils.copytree(self.params.initial, localBaseDir)
for path in inputs:
para_loop = path
path_to_base_dir = os.path.dirname(para_loop)
# self.log.debug("Processing loop file '%s' ...", para_loop)
for jobname, substs in self.process_para_file(para_loop):
## self.log.debug("Job '%s' defined by substitutions: %s.",
## jobname, substs)
executable = os.path.basename(self.params.executable)
inputs = { self.params.executable:executable }
# make a "stage" directory where input files are collected
path_to_stage_dir = self.make_directory_path(
self.params.output, jobname)
input_dir = path_to_stage_dir #os.path.join(path_to_stage_dir, 'input')
gc3libs.utils.mkdir(input_dir)
prefix_len = len(input_dir) + 1
# 2. apply substitutions to parameter files
for (path, changes) in substs.iteritems():
for (var, val, index, regex) in changes:
update_parameter_in_file(os.path.join(localBaseDir, path),
var, index, val, regex)
fillInputDir(localBaseDir, input_dir)
# 3. build input file list
for dirpath,dirnames,filenames in os.walk(input_dir):
for filename in filenames:
# cut the leading part, which is == to path_to_stage_dir
relpath = dirpath[prefix_len:]
# ignore output directory contents in resubmission
if relpath.startswith('output'):
continue
remote_path = os.path.join(relpath, filename)
inputs[os.path.join(dirpath, filename)] = remote_path
# all contents of the `output` directory are to be fetched
outputs = { 'output/':'' }
kwargs = extra.copy()
kwargs['stdout'] = 'forwardPremiumOut.log'
kwargs['join'] = True
kwargs['output_dir'] = os.path.join(path_to_stage_dir, 'output')
kwargs['requested_architecture'] = self.params.architecture
print 'kwargs = %s' % kwargs
print 'inputs = %s' % inputs
print 'outputs = %s' % outputs
# hand over job to create
yield (jobname, gIdRisk.idRiskApplication,
['./' + executable, [], inputs, outputs], kwargs)
def generateTaskList(self, xVars, paraCombos, substs, sessionParas):
# setup AppPot parameters
use_apppot = False
apppot_img = None
apppot_changes = None
apppot_file = sessionParas['AppPotFile']
if apppot_file:
# print 'apppot_file = %s' % apppot_file
# os._exit(1)
use_apppot = True
if apppot_file.endswith('.changes.tar.gz'):
apppot_changes = apppot_file
else:
apppot_img = apppot_file
pathToExecutable = sessionParas['pathToExecutable']
localBaseDir = sessionParas['localBaseDir']
architecture = sessionParas['architecture']
jobname = sessionParas['jobname']
rte = sessionParas['rte']
# Fill the task list
tasks = []
for paraCombo in paraCombos:
logger.debug('paraCombo = %s' % paraCombo)
executable = os.path.basename(pathToExecutable)
inputs = {pathToExecutable: executable}
# make a "stage" directory where input files are collected
path_to_stage_dir = os.path.join(self.optimFolder, jobname)
# 2. apply substitutions to parameter files in local base dir
for (path, changes) in substs.iteritems():
for (var, val, index, regex) in changes:
support.update_parameter_in_file(
os.path.join(localBaseDir, path), var, index, val,
regex)
# adjust xVar in parameter file
index = 0
regex = 'bar-separated'
for xVar, xVal in zip(xVars, paraCombo):
support.update_parameter_in_file(
os.path.join(localBaseDir, path), xVar, 0, xVal, regex)
path_to_stage_dir += '_' + xVar + '=' + str(xVal)
gc3libs.utils.mkdir(path_to_stage_dir)
prefix_len = len(path_to_stage_dir) + 1
# fill stage dir
fillInputDir(localBaseDir, path_to_stage_dir)
# 3. build input file list
for dirpath, dirnames, filenames in os.walk(path_to_stage_dir):
for filename in filenames:
# cut the leading part, which is == to path_to_stage_dir
relpath = dirpath[prefix_len:]
# ignore output directory contents in resubmission
if relpath.startswith('output'):
continue
remote_path = os.path.join(relpath, filename)
inputs[os.path.join(dirpath, filename)] = remote_path
# all contents of the `output` directory are to be fetched
outputs = {'output/': ''}
#kwargs = sessionParas.copy()
kwargs = {}
kwargs['jobname'] = self.jobname
kwargs['stdout'] = 'idriskOut.log'
kwargs['join'] = True
kwargs['output_dir'] = os.path.join(path_to_stage_dir, 'output')
kwargs['requested_architecture'] = architecture
# adaptions for uml
if use_apppot:
if apppot_img is not None:
kwargs['apppot_img'] = apppot_img
if apppot_changes is not None:
kwargs['apppot_changes'] = apppot_changes
cls = idRiskApppotApplication
callExecutable = '/home/user/job/' + executable
elif rte:
kwargs['apppot_tag'] = 'ENV/APPPOT-0.26'
kwargs['tags'] = ['TEST/APPPOT-IBF-1.0']
cls = idRiskApppotApplication
callExecutable = '/home/user/job/' + executable
else:
cls = idRiskApplication
callExecutable = './' + executable
kwargs.setdefault('tags', [])
print 'cls = %s' % cls
print 'callExecutable = %s' % callExecutable
print 'kwargs = %s' % kwargs
# hand over job to create
curApplication = cls(callExecutable, [], inputs, outputs, **kwargs)
tasks.append(curApplication)
return tasks
def new_tasks(self, extra):
# setup AppPot parameters
use_apppot = False
apppot_img = None
apppot_changes = None
if self.params.apppot:
use_apppot = True
if self.params.apppot.endswith('.changes.tar.gz'):
apppot_changes = self.params.apppot
else:
apppot_img = self.params.apppot
inputs = self._search_for_input_files(self.params.args)
# Copy base dir
localBaseDir = os.path.join(os.getcwd(), 'localBaseDir')
# gc3libs.utils.copytree(self.params.initial, '/mnt/shareOffice/ForwardPremium/Results/sensitivity/wGridSize/dfs')
gc3libs.utils.copytree(self.params.initial, localBaseDir)
for path in inputs:
para_loop = path
path_to_base_dir = os.path.dirname(para_loop)
# self.log.debug("Processing loop file '%s' ...", para_loop)
for jobname, substs in self.process_para_file(para_loop):
## self.log.debug("Job '%s' defined by substitutions: %s.",
## jobname, substs)
executable = os.path.basename(self.params.executable)
inputs = { self.params.executable:executable }
# make a "stage" directory where input files are collected
path_to_stage_dir = self.make_directory_path(
self.params.output, jobname)
input_dir = path_to_stage_dir #os.path.join(path_to_stage_dir, 'input')
gc3libs.utils.mkdir(input_dir)
prefix_len = len(input_dir) + 1
# 2. apply substitutions to parameter files
for (path, changes) in substs.iteritems():
for (var, val, index, regex) in changes:
update_parameter_in_file(os.path.join(localBaseDir, path),
var, index, val, regex)
fillInputDir(localBaseDir, input_dir)
# 3. build input file list
for dirpath,dirnames,filenames in os.walk(input_dir):
for filename in filenames:
# cut the leading part, which is == to path_to_stage_dir
relpath = dirpath[prefix_len:]
# ignore output directory contents in resubmission
if relpath. startswith('output'):
continue
remote_path = os.path.join(relpath, filename)
inputs[os.path.join(dirpath, filename)] = remote_path
# all contents of the `output` directory are to be fetched
outputs = { 'output/':'' }
kwargs = extra.copy()
kwargs['stdout'] = 'idRisk.log'
kwargs['join'] = True
kwargs['output_dir'] = os.path.join(path_to_stage_dir, 'output')
kwargs['requested_architecture'] = self.params.architecture
print 'kwargs = %s' % kwargs
print 'inputs = %s' % inputs
print 'outputs = %s' % outputs
# adaptions for uml
if use_apppot:
if apppot_img is not None:
kwargs['apppot_img'] = apppot_img
if apppot_changes is not None:
kwargs['apppot_changes'] = apppot_changes
cls = idRiskApppotApplication
else:
cls = idRiskApplication
#kwargs['apppot_img'] = '/home/benjamin/workspace/apppot0+ben.disk.img'
#kwargs['linux'] = '/home/benjamin/workspace/kernel64-3.0.8'
#kwargs['uml'] = '/home/benjamin/workspace/kernel64-3.0.8'
#kwargs['tags'] = []
kwargs.setdefault('tags', [ ])
# hand over job to create
yield (jobname, cls, ['/home/user/job/' + executable, [], inputs, outputs], kwargs)
def __init__(self, paraFolder, substs, solverParas, **sessionParas):
logger.debug('entering solveParaCombination.__init__ for job %s' % sessionParas['jobname'])
self.iter = 0
self.jobname = 'idRiskParaSearchDriver' + sessionParas['jobname']
self.substs = substs
self.sessionParas = sessionParas
self.pathToExecutable = sessionParas['pathToExecutable']
self.architecture = sessionParas['architecture']
self.localBaseDir = sessionParas['localBaseDir']
self.paraFolder = os.path.join(os.getcwd(), sessionParas['jobname'])
# setup AppPot parameters
use_apppot = False
apppot_img = None
apppot_changes = None
apppot_file = sessionParas['AppPotFile']
if apppot_file:
use_apppot = True
if apppot_file.endswith('.changes.tar.gz'):
apppot_changes = apppot_file
else:
apppot_img = apppot_file
pathToExecutable = sessionParas['pathToExecutable']
localBaseDir = sessionParas['localBaseDir']
architecture = sessionParas['architecture']
jobname = sessionParas['jobname']
executable = os.path.basename(self.pathToExecutable)
inputs = { self.pathToExecutable:executable }
# make a "stage" directory where input files are collected
path_to_stage_dir = os.path.join(self.paraFolder, jobname)
gc3libs.utils.mkdir(path_to_stage_dir)
prefix_len = len(path_to_stage_dir) + 1
# 2. apply substitutions to parameter files
for (path, changes) in substs.iteritems():
for (var, val, index, regex) in changes:
support.update_parameter_in_file(os.path.join(localBaseDir, path),
var, index, val, regex)
support.fillInputDir(localBaseDir, path_to_stage_dir)
# 3. build input file list
for dirpath,dirnames,filenames in os.walk(path_to_stage_dir):
for filename in filenames:
# cut the leading part, which is == to path_to_stage_dir
relpath = dirpath[prefix_len:]
# ignore output directory contents in resubmission
if relpath. startswith('output'):
continue
remote_path = os.path.join(relpath, filename)
inputs[os.path.join(dirpath, filename)] = remote_path
# all contents of the `output` directory are to be fetched
outputs = { 'output/':'' }
kwargs = {}
kwargs['stdout'] = 'idRisk.log'
kwargs['join'] = True
kwargs['output_dir'] = os.path.join(path_to_stage_dir, 'output')
kwargs['requested_architecture'] = self.architecture
# adaptions for uml
if use_apppot:
if apppot_img is not None:
kwargs['apppot_img'] = apppot_img
if apppot_changes is not None:
kwargs['apppot_changes'] = apppot_changes
cls = idRiskApppotApplication
else:
cls = idRiskApplication
kwargs.setdefault('tags', [ ])
# hand over job to create
# self.curApplication = cls('/home/user/job/' + executable, [], inputs, outputs, **kwargs)
self.curApplication = cls('./' + executable, [], inputs, outputs, **kwargs)
SequentialTaskCollection.__init__(self, self.jobname, [ self.curApplication ])
logger.debug('done gParaSearchDriver.__init__ for job %s' % sessionParas['jobname'])
def generateTaskList(self, xVars, paraCombos, substs, sessionParas):
# setup AppPot parameters
use_apppot = False
apppot_img = None
apppot_changes = None
apppot_file = sessionParas["AppPotFile"]
if apppot_file:
use_apppot = True
if apppot_file.endswith(".changes.tar.gz"):
apppot_changes = apppot_file
else:
apppot_img = apppot_file
pathToExecutable = sessionParas["pathToExecutable"]
localBaseDir = sessionParas["localBaseDir"]
architecture = sessionParas["architecture"]
jobname = sessionParas["jobname"]
# Fill the task list
tasks = []
for paraCombo in paraCombos:
logger.debug("paraCombo = %s" % paraCombo)
executable = os.path.basename(pathToExecutable)
inputs = {pathToExecutable: executable}
# make a "stage" directory where input files are collected
path_to_stage_dir = os.path.join(self.optimFolder, jobname)
# 2. apply substitutions to parameter files in local base dir
for (path, changes) in substs.iteritems():
for (var, val, index, regex) in changes:
support.update_parameter_in_file(os.path.join(localBaseDir, path), var, index, val, regex)
# adjust xVar in parameter file
index = 0
regex = "bar-separated"
for xVar, xVal in zip(xVars, paraCombo):
support.update_parameter_in_file(os.path.join(localBaseDir, path), xVar, 0, xVal, regex)
path_to_stage_dir += "_" + xVar + "=" + str(xVal)
gc3libs.utils.mkdir(path_to_stage_dir)
prefix_len = len(path_to_stage_dir) + 1
# fill stage dir
fillInputDir(localBaseDir, path_to_stage_dir)
# 3. build input file list
for dirpath, dirnames, filenames in os.walk(path_to_stage_dir):
for filename in filenames:
# cut the leading part, which is == to path_to_stage_dir
relpath = dirpath[prefix_len:]
# ignore output directory contents in resubmission
if relpath.startswith("output"):
continue
remote_path = os.path.join(relpath, filename)
inputs[os.path.join(dirpath, filename)] = remote_path
# all contents of the `output` directory are to be fetched
outputs = {"output/": ""}
# kwargs = sessionParas.copy()
kwargs = {}
kwargs["jobname"] = self.jobname
kwargs["stdout"] = "idriskOut.log"
kwargs["join"] = True
kwargs["output_dir"] = os.path.join(path_to_stage_dir, "output")
kwargs["requested_architecture"] = architecture
# adaptions for uml
if use_apppot:
if apppot_img is not None:
kwargs["apppot_img"] = apppot_img
if apppot_changes is not None:
kwargs["apppot_changes"] = apppot_changes
cls = idRiskApppotApplication
else:
cls = idRiskApplication
kwargs.setdefault("tags", [])
# hand over job to create
curApplication = cls("./" + executable, [], inputs, outputs, **kwargs)
tasks.append(curApplication)
return tasks
def generateTaskList(self, xVars, paraCombos, substs, sessionParas):
# setup AppPot parameters
use_apppot = False
apppot_img = None
apppot_changes = None
apppot_file = sessionParas['AppPotFile']
if apppot_file:
# print 'apppot_file = %s' % apppot_file
# os._exit(1)
use_apppot = True
if apppot_file.endswith('.changes.tar.gz'):
apppot_changes = apppot_file
else:
apppot_img = apppot_file
pathToExecutable = sessionParas['pathToExecutable']
localBaseDir = sessionParas['localBaseDir']
architecture = sessionParas['architecture']
jobname = sessionParas['jobname']
rte = sessionParas['rte']
# Fill the task list
tasks = []
for paraCombo in paraCombos:
logger.debug('paraCombo = %s' % paraCombo)
executable = os.path.basename(pathToExecutable)
inputs = { pathToExecutable:executable }
# make a "stage" directory where input files are collected
path_to_stage_dir = os.path.join(self.optimFolder, jobname)
# 2. apply substitutions to parameter files in local base dir
for (path, changes) in substs.iteritems():
for (var, val, index, regex) in changes:
support.update_parameter_in_file(os.path.join(localBaseDir, path), var, index, val, regex)
# adjust xVar in parameter file
index = 0
regex = 'bar-separated'
for xVar, xVal in zip(xVars, paraCombo):
support.update_parameter_in_file(os.path.join(localBaseDir, path), xVar, 0, xVal, regex)
path_to_stage_dir += '_' + xVar + '=' + str(xVal)
gc3libs.utils.mkdir(path_to_stage_dir)
prefix_len = len(path_to_stage_dir) + 1
# fill stage dir
fillInputDir(localBaseDir, path_to_stage_dir)
# 3. build input file list
for dirpath,dirnames,filenames in os.walk(path_to_stage_dir):
for filename in filenames:
# cut the leading part, which is == to path_to_stage_dir
relpath = dirpath[prefix_len:]
# ignore output directory contents in resubmission
if relpath.startswith('output'):
continue
remote_path = os.path.join(relpath, filename)
inputs[os.path.join(dirpath, filename)] = remote_path
# all contents of the `output` directory are to be fetched
outputs = { 'output/':'' }
#kwargs = sessionParas.copy()
kwargs = {}
kwargs['jobname'] = self.jobname
kwargs['stdout'] = 'idriskOut.log'
kwargs['join'] = True
kwargs['output_dir'] = os.path.join(path_to_stage_dir, 'output')
kwargs['requested_architecture'] = architecture
# adaptions for uml
if use_apppot:
if apppot_img is not None:
kwargs['apppot_img'] = apppot_img
if apppot_changes is not None:
kwargs['apppot_changes'] = apppot_changes
cls = idRiskApppotApplication
callExecutable = '/home/user/job/' + executable
elif rte:
kwargs['apppot_tag'] = 'ENV/APPPOT-0.26'
kwargs['tags'] = ['TEST/APPPOT-IBF-1.0']
cls = idRiskApppotApplication
callExecutable = '/home/user/job/' + executable
else:
cls = idRiskApplication
callExecutable = './' + executable
kwargs.setdefault('tags', [ ])
print 'cls = %s' % cls
print 'callExecutable = %s' % callExecutable
print 'kwargs = %s' % kwargs
# hand over job to create
curApplication = cls(callExecutable, [], inputs, outputs, **kwargs)
tasks.append(curApplication)
return tasks
def new_tasks(self, extra):
# setup AppPot parameters
use_apppot = False
apppot_img = None
apppot_changes = None
if self.params.apppot:
use_apppot = True
if self.params.apppot.endswith('.changes.tar.gz'):
apppot_changes = self.params.apppot
else:
apppot_img = self.params.apppot
inputs = self._search_for_input_files(self.params.args)
# Copy base dir
localBaseDir = os.path.join(os.getcwd(), 'localBaseDir')
# gc3libs.utils.copytree(self.params.initial, '/mnt/shareOffice/ForwardPremium/Results/sensitivity/wGridSize/dfs')
shutil.copytree(self.params.initial, localBaseDir)
for path in inputs:
para_loop = path
path_to_base_dir = os.path.dirname(para_loop)
# self.log.debug("Processing loop file '%s' ...", para_loop)
for jobname, substs in self.process_para_file(para_loop):
## self.log.debug("Job '%s' defined by substitutions: %s.",
## jobname, substs)
executable = os.path.basename(self.params.executable)
inputs = {self.params.executable: executable}
# make a "stage" directory where input files are collected
path_to_stage_dir = self.make_directory_path(
self.params.output, jobname)
input_dir = path_to_stage_dir #os.path.join(path_to_stage_dir, 'input')
gc3libs.utils.mkdir(input_dir)
prefix_len = len(input_dir) + 1
# 2. apply substitutions to parameter files
for (path, changes) in substs.iteritems():
for (var, val, index, regex) in changes:
update_parameter_in_file(
os.path.join(localBaseDir, path), var, index, val,
regex)
fillInputDir(localBaseDir, input_dir)
# 3. build input file list
for dirpath, dirnames, filenames in os.walk(input_dir):
for filename in filenames:
# cut the leading part, which is == to path_to_stage_dir
relpath = dirpath[prefix_len:]
# ignore output directory contents in resubmission
if relpath.startswith('output'):
continue
remote_path = os.path.join(relpath, filename)
inputs[os.path.join(dirpath, filename)] = remote_path
# all contents of the `output` directory are to be fetched
outputs = {'output/': ''}
kwargs = extra.copy()
kwargs['stdout'] = 'idRisk.log'
kwargs['join'] = True
kwargs['output_dir'] = os.path.join(path_to_stage_dir,
'output')
kwargs['requested_architecture'] = self.params.architecture
print 'kwargs = %s' % kwargs
print 'inputs = %s' % inputs
print 'outputs = %s' % outputs
# adaptions for uml
if use_apppot:
if apppot_img is not None:
kwargs['apppot_img'] = apppot_img
if apppot_changes is not None:
kwargs['apppot_changes'] = apppot_changes
cls = idRiskApppotApplication
else:
cls = idRiskApplication
#kwargs['apppot_img'] = '/home/benjamin/workspace/apppot0+ben.disk.img'
#kwargs['linux'] = '/home/benjamin/workspace/kernel64-3.0.8'
#kwargs['uml'] = '/home/benjamin/workspace/kernel64-3.0.8'
#kwargs['tags'] = []
kwargs.setdefault('tags', [])
# hand over job to create
yield (jobname, cls,
['/home/user/job/' + executable, [], inputs,
outputs], kwargs)
