def new_tasks(self, extra): inputs = self._search_for_input_files(self.params.args) # Copy base dir localBaseDir = os.path.join(os.getcwd(), 'localBaseDir') # gc3libs.utils.copytree(self.params.initial, '/mnt/shareOffice/ForwardPremium/Results/sensitivity/wGridSize/dfs') gc3libs.utils.copytree(self.params.initial, localBaseDir) for path in inputs: para_loop = path path_to_base_dir = os.path.dirname(para_loop) # self.log.debug("Processing loop file '%s' ...", para_loop) for jobname, substs in self.process_para_file(para_loop): ## self.log.debug("Job '%s' defined by substitutions: %s.", ## jobname, substs) executable = os.path.basename(self.params.executable) inputs = {self.params.executable: executable} # make a "stage" directory where input files are collected path_to_stage_dir = self.make_directory_path( self.params.output, jobname) input_dir = path_to_stage_dir #os.path.join(path_to_stage_dir, 'input') gc3libs.utils.mkdir(input_dir) prefix_len = len(input_dir) + 1 # 2. apply substitutions to parameter files for (path, changes) in substs.iteritems(): for (var, val, index, regex) in changes: update_parameter_in_file( os.path.join(localBaseDir, path), var, index, val, regex) fillInputDir(localBaseDir, input_dir) # 3. build input file list for dirpath, dirnames, filenames in os.walk(input_dir): for filename in filenames: # cut the leading part, which is == to path_to_stage_dir relpath = dirpath[prefix_len:] # ignore output directory contents in resubmission if relpath.startswith('output'): continue remote_path = os.path.join(relpath, filename) inputs[os.path.join(dirpath, filename)] = remote_path # all contents of the `output` directory are to be fetched outputs = {'output/': ''} kwargs = extra.copy() kwargs['stdout'] = 'forwardPremiumOut.log' kwargs['join'] = True kwargs['output_dir'] = os.path.join(path_to_stage_dir, 'output') kwargs['requested_architecture'] = self.params.architecture print 'kwargs = %s' % kwargs print 'inputs = %s' % inputs print 'outputs = %s' % outputs # hand over job to create yield (jobname, gIdRisk.idRiskApplication, ['./' + executable, [], inputs, outputs], kwargs)
def new_tasks(self, extra): inputs = self._search_for_input_files(self.params.args) # Copy base dir localBaseDir = os.path.join(os.getcwd(), 'localBaseDir') # gc3libs.utils.copytree(self.params.initial, '/mnt/shareOffice/ForwardPremium/Results/sensitivity/wGridSize/dfs') gc3libs.utils.copytree(self.params.initial, localBaseDir) for path in inputs: para_loop = path path_to_base_dir = os.path.dirname(para_loop) # self.log.debug("Processing loop file '%s' ...", para_loop) for jobname, substs in self.process_para_file(para_loop): ## self.log.debug("Job '%s' defined by substitutions: %s.", ## jobname, substs) executable = os.path.basename(self.params.executable) inputs = { self.params.executable:executable } # make a "stage" directory where input files are collected path_to_stage_dir = self.make_directory_path( self.params.output, jobname) input_dir = path_to_stage_dir #os.path.join(path_to_stage_dir, 'input') gc3libs.utils.mkdir(input_dir) prefix_len = len(input_dir) + 1 # 2. apply substitutions to parameter files for (path, changes) in substs.iteritems(): for (var, val, index, regex) in changes: update_parameter_in_file(os.path.join(localBaseDir, path), var, index, val, regex) fillInputDir(localBaseDir, input_dir) # 3. build input file list for dirpath,dirnames,filenames in os.walk(input_dir): for filename in filenames: # cut the leading part, which is == to path_to_stage_dir relpath = dirpath[prefix_len:] # ignore output directory contents in resubmission if relpath.startswith('output'): continue remote_path = os.path.join(relpath, filename) inputs[os.path.join(dirpath, filename)] = remote_path # all contents of the `output` directory are to be fetched outputs = { 'output/':'' } kwargs = extra.copy() kwargs['stdout'] = 'forwardPremiumOut.log' kwargs['join'] = True kwargs['output_dir'] = os.path.join(path_to_stage_dir, 'output') kwargs['requested_architecture'] = self.params.architecture print 'kwargs = %s' % kwargs print 'inputs = %s' % inputs print 'outputs = %s' % outputs # hand over job to create yield (jobname, gIdRisk.idRiskApplication, ['./' + executable, [], inputs, outputs], kwargs)
def generateTaskList(self, xVars, paraCombos, substs, sessionParas): # setup AppPot parameters use_apppot = False apppot_img = None apppot_changes = None apppot_file = sessionParas['AppPotFile'] if apppot_file: # print 'apppot_file = %s' % apppot_file # os._exit(1) use_apppot = True if apppot_file.endswith('.changes.tar.gz'): apppot_changes = apppot_file else: apppot_img = apppot_file pathToExecutable = sessionParas['pathToExecutable'] localBaseDir = sessionParas['localBaseDir'] architecture = sessionParas['architecture'] jobname = sessionParas['jobname'] rte = sessionParas['rte'] # Fill the task list tasks = [] for paraCombo in paraCombos: logger.debug('paraCombo = %s' % paraCombo) executable = os.path.basename(pathToExecutable) inputs = {pathToExecutable: executable} # make a "stage" directory where input files are collected path_to_stage_dir = os.path.join(self.optimFolder, jobname) # 2. apply substitutions to parameter files in local base dir for (path, changes) in substs.iteritems(): for (var, val, index, regex) in changes: support.update_parameter_in_file( os.path.join(localBaseDir, path), var, index, val, regex) # adjust xVar in parameter file index = 0 regex = 'bar-separated' for xVar, xVal in zip(xVars, paraCombo): support.update_parameter_in_file( os.path.join(localBaseDir, path), xVar, 0, xVal, regex) path_to_stage_dir += '_' + xVar + '=' + str(xVal) gc3libs.utils.mkdir(path_to_stage_dir) prefix_len = len(path_to_stage_dir) + 1 # fill stage dir fillInputDir(localBaseDir, path_to_stage_dir) # 3. build input file list for dirpath, dirnames, filenames in os.walk(path_to_stage_dir): for filename in filenames: # cut the leading part, which is == to path_to_stage_dir relpath = dirpath[prefix_len:] # ignore output directory contents in resubmission if relpath.startswith('output'): continue remote_path = os.path.join(relpath, filename) inputs[os.path.join(dirpath, filename)] = remote_path # all contents of the `output` directory are to be fetched outputs = {'output/': ''} #kwargs = sessionParas.copy() kwargs = {} kwargs['jobname'] = self.jobname kwargs['stdout'] = 'idriskOut.log' kwargs['join'] = True kwargs['output_dir'] = os.path.join(path_to_stage_dir, 'output') kwargs['requested_architecture'] = architecture # adaptions for uml if use_apppot: if apppot_img is not None: kwargs['apppot_img'] = apppot_img if apppot_changes is not None: kwargs['apppot_changes'] = apppot_changes cls = idRiskApppotApplication callExecutable = '/home/user/job/' + executable elif rte: kwargs['apppot_tag'] = 'ENV/APPPOT-0.26' kwargs['tags'] = ['TEST/APPPOT-IBF-1.0'] cls = idRiskApppotApplication callExecutable = '/home/user/job/' + executable else: cls = idRiskApplication callExecutable = './' + executable kwargs.setdefault('tags', []) print 'cls = %s' % cls print 'callExecutable = %s' % callExecutable print 'kwargs = %s' % kwargs # hand over job to create curApplication = cls(callExecutable, [], inputs, outputs, **kwargs) tasks.append(curApplication) return tasks
def new_tasks(self, extra): # setup AppPot parameters use_apppot = False apppot_img = None apppot_changes = None if self.params.apppot: use_apppot = True if self.params.apppot.endswith('.changes.tar.gz'): apppot_changes = self.params.apppot else: apppot_img = self.params.apppot inputs = self._search_for_input_files(self.params.args) # Copy base dir localBaseDir = os.path.join(os.getcwd(), 'localBaseDir') # gc3libs.utils.copytree(self.params.initial, '/mnt/shareOffice/ForwardPremium/Results/sensitivity/wGridSize/dfs') gc3libs.utils.copytree(self.params.initial, localBaseDir) for path in inputs: para_loop = path path_to_base_dir = os.path.dirname(para_loop) # self.log.debug("Processing loop file '%s' ...", para_loop) for jobname, substs in self.process_para_file(para_loop): ## self.log.debug("Job '%s' defined by substitutions: %s.", ## jobname, substs) executable = os.path.basename(self.params.executable) inputs = { self.params.executable:executable } # make a "stage" directory where input files are collected path_to_stage_dir = self.make_directory_path( self.params.output, jobname) input_dir = path_to_stage_dir #os.path.join(path_to_stage_dir, 'input') gc3libs.utils.mkdir(input_dir) prefix_len = len(input_dir) + 1 # 2. apply substitutions to parameter files for (path, changes) in substs.iteritems(): for (var, val, index, regex) in changes: update_parameter_in_file(os.path.join(localBaseDir, path), var, index, val, regex) fillInputDir(localBaseDir, input_dir) # 3. build input file list for dirpath,dirnames,filenames in os.walk(input_dir): for filename in filenames: # cut the leading part, which is == to path_to_stage_dir relpath = dirpath[prefix_len:] # ignore output directory contents in resubmission if relpath. startswith('output'): continue remote_path = os.path.join(relpath, filename) inputs[os.path.join(dirpath, filename)] = remote_path # all contents of the `output` directory are to be fetched outputs = { 'output/':'' } kwargs = extra.copy() kwargs['stdout'] = 'idRisk.log' kwargs['join'] = True kwargs['output_dir'] = os.path.join(path_to_stage_dir, 'output') kwargs['requested_architecture'] = self.params.architecture print 'kwargs = %s' % kwargs print 'inputs = %s' % inputs print 'outputs = %s' % outputs # adaptions for uml if use_apppot: if apppot_img is not None: kwargs['apppot_img'] = apppot_img if apppot_changes is not None: kwargs['apppot_changes'] = apppot_changes cls = idRiskApppotApplication else: cls = idRiskApplication #kwargs['apppot_img'] = '/home/benjamin/workspace/apppot0+ben.disk.img' #kwargs['linux'] = '/home/benjamin/workspace/kernel64-3.0.8' #kwargs['uml'] = '/home/benjamin/workspace/kernel64-3.0.8' #kwargs['tags'] = [] kwargs.setdefault('tags', [ ]) # hand over job to create yield (jobname, cls, ['/home/user/job/' + executable, [], inputs, outputs], kwargs)
def __init__(self, paraFolder, substs, solverParas, **sessionParas): logger.debug('entering solveParaCombination.__init__ for job %s' % sessionParas['jobname']) self.iter = 0 self.jobname = 'idRiskParaSearchDriver' + sessionParas['jobname'] self.substs = substs self.sessionParas = sessionParas self.pathToExecutable = sessionParas['pathToExecutable'] self.architecture = sessionParas['architecture'] self.localBaseDir = sessionParas['localBaseDir'] self.paraFolder = os.path.join(os.getcwd(), sessionParas['jobname']) # setup AppPot parameters use_apppot = False apppot_img = None apppot_changes = None apppot_file = sessionParas['AppPotFile'] if apppot_file: use_apppot = True if apppot_file.endswith('.changes.tar.gz'): apppot_changes = apppot_file else: apppot_img = apppot_file pathToExecutable = sessionParas['pathToExecutable'] localBaseDir = sessionParas['localBaseDir'] architecture = sessionParas['architecture'] jobname = sessionParas['jobname'] executable = os.path.basename(self.pathToExecutable) inputs = { self.pathToExecutable:executable } # make a "stage" directory where input files are collected path_to_stage_dir = os.path.join(self.paraFolder, jobname) gc3libs.utils.mkdir(path_to_stage_dir) prefix_len = len(path_to_stage_dir) + 1 # 2. apply substitutions to parameter files for (path, changes) in substs.iteritems(): for (var, val, index, regex) in changes: support.update_parameter_in_file(os.path.join(localBaseDir, path), var, index, val, regex) support.fillInputDir(localBaseDir, path_to_stage_dir) # 3. build input file list for dirpath,dirnames,filenames in os.walk(path_to_stage_dir): for filename in filenames: # cut the leading part, which is == to path_to_stage_dir relpath = dirpath[prefix_len:] # ignore output directory contents in resubmission if relpath. startswith('output'): continue remote_path = os.path.join(relpath, filename) inputs[os.path.join(dirpath, filename)] = remote_path # all contents of the `output` directory are to be fetched outputs = { 'output/':'' } kwargs = {} kwargs['stdout'] = 'idRisk.log' kwargs['join'] = True kwargs['output_dir'] = os.path.join(path_to_stage_dir, 'output') kwargs['requested_architecture'] = self.architecture # adaptions for uml if use_apppot: if apppot_img is not None: kwargs['apppot_img'] = apppot_img if apppot_changes is not None: kwargs['apppot_changes'] = apppot_changes cls = idRiskApppotApplication else: cls = idRiskApplication kwargs.setdefault('tags', [ ]) # hand over job to create # self.curApplication = cls('/home/user/job/' + executable, [], inputs, outputs, **kwargs) self.curApplication = cls('./' + executable, [], inputs, outputs, **kwargs) SequentialTaskCollection.__init__(self, self.jobname, [ self.curApplication ]) logger.debug('done gParaSearchDriver.__init__ for job %s' % sessionParas['jobname'])
def generateTaskList(self, xVars, paraCombos, substs, sessionParas): # setup AppPot parameters use_apppot = False apppot_img = None apppot_changes = None apppot_file = sessionParas["AppPotFile"] if apppot_file: use_apppot = True if apppot_file.endswith(".changes.tar.gz"): apppot_changes = apppot_file else: apppot_img = apppot_file pathToExecutable = sessionParas["pathToExecutable"] localBaseDir = sessionParas["localBaseDir"] architecture = sessionParas["architecture"] jobname = sessionParas["jobname"] # Fill the task list tasks = [] for paraCombo in paraCombos: logger.debug("paraCombo = %s" % paraCombo) executable = os.path.basename(pathToExecutable) inputs = {pathToExecutable: executable} # make a "stage" directory where input files are collected path_to_stage_dir = os.path.join(self.optimFolder, jobname) # 2. apply substitutions to parameter files in local base dir for (path, changes) in substs.iteritems(): for (var, val, index, regex) in changes: support.update_parameter_in_file(os.path.join(localBaseDir, path), var, index, val, regex) # adjust xVar in parameter file index = 0 regex = "bar-separated" for xVar, xVal in zip(xVars, paraCombo): support.update_parameter_in_file(os.path.join(localBaseDir, path), xVar, 0, xVal, regex) path_to_stage_dir += "_" + xVar + "=" + str(xVal) gc3libs.utils.mkdir(path_to_stage_dir) prefix_len = len(path_to_stage_dir) + 1 # fill stage dir fillInputDir(localBaseDir, path_to_stage_dir) # 3. build input file list for dirpath, dirnames, filenames in os.walk(path_to_stage_dir): for filename in filenames: # cut the leading part, which is == to path_to_stage_dir relpath = dirpath[prefix_len:] # ignore output directory contents in resubmission if relpath.startswith("output"): continue remote_path = os.path.join(relpath, filename) inputs[os.path.join(dirpath, filename)] = remote_path # all contents of the `output` directory are to be fetched outputs = {"output/": ""} # kwargs = sessionParas.copy() kwargs = {} kwargs["jobname"] = self.jobname kwargs["stdout"] = "idriskOut.log" kwargs["join"] = True kwargs["output_dir"] = os.path.join(path_to_stage_dir, "output") kwargs["requested_architecture"] = architecture # adaptions for uml if use_apppot: if apppot_img is not None: kwargs["apppot_img"] = apppot_img if apppot_changes is not None: kwargs["apppot_changes"] = apppot_changes cls = idRiskApppotApplication else: cls = idRiskApplication kwargs.setdefault("tags", []) # hand over job to create curApplication = cls("./" + executable, [], inputs, outputs, **kwargs) tasks.append(curApplication) return tasks
def generateTaskList(self, xVars, paraCombos, substs, sessionParas): # setup AppPot parameters use_apppot = False apppot_img = None apppot_changes = None apppot_file = sessionParas['AppPotFile'] if apppot_file: # print 'apppot_file = %s' % apppot_file # os._exit(1) use_apppot = True if apppot_file.endswith('.changes.tar.gz'): apppot_changes = apppot_file else: apppot_img = apppot_file pathToExecutable = sessionParas['pathToExecutable'] localBaseDir = sessionParas['localBaseDir'] architecture = sessionParas['architecture'] jobname = sessionParas['jobname'] rte = sessionParas['rte'] # Fill the task list tasks = [] for paraCombo in paraCombos: logger.debug('paraCombo = %s' % paraCombo) executable = os.path.basename(pathToExecutable) inputs = { pathToExecutable:executable } # make a "stage" directory where input files are collected path_to_stage_dir = os.path.join(self.optimFolder, jobname) # 2. apply substitutions to parameter files in local base dir for (path, changes) in substs.iteritems(): for (var, val, index, regex) in changes: support.update_parameter_in_file(os.path.join(localBaseDir, path), var, index, val, regex) # adjust xVar in parameter file index = 0 regex = 'bar-separated' for xVar, xVal in zip(xVars, paraCombo): support.update_parameter_in_file(os.path.join(localBaseDir, path), xVar, 0, xVal, regex) path_to_stage_dir += '_' + xVar + '=' + str(xVal) gc3libs.utils.mkdir(path_to_stage_dir) prefix_len = len(path_to_stage_dir) + 1 # fill stage dir fillInputDir(localBaseDir, path_to_stage_dir) # 3. build input file list for dirpath,dirnames,filenames in os.walk(path_to_stage_dir): for filename in filenames: # cut the leading part, which is == to path_to_stage_dir relpath = dirpath[prefix_len:] # ignore output directory contents in resubmission if relpath.startswith('output'): continue remote_path = os.path.join(relpath, filename) inputs[os.path.join(dirpath, filename)] = remote_path # all contents of the `output` directory are to be fetched outputs = { 'output/':'' } #kwargs = sessionParas.copy() kwargs = {} kwargs['jobname'] = self.jobname kwargs['stdout'] = 'idriskOut.log' kwargs['join'] = True kwargs['output_dir'] = os.path.join(path_to_stage_dir, 'output') kwargs['requested_architecture'] = architecture # adaptions for uml if use_apppot: if apppot_img is not None: kwargs['apppot_img'] = apppot_img if apppot_changes is not None: kwargs['apppot_changes'] = apppot_changes cls = idRiskApppotApplication callExecutable = '/home/user/job/' + executable elif rte: kwargs['apppot_tag'] = 'ENV/APPPOT-0.26' kwargs['tags'] = ['TEST/APPPOT-IBF-1.0'] cls = idRiskApppotApplication callExecutable = '/home/user/job/' + executable else: cls = idRiskApplication callExecutable = './' + executable kwargs.setdefault('tags', [ ]) print 'cls = %s' % cls print 'callExecutable = %s' % callExecutable print 'kwargs = %s' % kwargs # hand over job to create curApplication = cls(callExecutable, [], inputs, outputs, **kwargs) tasks.append(curApplication) return tasks
def new_tasks(self, extra): # setup AppPot parameters use_apppot = False apppot_img = None apppot_changes = None if self.params.apppot: use_apppot = True if self.params.apppot.endswith('.changes.tar.gz'): apppot_changes = self.params.apppot else: apppot_img = self.params.apppot inputs = self._search_for_input_files(self.params.args) # Copy base dir localBaseDir = os.path.join(os.getcwd(), 'localBaseDir') # gc3libs.utils.copytree(self.params.initial, '/mnt/shareOffice/ForwardPremium/Results/sensitivity/wGridSize/dfs') shutil.copytree(self.params.initial, localBaseDir) for path in inputs: para_loop = path path_to_base_dir = os.path.dirname(para_loop) # self.log.debug("Processing loop file '%s' ...", para_loop) for jobname, substs in self.process_para_file(para_loop): ## self.log.debug("Job '%s' defined by substitutions: %s.", ## jobname, substs) executable = os.path.basename(self.params.executable) inputs = {self.params.executable: executable} # make a "stage" directory where input files are collected path_to_stage_dir = self.make_directory_path( self.params.output, jobname) input_dir = path_to_stage_dir #os.path.join(path_to_stage_dir, 'input') gc3libs.utils.mkdir(input_dir) prefix_len = len(input_dir) + 1 # 2. apply substitutions to parameter files for (path, changes) in substs.iteritems(): for (var, val, index, regex) in changes: update_parameter_in_file( os.path.join(localBaseDir, path), var, index, val, regex) fillInputDir(localBaseDir, input_dir) # 3. build input file list for dirpath, dirnames, filenames in os.walk(input_dir): for filename in filenames: # cut the leading part, which is == to path_to_stage_dir relpath = dirpath[prefix_len:] # ignore output directory contents in resubmission if relpath.startswith('output'): continue remote_path = os.path.join(relpath, filename) inputs[os.path.join(dirpath, filename)] = remote_path # all contents of the `output` directory are to be fetched outputs = {'output/': ''} kwargs = extra.copy() kwargs['stdout'] = 'idRisk.log' kwargs['join'] = True kwargs['output_dir'] = os.path.join(path_to_stage_dir, 'output') kwargs['requested_architecture'] = self.params.architecture print 'kwargs = %s' % kwargs print 'inputs = %s' % inputs print 'outputs = %s' % outputs # adaptions for uml if use_apppot: if apppot_img is not None: kwargs['apppot_img'] = apppot_img if apppot_changes is not None: kwargs['apppot_changes'] = apppot_changes cls = idRiskApppotApplication else: cls = idRiskApplication #kwargs['apppot_img'] = '/home/benjamin/workspace/apppot0+ben.disk.img' #kwargs['linux'] = '/home/benjamin/workspace/kernel64-3.0.8' #kwargs['uml'] = '/home/benjamin/workspace/kernel64-3.0.8' #kwargs['tags'] = [] kwargs.setdefault('tags', []) # hand over job to create yield (jobname, cls, ['/home/user/job/' + executable, [], inputs, outputs], kwargs)