def new_tasks(self, extra): inputs = self._search_for_input_files(self.params.args) # Copy base dir localBaseDir = os.path.join(os.getcwd(), 'localBaseDir') # gc3libs.utils.copytree(self.params.initial, '/mnt/shareOffice/ForwardPremium/Results/sensitivity/wGridSize/dfs') gc3libs.utils.copytree(self.params.initial, localBaseDir) for path in inputs: para_loop = path path_to_base_dir = os.path.dirname(para_loop) # self.log.debug("Processing loop file '%s' ...", para_loop) for jobname, substs in self.process_para_file(para_loop): ## self.log.debug("Job '%s' defined by substitutions: %s.", ## jobname, substs) executable = os.path.basename(self.params.executable) inputs = {self.params.executable: executable} # make a "stage" directory where input files are collected path_to_stage_dir = self.make_directory_path( self.params.output, jobname) input_dir = path_to_stage_dir #os.path.join(path_to_stage_dir, 'input') gc3libs.utils.mkdir(input_dir) prefix_len = len(input_dir) + 1 # 2. apply substitutions to parameter files for (path, changes) in substs.iteritems(): for (var, val, index, regex) in changes: update_parameter_in_file( os.path.join(localBaseDir, path), var, index, val, regex) fillInputDir(localBaseDir, input_dir) # 3. build input file list for dirpath, dirnames, filenames in os.walk(input_dir): for filename in filenames: # cut the leading part, which is == to path_to_stage_dir relpath = dirpath[prefix_len:] # ignore output directory contents in resubmission if relpath.startswith('output'): continue remote_path = os.path.join(relpath, filename) inputs[os.path.join(dirpath, filename)] = remote_path # all contents of the `output` directory are to be fetched outputs = {'output/': ''} kwargs = extra.copy() kwargs['stdout'] = 'forwardPremiumOut.log' kwargs['join'] = True kwargs['output_dir'] = os.path.join(path_to_stage_dir, 'output') kwargs['requested_architecture'] = self.params.architecture print 'kwargs = %s' % kwargs print 'inputs = %s' % inputs print 'outputs = %s' % outputs # hand over job to create yield (jobname, gIdRisk.idRiskApplication, ['./' + executable, [], inputs, outputs], kwargs)
def generateTaskList(self, para_loop, iterationFolder): # Fill the task list tasks = [] for jobname, substs in self.process_para_file(para_loop): executable = os.path.basename(self.executable) # start the inputs dictionary with syntax: client_path: server_path inputs = { self.executable:executable } # make a "stage" directory where input files are collected on the client machine. #path_to_stage_dir = self.make_directory_path(os.path.join(iterationFolder, 'NAME'), jobname) path_to_stage_dir = os.path.join(iterationFolder, jobname) # input_dir is cwd/jobname (also referred to as "stage" dir. input_dir = path_to_stage_dir gc3libs.utils.mkdir(input_dir) prefix_len = len(input_dir) + 1 # 1. files in the "initial" dir are copied verbatim # ugly work around to determine the correct base dir in case of one4all. baseDir = os.path.join(os.getcwd(), 'base') for (path, changes) in substs.iteritems(): for (var, val, index, regex) in changes: if var == 'Ctry': baseDir += val if baseDir == 'base': baseDir = self.baseDir self.fillInputDir( baseDir, input_dir) # 2. apply substitutions to parameter files for (path, changes) in substs.iteritems(): for (var, val, index, regex) in changes: # new. make adjustments in the base dir itself. update_parameter_in_file(os.path.join(input_dir, path), var, index, val, regex) # 3. build input file list for dirpath,dirnames,filenames in os.walk(input_dir): for filename in filenames: # cut the leading part, which is == to path_to_stage_dir relpath = dirpath[prefix_len:] # ignore output directory contents in resubmission if relpath.startswith('output'): continue remote_path = os.path.join(relpath, filename) inputs[os.path.join(dirpath, filename)] = remote_path # all contents of the `output` directory are to be fetched outputs = { 'output/':'' } #kwargs = extra.copy() kwargs = self.extra_args kwargs['stdout'] = 'forwardPremiumOut.log' kwargs['join'] = True kwargs['output_dir'] = os.path.join(path_to_stage_dir, 'output') kwargs['requested_architecture'] = self.architecture kwargs['requested_cores'] = 1 # hand over job to create tasks.append(GPremiumApplication('./' + executable, [], inputs, outputs, **kwargs)) return tasks
def new_tasks(self, extra): inputs = self._search_for_input_files(self.params.args) # Copy base dir localBaseDir = os.path.join(os.getcwd(), 'localBaseDir') # gc3libs.utils.copytree(self.params.initial, '/mnt/shareOffice/ForwardPremium/Results/sensitivity/wGridSize/dfs') gc3libs.utils.copytree(self.params.initial, localBaseDir) for path in inputs: para_loop = path path_to_base_dir = os.path.dirname(para_loop) # self.log.debug("Processing loop file '%s' ...", para_loop) for jobname, substs in self.process_para_file(para_loop): ## self.log.debug("Job '%s' defined by substitutions: %s.", ## jobname, substs) executable = os.path.basename(self.params.executable) inputs = { self.params.executable:executable } # make a "stage" directory where input files are collected path_to_stage_dir = self.make_directory_path( self.params.output, jobname) input_dir = path_to_stage_dir #os.path.join(path_to_stage_dir, 'input') gc3libs.utils.mkdir(input_dir) prefix_len = len(input_dir) + 1 # 2. apply substitutions to parameter files for (path, changes) in substs.iteritems(): for (var, val, index, regex) in changes: update_parameter_in_file(os.path.join(localBaseDir, path), var, index, val, regex) fillInputDir(localBaseDir, input_dir) # 3. build input file list for dirpath,dirnames,filenames in os.walk(input_dir): for filename in filenames: # cut the leading part, which is == to path_to_stage_dir relpath = dirpath[prefix_len:] # ignore output directory contents in resubmission if relpath.startswith('output'): continue remote_path = os.path.join(relpath, filename) inputs[os.path.join(dirpath, filename)] = remote_path # all contents of the `output` directory are to be fetched outputs = { 'output/':'' } kwargs = extra.copy() kwargs['stdout'] = 'forwardPremiumOut.log' kwargs['join'] = True kwargs['output_dir'] = os.path.join(path_to_stage_dir, 'output') kwargs['requested_architecture'] = self.params.architecture print 'kwargs = %s' % kwargs print 'inputs = %s' % inputs print 'outputs = %s' % outputs # hand over job to create yield (jobname, gIdRisk.idRiskApplication, ['./' + executable, [], inputs, outputs], kwargs)
def new_tasks(self, extra): logger.info('entering new_tasks') # Adjust logging level for stream handler. Could be extracted into a general function. for handler in logger.handlers: if not isinstance(handler, logbook.FileHandler): logger.handlers.remove(handler) mySH = logbook.StreamHandler(stream = sys.stdout, level = self.params.solverVerb.upper(), format_string = '{record.message}', bubble = True) logger.handlers.append(mySH) baseDir = self.params.initial xVars = self.params.xVars countryList = self.params.countryList.split() ctryIndices = getIndex(base = [len(countryList), len(countryList)], restr = 'lowerTr') for ctryIndex in ctryIndices: logger.info(countryList[ctryIndex[0]] + countryList[ctryIndex[1]]) # Compute domain xVarsDom = self.params.xVarsDom.split() lowerBds = np.array([xVarsDom[i] for i in range(len(xVarsDom)) if i % 2 == 0], dtype = 'float64') upperBds = np.array([xVarsDom[i] for i in range(len(xVarsDom)) if i % 2 == 1], dtype = 'float64') domain = zip(lowerBds, upperBds) # Make problem type specific adjustments. if self.params.problemType == 'one4all': gdpTable = tableDict.fromTextFile(fileIn = os.path.join(self.params.pathEmpirical, 'outputInput/momentTable/Gdp/gdpMoments.csv'), delim = ',', width = 20) jobname = 'one4all' logger.info('%s' % self.params.norm) norm = self.params.norm try: norm = int(norm) except ValueError: if norm == "np.inf" or norm == "inf": norm = np.inf else: pass # in particular custom norms logger.info('using norm %s' % (norm)) path_to_stage_dir = os.getcwd() executable = os.path.basename(self.params.executable) analyzeResults = anaOne4all(len(list(ctryIndices)), norm = norm) nlc = nlcOne4all(gdpTable = gdpTable, ctryList = countryList, domain = domain, logFile = os.path.join(path_to_stage_dir, 'nlc.log')) combOverviews = combineOverviews.combineOverviews(overviewSimuFile = 'eSigmaTable', tableName = 'ag_eSigmaTable', sortKeys = ['norm']) plot3dTable = combineOverviews.plotTable(tablePath =os.path.join(path_to_stage_dir, 'ag_eSigmaTable'), savePath = os.path.join(path_to_stage_dir, 'scatter3d')) plot3dTable.columnNames = ['E', 'sigma', 'norm'] for ctryIndex in ctryIndices: Ctry1 = countryList[ctryIndex[0]] Ctry2 = countryList[ctryIndex[1]] # Set Ctry information for this run. update_parameter_in_file(os.path.join(baseDir, 'input/markovA.in'), 'Ctry', 0, Ctry1, 'space-separated') update_parameter_in_file(os.path.join(baseDir, 'input/markovB.in'), 'Ctry', 0, Ctry2, 'space-separated') # Get the correct Ctry Paras into base dir. self.getCtryParas(baseDir, Ctry1, Ctry2) # Copy base dir ctryBaseDir = os.path.join(path_to_stage_dir, 'base' + Ctry1 + Ctry2) try: shutil.copytree(baseDir, ctryBaseDir) except: logger.info('%s already exists' % baseDir) kwargs = extra.copy() kwargs['output_dir'] = path_to_stage_dir # yield job yield (jobname, gParaSearchDriver, [ self.params.executable, path_to_stage_dir, self.params.architecture, baseDir, self.params.xVars, self.params.initialPop, self.params.nPopulation, domain, self.params.solverVerb, self.params.problemType, self.params.pathEmpirical, self.params.itermax, self.params.xConvCrit, self.params.yConvCrit, self.params.makePlots, self.params.optStrategy, self.params.fWeight, self.params.fCritical, self.params.countryList, analyzeResults, nlc, plot3dTable, combOverviews ], kwargs) elif self.params.problemType == 'one4eachPair': for ctryIndex in ctryIndices: Ctry1 = countryList[ctryIndex[0]] Ctry2 = countryList[ctryIndex[1]] logger.info(Ctry1 + Ctry2) jobname = Ctry1 + '-' + Ctry2 # set stage dir. path_to_stage_dir = self.make_directory_path(self.params.output, jobname) gc3libs.utils.mkdir(path_to_stage_dir) #path_to_stage_dir = os.path.join(iterationFolder, jobname) # Get moments table from empirical analysis gdpTable = tableDict.fromTextFile(fileIn = os.path.join(self.params.pathEmpirical, 'outputInput/momentTable/Gdp/gdpMoments.csv'), delim = ',', width = 20) # Set Ctry information for this run. update_parameter_in_file(os.path.join(baseDir, 'input/markovA.in'), 'Ctry', 0, Ctry1, 'space-separated') update_parameter_in_file(os.path.join(baseDir, 'input/markovB.in'), 'Ctry', 0, Ctry2, 'space-separated') # Get the correct Ctry Paras into base dir. self.getCtryParas(baseDir, Ctry1, Ctry2) # Copy base dir ctryBaseDir = os.path.join(path_to_stage_dir, 'base') try: shutil.copytree(baseDir, ctryBaseDir) except: logger.info('%s already exists' % baseDir) EA = getParameter(fileIn = os.path.join(baseDir, 'input/parameters.in'), varIn = 'EA', regexIn = 'bar-separated') EB = getParameter(fileIn = os.path.join(baseDir, 'input/parameters.in'), varIn = 'EB', regexIn = 'bar-separated') sigmaA = getParameter(fileIn = os.path.join(baseDir, 'input/parameters.in'), varIn = 'sigmaA', regexIn = 'bar-separated') sigmaB = getParameter(fileIn = os.path.join(baseDir, 'input/parameters.in'), varIn = 'sigmaB', regexIn = 'bar-separated') # Pass ctry information to nlc analyzeResults = anaOne4eachPair nlc = nlcOne4eachPair(gdpTable = gdpTable, ctryPair = [Ctry1, Ctry2], domain = domain, logFile = os.path.join(path_to_stage_dir, 'nlc.log')) combOverviews = combineOverviews.combineOverviews(overviewSimuFile = 'overviewSimu', tableName = 'agTable', sortKeys = ['normDev']) plot3dTable = combineOverviews.plotTable(tablePath =os.path.join(path_to_stage_dir, 'agTable'), savePath = os.path.join(path_to_stage_dir, 'scatter3d')) plot3dTable.columnNames = ['EA', 'sigmaA', 'normDev'] executable = os.path.basename(self.params.executable) kwargs = extra.copy() kwargs['output_dir'] = path_to_stage_dir # yield job yield (jobname, gParaSearchDriver, [ self.params.executable, path_to_stage_dir, self.params.architecture, ctryBaseDir, self.params.xVars, self.params.initialPop, self.params.nPopulation, domain, self.params.solverVerb, self.params.problemType, self.params.pathEmpirical, self.params.itermax, self.params.xConvCrit, self.params.yConvCrit, self.params.makePlots, self.params.optStrategy, self.params.fWeight, self.params.fCritical, self.params.countryList, analyzeResults, nlc, plot3dTable, combOverviews ], kwargs) elif self.params.problemType == 'one4eachCtry': gdpTable = tableDict.fromTextFile(fileIn = os.path.join(self.params.pathEmpirical, 'outputInput/momentTable/Gdp/gdpMoments.csv'), delim = ',', width = 20) if len(countryList) > len(self.params.xVars.split()) / 2 and len(xVars.split()) == 2: if self.params.xVars[-1:] != " ": xVars = ( self.params.xVars + ' ' ) * len(countryList) xVars = xVars[:-1] else: xVars = self.params.xVars * len(countryList) if self.params.xVarsDom[-1:] != " ": xVarsDom = ( self.params.xVarsDom + ' ' ) * len(countryList) xVarsDom = xVarsDom[:-1].split() else: xVarsDom = self.params.xVarsDom * len(countryList) xVarsDom = xVarsDom.split() else: xVars = self.params.xVars xVarsDom = self.params.xVarsDom[:-1].split() jobname = 'one4eachCtry' lowerBds = np.array([xVarsDom[i] for i in range(len(xVarsDom)) if i % 2 == 0], dtype = 'float64') upperBds = np.array([xVarsDom[i] for i in range(len(xVarsDom)) if i % 2 == 1], dtype = 'float64') domain = zip(lowerBds, upperBds) norm = self.params.norm try: norm = int(norm) except ValueError: if norm == "np.inf" or norm == "inf": norm = np.inf else: pass # in particular custom norms logger.info('using norm %s' % (norm)) path_to_stage_dir = os.getcwd() executable = os.path.basename(self.params.executable) analyzeResults = anaOne4eachCtry(countryList, len(list(ctryIndices)), norm = norm) nlc = nlcOne4eachCtry(gdpTable = gdpTable, ctryList = countryList, domain = domain, logFile = os.path.join(path_to_stage_dir, 'nlc.log')) combOverviews = combineOverviews.combineOverviews(overviewSimuFile = 'eSigmaTable', tableName = 'ag_eSigmaTable', sortKeys = ['norm']) deKenPrice.plotPopulation = plotPopOne4eachCtry(countryList) ## # Set solver variables ## nXvars = len(xVars.split()) ## deKenPrice.I_NP = int(self.params.nPopulation) ## deKenPrice.F_weight = float(self.params.fWeight) ## deKenPrice.F_CR = float(self.params.fCritical) ## deKenPrice.I_D = int(nXvars) ## deKenPrice.lowerBds = np.array([ element[0] for element in domain ], dtype = 'float64') ## deKenPrice.upperBds = np.array([ element[1] for element in domain ], dtype = 'float64') ## deKenPrice.I_itermax = int(self.params.itermax) ## deKenPrice.F_VTR = float(self.params.yConvCrit) ## deKenPrice.I_strategy = int(self.params.optStrategy) ## deKenPrice.I_plotting = int(self.params.makePlots) ## deKenPrice.xConvCrit = float(self.params.xConvCrit) ## deKenPrice.workingDir = path_to_stage_dir ## deKenPrice.verbosity = self.params.solverVerb plot3dTable = emptyFun # plot3dTable = combineOverviews.plotTable(tablePath =os.path.join(path_to_stage_dir, 'ag_eSigmaTable'), savePath = os.path.join(path_to_stage_dir, 'scatter3d')) # plot3dTable.columnNames = ['E', 'sigma', 'norm'] for ctryIndex in ctryIndices: Ctry1 = countryList[ctryIndex[0]] Ctry2 = countryList[ctryIndex[1]] # Set Ctry information for this run. update_parameter_in_file(os.path.join(baseDir, 'input/markovA.in'), 'Ctry', 0, Ctry1, 'space-separated') update_parameter_in_file(os.path.join(baseDir, 'input/markovB.in'), 'Ctry', 0, Ctry2, 'space-separated') # Get the correct Ctry Paras into base dir. self.getCtryParas(baseDir, Ctry1, Ctry2) # Copy base dir ctryBaseDir = os.path.join(path_to_stage_dir, 'base' + Ctry1 + Ctry2) try: shutil.copytree(baseDir, ctryBaseDir) except: logger.info('%s already exists' % baseDir) kwargs = extra.copy() kwargs['output_dir'] = path_to_stage_dir # yield job yield (jobname, gParaSearchDriver, [ self.params.executable, path_to_stage_dir, self.params.architecture, baseDir, xVars, self.params.initialPop, self.params.nPopulation, domain, self.params.solverVerb, self.params.problemType, self.params.pathEmpirical, self.params.itermax, self.params.xConvCrit, self.params.yConvCrit, self.params.makePlots, self.params.optStrategy, self.params.fWeight, self.params.fCritical, self.params.countryList, analyzeResults, nlc, plot3dTable, combOverviews ], kwargs) # Set solver variables nXvars = len(xVars.split()) deKenPrice.I_NP = int(self.params.nPopulation) deKenPrice.F_weight = float(self.params.fWeight) deKenPrice.F_CR = float(self.params.fCritical) deKenPrice.I_D = int(nXvars) if self.params.problemType == 'one4eachCtry': deKenPrice.lowerBds = np.array([xVarsDom[i] for i in range(len(xVarsDom)) if i % 2 == 0], dtype = 'float64') deKenPrice.upperBds = np.array([xVarsDom[i] for i in range(len(xVarsDom)) if i % 2 == 1], dtype = 'float64') else: deKenPrice.lowerBds = np.array([ element[0] for element in domain ], dtype = 'float64') deKenPrice.upperBds = np.array([ element[1] for element in domain ], dtype = 'float64') deKenPrice.I_itermax = int(self.params.itermax) deKenPrice.F_VTR = float(self.params.yConvCrit) deKenPrice.I_strategy = int(self.params.optStrategy) deKenPrice.I_plotting = int(self.params.makePlots) deKenPrice.xConvCrit = float(self.params.xConvCrit) deKenPrice.workingDir = path_to_stage_dir deKenPrice.verbosity = self.params.solverVerb logger.info('done with new_tasks')
def new_tasks(self, extra): # Generating new tasks for both paramter files self.logger.info('\ngenParamters.in: ') self.logger.info('-----------------') catFile(os.path.join(self.params.initial, 'input', 'genParameters.in')) self.logger.info('\nctryParameters.in: ') self.logger.info('-----------------') catFile(os.path.join(self.params.initial, 'input', 'ctryParameters.in')) self.logger.info('\npara.loop: ') self.logger.info('-----------------') catFile(os.path.join(os.getcwd(), 'para.loop')) self.logger.info( '\nPress [y] to confirm the input files and continue execution of ghousing.py. Press [q] to exit' ) #import select #rlist, wlist, xlist = select.select([sys.stdin], [], [], None) selection = raw_input() if selection.lower() == 'q': self.logger.critical('Exiting upon user request...') os._exit(1) # setup AppPot parameters use_apppot = False #bug = use_apppot[0] apppot_img = None apppot_changes = None if self.params.apppot: use_apppot = True if self.params.apppot.endswith('.changes.tar.gz'): apppot_changes = self.params.apppot else: apppot_img = self.params.apppot inputs = self._search_for_input_files(self.params.args) # create a tar.gz archive of the code import tarfile tar = tarfile.open(os.path.join(os.getcwd(), 'codeBase.tar.gz'), "w:gz") for name in [ self.params.initial, os.path.join(self.params.initial, '../code') ]: tar.add(name) tar.close() #codeBaseFolder = os.path.join(os.getcwd(), 'codeBase/') #codeFolder = os.path.join(self.params.initial, '../code/') #shutil.copytree(self.params.initial, os.path.join(codeBaseFolder, 'base')) #shutil.copytree(codeFolder, os.path.join(codeBaseFolder, 'code')) # Copy base dir localBaseDir = os.path.join(os.getcwd(), 'localBaseDir') gc3libs.utils.copytree(self.params.initial, localBaseDir) # update ctry Parameters. Important, before I do the para.loop adjustments ctryInParaLoop = False for para_loop in inputs: if os.path.isdir(para_loop): para_loop = os.path.join(para_loop, 'para.loop') paraLoopFile = open(para_loop, 'r') paraLoopFile.readline() for line in paraLoopFile: if not line.rstrip(): continue eles = line.split() var = eles[0] val = eles[6] if var == 'ctry': ctryInParaLoop = True ctry = val localBaseDirInputFolder = os.path.join(localBaseDir, 'input') genParametersFile = os.path.join(localBaseDirInputFolder, 'genParameters.in') ctryParametersFile = os.path.join(localBaseDirInputFolder, 'ctryParameters.in') updateCtryParametersFile = bool( getParameter(genParametersFile, 'updateCtryParametersFile', 'space-separated')) if not ctryInParaLoop: ctry = getParameter(genParametersFile, 'ctry', 'space-separated') if updateCtryParametersFile: shutil.copy( os.path.join(localBaseDirInputFolder, ctry + 'CtryParameters.in'), os.path.join(localBaseDirInputFolder, 'ctryParameters.in')) for para_loop in inputs: path_to_base_dir = os.path.dirname(para_loop) # self.log.debug("Processing loop file '%s' ...", para_loop) for jobname, substs in self.process_para_file(para_loop): ## self.log.debug("Job '%s' defined by substitutions: %s.", ## jobname, substs) executable = os.path.basename(self.params.executable) inputs = {self.params.executable: executable} # make a "stage" directory where input files are collected path_to_stage_dir = self.make_directory_path( self.params.output, jobname) input_dir = path_to_stage_dir #os.path.join(path_to_stage_dir, 'input') gc3libs.utils.mkdir(input_dir) prefix_len = len(input_dir) + 1 # 2. apply substitutions to parameter files for (path, changes) in substs.iteritems(): for (var, val, index, regex) in changes: update_parameter_in_file( os.path.join(localBaseDir, path), var, index, val, regex) fillInputDir(localBaseDir, input_dir) # 3. build input file list for dirpath, dirnames, filenames in os.walk(input_dir): for filename in filenames: # cut the leading part, which is == to path_to_stage_dir relpath = dirpath[prefix_len:] # ignore output directory contents in resubmission if relpath.startswith('output'): continue remote_path = os.path.join(relpath, filename) inputs[os.path.join(dirpath, filename)] = remote_path # all contents of the `output` directory are to be fetched outputs = {'output/': ''} kwargs = extra.copy() kwargs['stdout'] = os.path.join('housingStdOut.log') kwargs['join'] = True kwargs['output_dir'] = os.path.join(path_to_stage_dir, 'output') kwargs['requested_architecture'] = self.params.architecture # print 'inputs = %s' % inputs # print 'outputs = %s' % outputs # kwargs.setdefault('tags', [ ]) # adaptions for uml if self.params.rte: kwargs['apppot_tag'] = 'ENV/APPPOT-0.26' # kwargs['tags'] = ['TEST/APPPOT-IBF-1.0'] # kwargs['tags'] = ['TEST/APPPOT-IBF-1.1'] kwargs['tags'] = ['APPS/ECON/APPPOT-IBF-1.0'] cls = housingApppotApplication pathToExecutable = '/home/user/job/' + executable elif use_apppot: if apppot_img is not None: kwargs['apppot_img'] = apppot_img if apppot_changes is not None: kwargs['apppot_changes'] = apppot_changes cls = housingApppotApplication pathToExecutable = '/home/user/job/' + executable else: cls = housingApplication pathToExecutable = executable # print 'kwargs = %s' % kwargs # hand over job to create yield (jobname, cls, [pathToExecutable, [], inputs, outputs], kwargs)
def new_tasks(self, extra): # setup AppPot parameters use_apppot = False apppot_img = None apppot_changes = None if self.params.apppot: use_apppot = True if self.params.apppot.endswith('.changes.tar.gz'): apppot_changes = self.params.apppot else: apppot_img = self.params.apppot inputs = self._search_for_input_files(self.params.args) # Copy base dir localBaseDir = os.path.join(os.getcwd(), 'localBaseDir') # gc3libs.utils.copytree(self.params.initial, '/mnt/shareOffice/ForwardPremium/Results/sensitivity/wGridSize/dfs') gc3libs.utils.copytree(self.params.initial, localBaseDir) for path in inputs: para_loop = path path_to_base_dir = os.path.dirname(para_loop) # self.log.debug("Processing loop file '%s' ...", para_loop) for jobname, substs in self.process_para_file(para_loop): ## self.log.debug("Job '%s' defined by substitutions: %s.", ## jobname, substs) executable = os.path.basename(self.params.executable) inputs = { self.params.executable:executable } # make a "stage" directory where input files are collected path_to_stage_dir = self.make_directory_path( self.params.output, jobname) input_dir = path_to_stage_dir #os.path.join(path_to_stage_dir, 'input') gc3libs.utils.mkdir(input_dir) prefix_len = len(input_dir) + 1 # 2. apply substitutions to parameter files for (path, changes) in substs.iteritems(): for (var, val, index, regex) in changes: update_parameter_in_file(os.path.join(localBaseDir, path), var, index, val, regex) fillInputDir(localBaseDir, input_dir) # 3. build input file list for dirpath,dirnames,filenames in os.walk(input_dir): for filename in filenames: # cut the leading part, which is == to path_to_stage_dir relpath = dirpath[prefix_len:] # ignore output directory contents in resubmission if relpath. startswith('output'): continue remote_path = os.path.join(relpath, filename) inputs[os.path.join(dirpath, filename)] = remote_path # all contents of the `output` directory are to be fetched outputs = { 'output/':'' } kwargs = extra.copy() kwargs['stdout'] = 'idRisk.log' kwargs['join'] = True kwargs['output_dir'] = os.path.join(path_to_stage_dir, 'output') kwargs['requested_architecture'] = self.params.architecture print 'kwargs = %s' % kwargs print 'inputs = %s' % inputs print 'outputs = %s' % outputs # adaptions for uml if use_apppot: if apppot_img is not None: kwargs['apppot_img'] = apppot_img if apppot_changes is not None: kwargs['apppot_changes'] = apppot_changes cls = idRiskApppotApplication else: cls = idRiskApplication #kwargs['apppot_img'] = '/home/benjamin/workspace/apppot0+ben.disk.img' #kwargs['linux'] = '/home/benjamin/workspace/kernel64-3.0.8' #kwargs['uml'] = '/home/benjamin/workspace/kernel64-3.0.8' #kwargs['tags'] = [] kwargs.setdefault('tags', [ ]) # hand over job to create yield (jobname, cls, ['/home/user/job/' + executable, [], inputs, outputs], kwargs)
def new_tasks(self, extra): # Generating new tasks for both paramter files self.logger.info('\ngenParamters.in: ') self.logger.info('-----------------') catFile(os.path.join(self.params.initial, 'input', 'genParameters.in')) self.logger.info('\nctryParameters.in: ') self.logger.info('-----------------') catFile(os.path.join(self.params.initial, 'input', 'ctryParameters.in')) self.logger.info('\npara.loop: ') self.logger.info('-----------------') catFile(os.path.join(os.getcwd(), 'para.loop')) self.logger.info('\nPress [y] to confirm the input files and continue execution of ghousing.py. Press [q] to exit') #import select #rlist, wlist, xlist = select.select([sys.stdin], [], [], None) selection = raw_input() if selection.lower() == 'q': self.logger.critical('Exiting upon user request...') os._exit(1) # setup AppPot parameters use_apppot = False #bug = use_apppot[0] apppot_img = None apppot_changes = None if self.params.apppot: use_apppot = True if self.params.apppot.endswith('.changes.tar.gz'): apppot_changes = self.params.apppot else: apppot_img = self.params.apppot inputs = self._search_for_input_files(self.params.args) # create a tar.gz archive of the code import tarfile tar = tarfile.open(os.path.join(os.getcwd(), 'codeBase.tar.gz'), "w:gz") for name in [self.params.initial, os.path.join(self.params.initial, '../code')]: tar.add(name) tar.close() #codeBaseFolder = os.path.join(os.getcwd(), 'codeBase/') #codeFolder = os.path.join(self.params.initial, '../code/') #shutil.copytree(self.params.initial, os.path.join(codeBaseFolder, 'base')) #shutil.copytree(codeFolder, os.path.join(codeBaseFolder, 'code')) # Copy base dir localBaseDir = os.path.join(os.getcwd(), 'localBaseDir') gc3libs.utils.copytree(self.params.initial, localBaseDir) # update ctry Parameters. Important, before I do the para.loop adjustments ctryInParaLoop = False for para_loop in inputs: if os.path.isdir(para_loop): para_loop = os.path.join(para_loop, 'para.loop') paraLoopFile = open(para_loop, 'r') paraLoopFile.readline() for line in paraLoopFile: if not line.rstrip(): continue eles = line.split() var = eles[0] val = eles[6] if var == 'ctry': ctryInParaLoop = True ctry = val localBaseDirInputFolder = os.path.join(localBaseDir, 'input') genParametersFile = os.path.join(localBaseDirInputFolder, 'genParameters.in') ctryParametersFile = os.path.join(localBaseDirInputFolder, 'ctryParameters.in') updateCtryParametersFile = bool(getParameter(genParametersFile, 'updateCtryParametersFile', 'space-separated')) if not ctryInParaLoop: ctry = getParameter(genParametersFile, 'ctry', 'space-separated') if updateCtryParametersFile: shutil.copy(os.path.join(localBaseDirInputFolder, ctry + 'CtryParameters.in'), os.path.join(localBaseDirInputFolder, 'ctryParameters.in')) for para_loop in inputs: path_to_base_dir = os.path.dirname(para_loop) # self.log.debug("Processing loop file '%s' ...", para_loop) for jobname, substs in self.process_para_file(para_loop): ## self.log.debug("Job '%s' defined by substitutions: %s.", ## jobname, substs) executable = os.path.basename(self.params.executable) inputs = { self.params.executable:executable } # make a "stage" directory where input files are collected path_to_stage_dir = self.make_directory_path(self.params.output, jobname) input_dir = path_to_stage_dir #os.path.join(path_to_stage_dir, 'input') gc3libs.utils.mkdir(input_dir) prefix_len = len(input_dir) + 1 # 2. apply substitutions to parameter files for (path, changes) in substs.iteritems(): for (var, val, index, regex) in changes: update_parameter_in_file(os.path.join(localBaseDir, path), var, index, val, regex) fillInputDir(localBaseDir, input_dir) # 3. build input file list for dirpath,dirnames,filenames in os.walk(input_dir): for filename in filenames: # cut the leading part, which is == to path_to_stage_dir relpath = dirpath[prefix_len:] # ignore output directory contents in resubmission if relpath. startswith('output'): continue remote_path = os.path.join(relpath, filename) inputs[os.path.join(dirpath, filename)] = remote_path # all contents of the `output` directory are to be fetched outputs = { 'output/':'' } kwargs = extra.copy() kwargs['stdout'] = os.path.join('housingStdOut.log') kwargs['join'] = True kwargs['output_dir'] = os.path.join(path_to_stage_dir, 'output') kwargs['requested_architecture'] = self.params.architecture # print 'inputs = %s' % inputs # print 'outputs = %s' % outputs # kwargs.setdefault('tags', [ ]) # adaptions for uml if self.params.rte: kwargs['apppot_tag'] = 'ENV/APPPOT-0.26' # kwargs['tags'] = ['TEST/APPPOT-IBF-1.0'] # kwargs['tags'] = ['TEST/APPPOT-IBF-1.1'] kwargs['tags'] = ['APPS/ECON/APPPOT-IBF-1.0'] cls = housingApppotApplication pathToExecutable = '/home/user/job/' + executable elif use_apppot: if apppot_img is not None: kwargs['apppot_img'] = apppot_img if apppot_changes is not None: kwargs['apppot_changes'] = apppot_changes cls = housingApppotApplication pathToExecutable = '/home/user/job/' + executable else: cls = housingApplication pathToExecutable = executable # print 'kwargs = %s' % kwargs # hand over job to create yield (jobname, cls, [pathToExecutable, [], inputs, outputs], kwargs)
def new_tasks(self, extra): # setup AppPot parameters use_apppot = False apppot_img = None apppot_changes = None if self.params.apppot: use_apppot = True if self.params.apppot.endswith('.changes.tar.gz'): apppot_changes = self.params.apppot else: apppot_img = self.params.apppot inputs = self._search_for_input_files(self.params.args) # Copy base dir localBaseDir = os.path.join(os.getcwd(), 'localBaseDir') # gc3libs.utils.copytree(self.params.initial, '/mnt/shareOffice/ForwardPremium/Results/sensitivity/wGridSize/dfs') shutil.copytree(self.params.initial, localBaseDir) for path in inputs: para_loop = path path_to_base_dir = os.path.dirname(para_loop) # self.log.debug("Processing loop file '%s' ...", para_loop) for jobname, substs in self.process_para_file(para_loop): ## self.log.debug("Job '%s' defined by substitutions: %s.", ## jobname, substs) executable = os.path.basename(self.params.executable) inputs = {self.params.executable: executable} # make a "stage" directory where input files are collected path_to_stage_dir = self.make_directory_path( self.params.output, jobname) input_dir = path_to_stage_dir #os.path.join(path_to_stage_dir, 'input') gc3libs.utils.mkdir(input_dir) prefix_len = len(input_dir) + 1 # 2. apply substitutions to parameter files for (path, changes) in substs.iteritems(): for (var, val, index, regex) in changes: update_parameter_in_file( os.path.join(localBaseDir, path), var, index, val, regex) fillInputDir(localBaseDir, input_dir) # 3. build input file list for dirpath, dirnames, filenames in os.walk(input_dir): for filename in filenames: # cut the leading part, which is == to path_to_stage_dir relpath = dirpath[prefix_len:] # ignore output directory contents in resubmission if relpath.startswith('output'): continue remote_path = os.path.join(relpath, filename) inputs[os.path.join(dirpath, filename)] = remote_path # all contents of the `output` directory are to be fetched outputs = {'output/': ''} kwargs = extra.copy() kwargs['stdout'] = 'idRisk.log' kwargs['join'] = True kwargs['output_dir'] = os.path.join(path_to_stage_dir, 'output') kwargs['requested_architecture'] = self.params.architecture print 'kwargs = %s' % kwargs print 'inputs = %s' % inputs print 'outputs = %s' % outputs # adaptions for uml if use_apppot: if apppot_img is not None: kwargs['apppot_img'] = apppot_img if apppot_changes is not None: kwargs['apppot_changes'] = apppot_changes cls = idRiskApppotApplication else: cls = idRiskApplication #kwargs['apppot_img'] = '/home/benjamin/workspace/apppot0+ben.disk.img' #kwargs['linux'] = '/home/benjamin/workspace/kernel64-3.0.8' #kwargs['uml'] = '/home/benjamin/workspace/kernel64-3.0.8' #kwargs['tags'] = [] kwargs.setdefault('tags', []) # hand over job to create yield (jobname, cls, ['/home/user/job/' + executable, [], inputs, outputs], kwargs)