def new_tasks(self, extra):
logger.info('entering new_tasks')
# Adjust logging level for stream handler. Could be extracted into a general function.
for handler in logger.handlers:
if not isinstance(handler, logbook.FileHandler):
logger.handlers.remove(handler)
mySH = logbook.StreamHandler(stream = sys.stdout, level = self.params.solverVerb.upper(), format_string = '{record.message}', bubble = True)
logger.handlers.append(mySH)
baseDir = self.params.initial
xVars = self.params.xVars
countryList = self.params.countryList.split()
ctryIndices = getIndex(base = [len(countryList), len(countryList)], restr = 'lowerTr')
for ctryIndex in ctryIndices:
logger.info(countryList[ctryIndex[0]] + countryList[ctryIndex[1]])
# Compute domain
xVarsDom = self.params.xVarsDom.split()
lowerBds = np.array([xVarsDom[i] for i in range(len(xVarsDom)) if i % 2 == 0], dtype = 'float64')
upperBds = np.array([xVarsDom[i] for i in range(len(xVarsDom)) if i % 2 == 1], dtype = 'float64')
domain = zip(lowerBds, upperBds)
# Make problem type specific adjustments.
if self.params.problemType == 'one4all':
gdpTable = tableDict.fromTextFile(fileIn = os.path.join(self.params.pathEmpirical, 'outputInput/momentTable/Gdp/gdpMoments.csv'),
delim = ',', width = 20)
jobname = 'one4all'
logger.info('%s' % self.params.norm)
norm = self.params.norm
try:
norm = int(norm)
except ValueError:
if norm == "np.inf" or norm == "inf":
norm = np.inf
else:
pass # in particular custom norms
logger.info('using norm %s' % (norm))
path_to_stage_dir = os.getcwd()
executable = os.path.basename(self.params.executable)
analyzeResults = anaOne4all(len(list(ctryIndices)), norm = norm)
nlc = nlcOne4all(gdpTable = gdpTable, ctryList = countryList, domain = domain, logFile = os.path.join(path_to_stage_dir, 'nlc.log'))
combOverviews = combineOverviews.combineOverviews(overviewSimuFile = 'eSigmaTable', tableName = 'ag_eSigmaTable', sortKeys = ['norm'])
plot3dTable = combineOverviews.plotTable(tablePath =os.path.join(path_to_stage_dir, 'ag_eSigmaTable'), savePath = os.path.join(path_to_stage_dir, 'scatter3d'))
plot3dTable.columnNames = ['E', 'sigma', 'norm']
for ctryIndex in ctryIndices:
Ctry1 = countryList[ctryIndex[0]]
Ctry2 = countryList[ctryIndex[1]]
# Set Ctry information for this run.
update_parameter_in_file(os.path.join(baseDir, 'input/markovA.in'), 'Ctry',
0, Ctry1, 'space-separated')
update_parameter_in_file(os.path.join(baseDir, 'input/markovB.in'), 'Ctry',
0, Ctry2, 'space-separated')
# Get the correct Ctry Paras into base dir.
self.getCtryParas(baseDir, Ctry1, Ctry2)
# Copy base dir
ctryBaseDir = os.path.join(path_to_stage_dir, 'base' + Ctry1 + Ctry2)
try:
shutil.copytree(baseDir, ctryBaseDir)
except:
logger.info('%s already exists' % baseDir)
kwargs = extra.copy()
kwargs['output_dir'] = path_to_stage_dir
# yield job
yield (jobname, gParaSearchDriver,
[ self.params.executable, path_to_stage_dir, self.params.architecture,
baseDir, self.params.xVars, self.params.initialPop,
self.params.nPopulation, domain, self.params.solverVerb, self.params.problemType,
self.params.pathEmpirical, self.params.itermax, self.params.xConvCrit, self.params.yConvCrit,
self.params.makePlots, self.params.optStrategy, self.params.fWeight, self.params.fCritical, self.params.countryList,
analyzeResults, nlc, plot3dTable, combOverviews
], kwargs)
elif self.params.problemType == 'one4eachPair':
for ctryIndex in ctryIndices:
Ctry1 = countryList[ctryIndex[0]]
Ctry2 = countryList[ctryIndex[1]]
logger.info(Ctry1 + Ctry2)
jobname = Ctry1 + '-' + Ctry2
# set stage dir.
path_to_stage_dir = self.make_directory_path(self.params.output, jobname)
gc3libs.utils.mkdir(path_to_stage_dir)
#path_to_stage_dir = os.path.join(iterationFolder, jobname)
# Get moments table from empirical analysis
gdpTable = tableDict.fromTextFile(fileIn = os.path.join(self.params.pathEmpirical, 'outputInput/momentTable/Gdp/gdpMoments.csv'),
delim = ',', width = 20)
# Set Ctry information for this run.
update_parameter_in_file(os.path.join(baseDir, 'input/markovA.in'), 'Ctry',
0, Ctry1, 'space-separated')
update_parameter_in_file(os.path.join(baseDir, 'input/markovB.in'), 'Ctry',
0, Ctry2, 'space-separated')
# Get the correct Ctry Paras into base dir.
self.getCtryParas(baseDir, Ctry1, Ctry2)
# Copy base dir
ctryBaseDir = os.path.join(path_to_stage_dir, 'base')
try:
shutil.copytree(baseDir, ctryBaseDir)
except:
logger.info('%s already exists' % baseDir)
EA = getParameter(fileIn = os.path.join(baseDir, 'input/parameters.in'), varIn = 'EA',
regexIn = 'bar-separated')
EB = getParameter(fileIn = os.path.join(baseDir, 'input/parameters.in'), varIn = 'EB',
regexIn = 'bar-separated')
sigmaA = getParameter(fileIn = os.path.join(baseDir, 'input/parameters.in'), varIn = 'sigmaA',
regexIn = 'bar-separated')
sigmaB = getParameter(fileIn = os.path.join(baseDir, 'input/parameters.in'), varIn = 'sigmaB',
regexIn = 'bar-separated')
# Pass ctry information to nlc
analyzeResults = anaOne4eachPair
nlc = nlcOne4eachPair(gdpTable = gdpTable, ctryPair = [Ctry1, Ctry2], domain = domain, logFile = os.path.join(path_to_stage_dir, 'nlc.log'))
combOverviews = combineOverviews.combineOverviews(overviewSimuFile = 'overviewSimu', tableName = 'agTable', sortKeys = ['normDev'])
plot3dTable = combineOverviews.plotTable(tablePath =os.path.join(path_to_stage_dir, 'agTable'), savePath = os.path.join(path_to_stage_dir, 'scatter3d'))
plot3dTable.columnNames = ['EA', 'sigmaA', 'normDev']
executable = os.path.basename(self.params.executable)
kwargs = extra.copy()
kwargs['output_dir'] = path_to_stage_dir
# yield job
yield (jobname, gParaSearchDriver,
[ self.params.executable, path_to_stage_dir, self.params.architecture,
ctryBaseDir, self.params.xVars, self.params.initialPop,
self.params.nPopulation, domain, self.params.solverVerb, self.params.problemType,
self.params.pathEmpirical, self.params.itermax, self.params.xConvCrit, self.params.yConvCrit,
self.params.makePlots, self.params.optStrategy, self.params.fWeight, self.params.fCritical, self.params.countryList,
analyzeResults, nlc, plot3dTable, combOverviews
], kwargs)
elif self.params.problemType == 'one4eachCtry':
gdpTable = tableDict.fromTextFile(fileIn = os.path.join(self.params.pathEmpirical, 'outputInput/momentTable/Gdp/gdpMoments.csv'),
delim = ',', width = 20)
if len(countryList) > len(self.params.xVars.split()) / 2 and len(xVars.split()) == 2:
if self.params.xVars[-1:] != " ":
xVars = ( self.params.xVars + ' ' ) * len(countryList)
xVars = xVars[:-1]
else:
xVars = self.params.xVars * len(countryList)
if self.params.xVarsDom[-1:] != " ":
xVarsDom = ( self.params.xVarsDom + ' ' ) * len(countryList)
xVarsDom = xVarsDom[:-1].split()
else:
xVarsDom = self.params.xVarsDom * len(countryList)
xVarsDom = xVarsDom.split()
else:
xVars = self.params.xVars
xVarsDom = self.params.xVarsDom[:-1].split()
jobname = 'one4eachCtry'
lowerBds = np.array([xVarsDom[i] for i in range(len(xVarsDom)) if i % 2 == 0], dtype = 'float64')
upperBds = np.array([xVarsDom[i] for i in range(len(xVarsDom)) if i % 2 == 1], dtype = 'float64')
domain = zip(lowerBds, upperBds)
norm = self.params.norm
try:
norm = int(norm)
except ValueError:
if norm == "np.inf" or norm == "inf":
norm = np.inf
else:
pass # in particular custom norms
logger.info('using norm %s' % (norm))
path_to_stage_dir = os.getcwd()
executable = os.path.basename(self.params.executable)
analyzeResults = anaOne4eachCtry(countryList, len(list(ctryIndices)), norm = norm)
nlc = nlcOne4eachCtry(gdpTable = gdpTable, ctryList = countryList, domain = domain, logFile = os.path.join(path_to_stage_dir, 'nlc.log'))
combOverviews = combineOverviews.combineOverviews(overviewSimuFile = 'eSigmaTable', tableName = 'ag_eSigmaTable', sortKeys = ['norm'])
deKenPrice.plotPopulation = plotPopOne4eachCtry(countryList)
## # Set solver variables
## nXvars = len(xVars.split())
## deKenPrice.I_NP = int(self.params.nPopulation)
## deKenPrice.F_weight = float(self.params.fWeight)
## deKenPrice.F_CR = float(self.params.fCritical)
## deKenPrice.I_D = int(nXvars)
## deKenPrice.lowerBds = np.array([ element[0] for element in domain ], dtype = 'float64')
## deKenPrice.upperBds = np.array([ element[1] for element in domain ], dtype = 'float64')
## deKenPrice.I_itermax = int(self.params.itermax)
## deKenPrice.F_VTR = float(self.params.yConvCrit)
## deKenPrice.I_strategy = int(self.params.optStrategy)
## deKenPrice.I_plotting = int(self.params.makePlots)
## deKenPrice.xConvCrit = float(self.params.xConvCrit)
## deKenPrice.workingDir = path_to_stage_dir
## deKenPrice.verbosity = self.params.solverVerb
plot3dTable = emptyFun
# plot3dTable = combineOverviews.plotTable(tablePath =os.path.join(path_to_stage_dir, 'ag_eSigmaTable'), savePath = os.path.join(path_to_stage_dir, 'scatter3d'))
# plot3dTable.columnNames = ['E', 'sigma', 'norm']
for ctryIndex in ctryIndices:
Ctry1 = countryList[ctryIndex[0]]
Ctry2 = countryList[ctryIndex[1]]
# Set Ctry information for this run.
update_parameter_in_file(os.path.join(baseDir, 'input/markovA.in'), 'Ctry',
0, Ctry1, 'space-separated')
update_parameter_in_file(os.path.join(baseDir, 'input/markovB.in'), 'Ctry',
0, Ctry2, 'space-separated')
# Get the correct Ctry Paras into base dir.
self.getCtryParas(baseDir, Ctry1, Ctry2)
# Copy base dir
ctryBaseDir = os.path.join(path_to_stage_dir, 'base' + Ctry1 + Ctry2)
try:
shutil.copytree(baseDir, ctryBaseDir)
except:
logger.info('%s already exists' % baseDir)
kwargs = extra.copy()
kwargs['output_dir'] = path_to_stage_dir
# yield job
yield (jobname, gParaSearchDriver,
[ self.params.executable, path_to_stage_dir, self.params.architecture,
baseDir, xVars, self.params.initialPop,
self.params.nPopulation, domain, self.params.solverVerb, self.params.problemType,
self.params.pathEmpirical, self.params.itermax, self.params.xConvCrit, self.params.yConvCrit,
self.params.makePlots, self.params.optStrategy, self.params.fWeight, self.params.fCritical, self.params.countryList,
analyzeResults, nlc, plot3dTable, combOverviews
], kwargs)
# Set solver variables
nXvars = len(xVars.split())
deKenPrice.I_NP = int(self.params.nPopulation)
deKenPrice.F_weight = float(self.params.fWeight)
deKenPrice.F_CR = float(self.params.fCritical)
deKenPrice.I_D = int(nXvars)
if self.params.problemType == 'one4eachCtry':
deKenPrice.lowerBds = np.array([xVarsDom[i] for i in range(len(xVarsDom)) if i % 2 == 0], dtype = 'float64')
deKenPrice.upperBds = np.array([xVarsDom[i] for i in range(len(xVarsDom)) if i % 2 == 1], dtype = 'float64')
else:
deKenPrice.lowerBds = np.array([ element[0] for element in domain ], dtype = 'float64')
deKenPrice.upperBds = np.array([ element[1] for element in domain ], dtype = 'float64')
deKenPrice.I_itermax = int(self.params.itermax)
deKenPrice.F_VTR = float(self.params.yConvCrit)
deKenPrice.I_strategy = int(self.params.optStrategy)
deKenPrice.I_plotting = int(self.params.makePlots)
deKenPrice.xConvCrit = float(self.params.xConvCrit)
deKenPrice.workingDir = path_to_stage_dir
deKenPrice.verbosity = self.params.solverVerb
logger.info('done with new_tasks')
def new_tasks(self, extra):
# Generating new tasks for both paramter files
self.logger.info('\ngenParamters.in: ')
self.logger.info('-----------------')
catFile(os.path.join(self.params.initial, 'input', 'genParameters.in'))
self.logger.info('\nctryParameters.in: ')
self.logger.info('-----------------')
catFile(os.path.join(self.params.initial, 'input',
'ctryParameters.in'))
self.logger.info('\npara.loop: ')
self.logger.info('-----------------')
catFile(os.path.join(os.getcwd(), 'para.loop'))
self.logger.info(
'\nPress [y] to confirm the input files and continue execution of ghousing.py. Press [q] to exit'
)
#import select
#rlist, wlist, xlist = select.select([sys.stdin], [], [], None)
selection = raw_input()
if selection.lower() == 'q':
self.logger.critical('Exiting upon user request...')
os._exit(1)
# setup AppPot parameters
use_apppot = False
#bug = use_apppot[0]
apppot_img = None
apppot_changes = None
if self.params.apppot:
use_apppot = True
if self.params.apppot.endswith('.changes.tar.gz'):
apppot_changes = self.params.apppot
else:
apppot_img = self.params.apppot
inputs = self._search_for_input_files(self.params.args)
# create a tar.gz archive of the code
import tarfile
tar = tarfile.open(os.path.join(os.getcwd(), 'codeBase.tar.gz'),
"w:gz")
for name in [
self.params.initial,
os.path.join(self.params.initial, '../code')
]:
tar.add(name)
tar.close()
#codeBaseFolder = os.path.join(os.getcwd(), 'codeBase/')
#codeFolder = os.path.join(self.params.initial, '../code/')
#shutil.copytree(self.params.initial, os.path.join(codeBaseFolder, 'base'))
#shutil.copytree(codeFolder, os.path.join(codeBaseFolder, 'code'))
# Copy base dir
localBaseDir = os.path.join(os.getcwd(), 'localBaseDir')
gc3libs.utils.copytree(self.params.initial, localBaseDir)
# update ctry Parameters. Important, before I do the para.loop adjustments
ctryInParaLoop = False
for para_loop in inputs:
if os.path.isdir(para_loop):
para_loop = os.path.join(para_loop, 'para.loop')
paraLoopFile = open(para_loop, 'r')
paraLoopFile.readline()
for line in paraLoopFile:
if not line.rstrip(): continue
eles = line.split()
var = eles[0]
val = eles[6]
if var == 'ctry':
ctryInParaLoop = True
ctry = val
localBaseDirInputFolder = os.path.join(localBaseDir, 'input')
genParametersFile = os.path.join(localBaseDirInputFolder,
'genParameters.in')
ctryParametersFile = os.path.join(localBaseDirInputFolder,
'ctryParameters.in')
updateCtryParametersFile = bool(
getParameter(genParametersFile, 'updateCtryParametersFile',
'space-separated'))
if not ctryInParaLoop:
ctry = getParameter(genParametersFile, 'ctry', 'space-separated')
if updateCtryParametersFile:
shutil.copy(
os.path.join(localBaseDirInputFolder,
ctry + 'CtryParameters.in'),
os.path.join(localBaseDirInputFolder, 'ctryParameters.in'))
for para_loop in inputs:
path_to_base_dir = os.path.dirname(para_loop)
# self.log.debug("Processing loop file '%s' ...", para_loop)
for jobname, substs in self.process_para_file(para_loop):
## self.log.debug("Job '%s' defined by substitutions: %s.",
## jobname, substs)
executable = os.path.basename(self.params.executable)
inputs = {self.params.executable: executable}
# make a "stage" directory where input files are collected
path_to_stage_dir = self.make_directory_path(
self.params.output, jobname)
input_dir = path_to_stage_dir #os.path.join(path_to_stage_dir, 'input')
gc3libs.utils.mkdir(input_dir)
prefix_len = len(input_dir) + 1
# 2. apply substitutions to parameter files
for (path, changes) in substs.iteritems():
for (var, val, index, regex) in changes:
update_parameter_in_file(
os.path.join(localBaseDir, path), var, index, val,
regex)
fillInputDir(localBaseDir, input_dir)
# 3. build input file list
for dirpath, dirnames, filenames in os.walk(input_dir):
for filename in filenames:
# cut the leading part, which is == to path_to_stage_dir
relpath = dirpath[prefix_len:]
# ignore output directory contents in resubmission
if relpath.startswith('output'):
continue
remote_path = os.path.join(relpath, filename)
inputs[os.path.join(dirpath, filename)] = remote_path
# all contents of the `output` directory are to be fetched
outputs = {'output/': ''}
kwargs = extra.copy()
kwargs['stdout'] = os.path.join('housingStdOut.log')
kwargs['join'] = True
kwargs['output_dir'] = os.path.join(path_to_stage_dir,
'output')
kwargs['requested_architecture'] = self.params.architecture
# print 'inputs = %s' % inputs
# print 'outputs = %s' % outputs
# kwargs.setdefault('tags', [ ])
# adaptions for uml
if self.params.rte:
kwargs['apppot_tag'] = 'ENV/APPPOT-0.26'
# kwargs['tags'] = ['TEST/APPPOT-IBF-1.0']
# kwargs['tags'] = ['TEST/APPPOT-IBF-1.1']
kwargs['tags'] = ['APPS/ECON/APPPOT-IBF-1.0']
cls = housingApppotApplication
pathToExecutable = '/home/user/job/' + executable
elif use_apppot:
if apppot_img is not None:
kwargs['apppot_img'] = apppot_img
if apppot_changes is not None:
kwargs['apppot_changes'] = apppot_changes
cls = housingApppotApplication
pathToExecutable = '/home/user/job/' + executable
else:
cls = housingApplication
pathToExecutable = executable
# print 'kwargs = %s' % kwargs
# hand over job to create
yield (jobname, cls, [pathToExecutable, [], inputs,
outputs], kwargs)
def new_tasks(self, extra):
inputs = self._search_for_input_files(self.params.args)
# Copy base dir
localBaseDir = os.path.join(os.getcwd(), 'localBaseDir')
# gc3libs.utils.copytree(self.params.initial, '/mnt/shareOffice/ForwardPremium/Results/sensitivity/wGridSize/dfs')
gc3libs.utils.copytree(self.params.initial, localBaseDir)
for path in inputs:
para_loop = path
path_to_base_dir = os.path.dirname(para_loop)
# self.log.debug("Processing loop file '%s' ...", para_loop)
for jobname, substs in self.process_para_file(para_loop):
## self.log.debug("Job '%s' defined by substitutions: %s.",
## jobname, substs)
executable = os.path.basename(self.params.executable)
inputs = {self.params.executable: executable}
# make a "stage" directory where input files are collected
path_to_stage_dir = self.make_directory_path(
self.params.output, jobname)
input_dir = path_to_stage_dir #os.path.join(path_to_stage_dir, 'input')
gc3libs.utils.mkdir(input_dir)
prefix_len = len(input_dir) + 1
# Determine if ctry is in parameters
isCtryInParaLoop = False
for (path, changes) in substs.iteritems():
for (var, val, index, regex) in changes:
if var == 'Ctry':
isCtryInParaLoop = True
# 1. files in the "initial" dir are copied verbatim
## if self.params.initial is not None:
## if not isCtryInParaLoop:
## markovA_file_path = os.path.join(self.params.initial, 'input', 'markovA.in')
## markovB_file_path = os.path.join(self.params.initial, 'input', 'markovB.in')
## Ctry1 = getParameter(markovA_file_path, 'Ctry')
## Ctry2 = getParameter(markovB_file_path, 'Ctry')
## self.getCtryParas(self.params.initial, Ctry1, Ctry2)
## self.fillInputDir(self.params.initial, input_dir)
## # gc3libs.utils.copytree(self.params.initial, input_dir)
# 2. apply substitutions to parameter files
for (path, changes) in substs.iteritems():
for (var, val, index, regex) in changes:
update_parameter_in_file(
os.path.join(localBaseDir, path), var, index, val,
regex)
markovA_file_path = os.path.join(localBaseDir, 'input',
'markovA.in')
markovB_file_path = os.path.join(localBaseDir, 'input',
'markovB.in')
Ctry1 = getParameter(markovA_file_path, 'Ctry')
Ctry2 = getParameter(markovB_file_path, 'Ctry')
self.getCtryParas(localBaseDir, Ctry1, Ctry2)
self.fillInputDir(localBaseDir, input_dir)
# 3. build input file list
for dirpath, dirnames, filenames in os.walk(input_dir):
for filename in filenames:
# cut the leading part, which is == to path_to_stage_dir
relpath = dirpath[prefix_len:]
# ignore output directory contents in resubmission
if relpath.startswith('output'):
continue
remote_path = os.path.join(relpath, filename)
inputs[os.path.join(dirpath, filename)] = remote_path
# all contents of the `output` directory are to be fetched
outputs = {'output/': ''}
kwargs = extra.copy()
kwargs['stdout'] = 'forwardPremiumOut.log'
kwargs['join'] = True
kwargs['output_dir'] = os.path.join(path_to_stage_dir,
'output')
kwargs['requested_architecture'] = self.params.architecture
# hand over job to create
yield (jobname, gpremium.GPremiumApplication,
['./' + executable, [], inputs, outputs], kwargs)
def new_tasks(self, extra):
# Generating new tasks for both paramter files
self.logger.info('\ngenParamters.in: ')
self.logger.info('-----------------')
catFile(os.path.join(self.params.initial, 'input', 'genParameters.in'))
self.logger.info('\nctryParameters.in: ')
self.logger.info('-----------------')
catFile(os.path.join(self.params.initial, 'input', 'ctryParameters.in'))
self.logger.info('\npara.loop: ')
self.logger.info('-----------------')
catFile(os.path.join(os.getcwd(), 'para.loop'))
self.logger.info('\nPress [y] to confirm the input files and continue execution of ghousing.py. Press [q] to exit')
#import select
#rlist, wlist, xlist = select.select([sys.stdin], [], [], None)
selection = raw_input()
if selection.lower() == 'q':
self.logger.critical('Exiting upon user request...')
os._exit(1)
# setup AppPot parameters
use_apppot = False
#bug = use_apppot[0]
apppot_img = None
apppot_changes = None
if self.params.apppot:
use_apppot = True
if self.params.apppot.endswith('.changes.tar.gz'):
apppot_changes = self.params.apppot
else:
apppot_img = self.params.apppot
inputs = self._search_for_input_files(self.params.args)
# create a tar.gz archive of the code
import tarfile
tar = tarfile.open(os.path.join(os.getcwd(), 'codeBase.tar.gz'), "w:gz")
for name in [self.params.initial, os.path.join(self.params.initial, '../code')]:
tar.add(name)
tar.close()
#codeBaseFolder = os.path.join(os.getcwd(), 'codeBase/')
#codeFolder = os.path.join(self.params.initial, '../code/')
#shutil.copytree(self.params.initial, os.path.join(codeBaseFolder, 'base'))
#shutil.copytree(codeFolder, os.path.join(codeBaseFolder, 'code'))
# Copy base dir
localBaseDir = os.path.join(os.getcwd(), 'localBaseDir')
gc3libs.utils.copytree(self.params.initial, localBaseDir)
# update ctry Parameters. Important, before I do the para.loop adjustments
ctryInParaLoop = False
for para_loop in inputs:
if os.path.isdir(para_loop):
para_loop = os.path.join(para_loop, 'para.loop')
paraLoopFile = open(para_loop, 'r')
paraLoopFile.readline()
for line in paraLoopFile:
if not line.rstrip(): continue
eles = line.split()
var = eles[0]
val = eles[6]
if var == 'ctry':
ctryInParaLoop = True
ctry = val
localBaseDirInputFolder = os.path.join(localBaseDir, 'input')
genParametersFile = os.path.join(localBaseDirInputFolder, 'genParameters.in')
ctryParametersFile = os.path.join(localBaseDirInputFolder, 'ctryParameters.in')
updateCtryParametersFile = bool(getParameter(genParametersFile, 'updateCtryParametersFile', 'space-separated'))
if not ctryInParaLoop:
ctry = getParameter(genParametersFile, 'ctry', 'space-separated')
if updateCtryParametersFile:
shutil.copy(os.path.join(localBaseDirInputFolder, ctry + 'CtryParameters.in'), os.path.join(localBaseDirInputFolder, 'ctryParameters.in'))
for para_loop in inputs:
path_to_base_dir = os.path.dirname(para_loop)
# self.log.debug("Processing loop file '%s' ...", para_loop)
for jobname, substs in self.process_para_file(para_loop):
## self.log.debug("Job '%s' defined by substitutions: %s.",
## jobname, substs)
executable = os.path.basename(self.params.executable)
inputs = { self.params.executable:executable }
# make a "stage" directory where input files are collected
path_to_stage_dir = self.make_directory_path(self.params.output, jobname)
input_dir = path_to_stage_dir #os.path.join(path_to_stage_dir, 'input')
gc3libs.utils.mkdir(input_dir)
prefix_len = len(input_dir) + 1
# 2. apply substitutions to parameter files
for (path, changes) in substs.iteritems():
for (var, val, index, regex) in changes:
update_parameter_in_file(os.path.join(localBaseDir, path),
var, index, val, regex)
fillInputDir(localBaseDir, input_dir)
# 3. build input file list
for dirpath,dirnames,filenames in os.walk(input_dir):
for filename in filenames:
# cut the leading part, which is == to path_to_stage_dir
relpath = dirpath[prefix_len:]
# ignore output directory contents in resubmission
if relpath. startswith('output'):
continue
remote_path = os.path.join(relpath, filename)
inputs[os.path.join(dirpath, filename)] = remote_path
# all contents of the `output` directory are to be fetched
outputs = { 'output/':'' }
kwargs = extra.copy()
kwargs['stdout'] = os.path.join('housingStdOut.log')
kwargs['join'] = True
kwargs['output_dir'] = os.path.join(path_to_stage_dir, 'output')
kwargs['requested_architecture'] = self.params.architecture
# print 'inputs = %s' % inputs
# print 'outputs = %s' % outputs
# kwargs.setdefault('tags', [ ])
# adaptions for uml
if self.params.rte:
kwargs['apppot_tag'] = 'ENV/APPPOT-0.26'
# kwargs['tags'] = ['TEST/APPPOT-IBF-1.0']
# kwargs['tags'] = ['TEST/APPPOT-IBF-1.1']
kwargs['tags'] = ['APPS/ECON/APPPOT-IBF-1.0']
cls = housingApppotApplication
pathToExecutable = '/home/user/job/' + executable
elif use_apppot:
if apppot_img is not None:
kwargs['apppot_img'] = apppot_img
if apppot_changes is not None:
kwargs['apppot_changes'] = apppot_changes
cls = housingApppotApplication
pathToExecutable = '/home/user/job/' + executable
else:
cls = housingApplication
pathToExecutable = executable
# print 'kwargs = %s' % kwargs
# hand over job to create
yield (jobname, cls, [pathToExecutable, [], inputs, outputs], kwargs)
def new_tasks(self, extra):
inputs = self._search_for_input_files(self.params.args)
# Copy base dir
localBaseDir = os.path.join(os.getcwd(), 'localBaseDir')
# gc3libs.utils.copytree(self.params.initial, '/mnt/shareOffice/ForwardPremium/Results/sensitivity/wGridSize/dfs')
gc3libs.utils.copytree(self.params.initial, localBaseDir)
for path in inputs:
para_loop = path
path_to_base_dir = os.path.dirname(para_loop)
# self.log.debug("Processing loop file '%s' ...", para_loop)
for jobname, substs in self.process_para_file(para_loop):
## self.log.debug("Job '%s' defined by substitutions: %s.",
## jobname, substs)
executable = os.path.basename(self.params.executable)
inputs = { self.params.executable:executable }
# make a "stage" directory where input files are collected
path_to_stage_dir = self.make_directory_path(
self.params.output, jobname)
input_dir = path_to_stage_dir #os.path.join(path_to_stage_dir, 'input')
gc3libs.utils.mkdir(input_dir)
prefix_len = len(input_dir) + 1
# Determine if ctry is in parameters
isCtryInParaLoop = False
for (path, changes) in substs.iteritems():
for (var, val, index, regex) in changes:
if var == 'Ctry':
isCtryInParaLoop = True
# 1. files in the "initial" dir are copied verbatim
## if self.params.initial is not None:
## if not isCtryInParaLoop:
## markovA_file_path = os.path.join(self.params.initial, 'input', 'markovA.in')
## markovB_file_path = os.path.join(self.params.initial, 'input', 'markovB.in')
## Ctry1 = getParameter(markovA_file_path, 'Ctry')
## Ctry2 = getParameter(markovB_file_path, 'Ctry')
## self.getCtryParas(self.params.initial, Ctry1, Ctry2)
## self.fillInputDir(self.params.initial, input_dir)
## # gc3libs.utils.copytree(self.params.initial, input_dir)
# 2. apply substitutions to parameter files
for (path, changes) in substs.iteritems():
for (var, val, index, regex) in changes:
update_parameter_in_file(os.path.join(localBaseDir, path),
var, index, val, regex)
markovA_file_path = os.path.join(localBaseDir, 'input', 'markovA.in')
markovB_file_path = os.path.join(localBaseDir, 'input', 'markovB.in')
Ctry1 = getParameter(markovA_file_path, 'Ctry')
Ctry2 = getParameter(markovB_file_path, 'Ctry')
self.getCtryParas(localBaseDir, Ctry1, Ctry2)
self.fillInputDir(localBaseDir, input_dir)
# 3. build input file list
for dirpath,dirnames,filenames in os.walk(input_dir):
for filename in filenames:
# cut the leading part, which is == to path_to_stage_dir
relpath = dirpath[prefix_len:]
# ignore output directory contents in resubmission
if relpath.startswith('output'):
continue
remote_path = os.path.join(relpath, filename)
inputs[os.path.join(dirpath, filename)] = remote_path
# all contents of the `output` directory are to be fetched
outputs = { 'output/':'' }
kwargs = extra.copy()
kwargs['stdout'] = 'forwardPremiumOut.log'
kwargs['join'] = True
kwargs['output_dir'] = os.path.join(path_to_stage_dir, 'output')
kwargs['requested_architecture'] = self.params.architecture
# hand over job to create
yield (jobname, gpremium.GPremiumApplication,
['./' + executable, [], inputs, outputs], kwargs)
def new_tasks(self, extra):
logger.info('entering new_tasks')
# Adjust logging level for stream handler. Could be extracted into a general function.
for handler in logger.handlers:
if not isinstance(handler, logbook.FileHandler):
logger.handlers.remove(handler)
mySH = logbook.StreamHandler(stream = sys.stdout, level = self.params.solverVerb.upper(), format_string = '{record.message}', bubble = True)
logger.handlers.append(mySH)
baseDir = self.params.initial
xVars = self.params.xVars
countryList = self.params.countryList.split()
ctryIndices = getIndex(base = [len(countryList), len(countryList)], restr = 'lowerTr')
for ctryIndex in ctryIndices:
logger.info(countryList[ctryIndex[0]] + countryList[ctryIndex[1]])
# Compute domain
xVarsDom = self.params.xVarsDom.split()
lowerBds = np.array([xVarsDom[i] for i in range(len(xVarsDom)) if i % 2 == 0], dtype = 'float64')
upperBds = np.array([xVarsDom[i] for i in range(len(xVarsDom)) if i % 2 == 1], dtype = 'float64')
domain = zip(lowerBds, upperBds)
# Make problem type specific adjustments.
if self.params.problemType == 'one4all':
gdpTable = tableDict.fromTextFile(fileIn = os.path.join(self.params.pathEmpirical, 'outputInput/momentTable/Gdp/gdpMoments.csv'),
delim = ',', width = 20)
jobname = 'one4all'
logger.info('%s' % self.params.norm)
norm = self.params.norm
try:
norm = int(norm)
except ValueError:
if norm == "np.inf" or norm == "inf":
norm = np.inf
else:
pass # in particular custom norms
logger.info('using norm %s' % (norm))
path_to_stage_dir = os.getcwd()
executable = os.path.basename(self.params.executable)
analyzeResults = anaOne4all(len(list(ctryIndices)), norm = norm)
nlc = nlcOne4all(gdpTable = gdpTable, ctryList = countryList, domain = domain, logFile = os.path.join(path_to_stage_dir, 'nlc.log'))
combOverviews = combineOverviews.combineOverviews(overviewSimuFile = 'eSigmaTable', tableName = 'ag_eSigmaTable', sortKeys = ['norm'])
plot3dTable = combineOverviews.plotTable(tablePath =os.path.join(path_to_stage_dir, 'ag_eSigmaTable'), savePath = os.path.join(path_to_stage_dir, 'scatter3d'))
plot3dTable.columnNames = ['E', 'sigma', 'norm']
for ctryIndex in ctryIndices:
Ctry1 = countryList[ctryIndex[0]]
Ctry2 = countryList[ctryIndex[1]]
# Set Ctry information for this run.
update_parameter_in_file(os.path.join(baseDir, 'input/markovA.in'), 'Ctry',
0, Ctry1, 'space-separated')
update_parameter_in_file(os.path.join(baseDir, 'input/markovB.in'), 'Ctry',
0, Ctry2, 'space-separated')
# Get the correct Ctry Paras into base dir.
self.getCtryParas(baseDir, Ctry1, Ctry2)
# Copy base dir
ctryBaseDir = os.path.join(path_to_stage_dir, 'base' + Ctry1 + Ctry2)
try:
shutil.copytree(baseDir, ctryBaseDir)
except:
logger.info('%s already exists' % baseDir)
kwargs = extra.copy()
kwargs['output_dir'] = path_to_stage_dir
# yield job
yield (jobname, gParaSearchDriver,
[ self.params.executable, path_to_stage_dir, self.params.architecture,
baseDir, self.params.xVars, self.params.initialPop,
self.params.nPopulation, domain, self.params.solverVerb, self.params.problemType,
self.params.pathEmpirical, self.params.itermax, self.params.xConvCrit, self.params.yConvCrit,
self.params.makePlots, self.params.optStrategy, self.params.fWeight, self.params.fCritical, self.params.countryList,
analyzeResults, nlc, plot3dTable, combOverviews
], kwargs)
elif self.params.problemType == 'one4eachPair':
for ctryIndex in ctryIndices:
Ctry1 = countryList[ctryIndex[0]]
Ctry2 = countryList[ctryIndex[1]]
logger.info(Ctry1 + Ctry2)
jobname = Ctry1 + '-' + Ctry2
# set stage dir.
path_to_stage_dir = self.make_directory_path(self.params.output, jobname)
gc3libs.utils.mkdir(path_to_stage_dir)
#path_to_stage_dir = os.path.join(iterationFolder, jobname)
# Get moments table from empirical analysis
gdpTable = tableDict.fromTextFile(fileIn = os.path.join(self.params.pathEmpirical, 'outputInput/momentTable/Gdp/gdpMoments.csv'),
delim = ',', width = 20)
# Set Ctry information for this run.
update_parameter_in_file(os.path.join(baseDir, 'input/markovA.in'), 'Ctry',
0, Ctry1, 'space-separated')
update_parameter_in_file(os.path.join(baseDir, 'input/markovB.in'), 'Ctry',
0, Ctry2, 'space-separated')
# Get the correct Ctry Paras into base dir.
self.getCtryParas(baseDir, Ctry1, Ctry2)
# Copy base dir
ctryBaseDir = os.path.join(path_to_stage_dir, 'base')
try:
shutil.copytree(baseDir, ctryBaseDir)
except:
logger.info('%s already exists' % baseDir)
EA = getParameter(fileIn = os.path.join(baseDir, 'input/parameters.in'), varIn = 'EA',
regexIn = 'bar-separated')
EB = getParameter(fileIn = os.path.join(baseDir, 'input/parameters.in'), varIn = 'EB',
regexIn = 'bar-separated')
sigmaA = getParameter(fileIn = os.path.join(baseDir, 'input/parameters.in'), varIn = 'sigmaA',
regexIn = 'bar-separated')
sigmaB = getParameter(fileIn = os.path.join(baseDir, 'input/parameters.in'), varIn = 'sigmaB',
regexIn = 'bar-separated')
# Pass ctry information to nlc
analyzeResults = anaOne4eachPair
nlc = nlcOne4eachPair(gdpTable = gdpTable, ctryPair = [Ctry1, Ctry2], domain = domain, logFile = os.path.join(path_to_stage_dir, 'nlc.log'))
combOverviews = combineOverviews.combineOverviews(overviewSimuFile = 'overviewSimu', tableName = 'agTable', sortKeys = ['normDev'])
plot3dTable = combineOverviews.plotTable(tablePath =os.path.join(path_to_stage_dir, 'agTable'), savePath = os.path.join(path_to_stage_dir, 'scatter3d'))
plot3dTable.columnNames = ['EA', 'sigmaA', 'normDev']
executable = os.path.basename(self.params.executable)
kwargs = extra.copy()
kwargs['output_dir'] = path_to_stage_dir
# yield job
yield (jobname, gParaSearchDriver,
[ self.params.executable, path_to_stage_dir, self.params.architecture,
ctryBaseDir, self.params.xVars, self.params.initialPop,
self.params.nPopulation, domain, self.params.solverVerb, self.params.problemType,
self.params.pathEmpirical, self.params.itermax, self.params.xConvCrit, self.params.yConvCrit,
self.params.makePlots, self.params.optStrategy, self.params.fWeight, self.params.fCritical, self.params.countryList,
analyzeResults, nlc, plot3dTable, combOverviews
], kwargs)
elif self.params.problemType == 'one4eachCtry':
gdpTable = tableDict.fromTextFile(fileIn = os.path.join(self.params.pathEmpirical, 'outputInput/momentTable/Gdp/gdpMoments.csv'),
delim = ',', width = 20)
if len(countryList) > len(self.params.xVars.split()) / 2 and len(xVars.split()) == 2:
if self.params.xVars[-1:] != " ":
xVars = ( self.params.xVars + ' ' ) * len(countryList)
xVars = xVars[:-1]
else:
xVars = self.params.xVars * len(countryList)
if self.params.xVarsDom[-1:] != " ":
xVarsDom = ( self.params.xVarsDom + ' ' ) * len(countryList)
xVarsDom = xVarsDom[:-1].split()
else:
xVarsDom = self.params.xVarsDom * len(countryList)
xVarsDom = xVarsDom.split()
else:
xVars = self.params.xVars
xVarsDom = self.params.xVarsDom[:-1].split()
jobname = 'one4eachCtry'
lowerBds = np.array([xVarsDom[i] for i in range(len(xVarsDom)) if i % 2 == 0], dtype = 'float64')
upperBds = np.array([xVarsDom[i] for i in range(len(xVarsDom)) if i % 2 == 1], dtype = 'float64')
domain = zip(lowerBds, upperBds)
norm = self.params.norm
try:
norm = int(norm)
except ValueError:
if norm == "np.inf" or norm == "inf":
norm = np.inf
else:
pass # in particular custom norms
logger.info('using norm %s' % (norm))
path_to_stage_dir = os.getcwd()
executable = os.path.basename(self.params.executable)
analyzeResults = anaOne4eachCtry(countryList, len(list(ctryIndices)), norm = norm)
nlc = nlcOne4eachCtry(gdpTable = gdpTable, ctryList = countryList, domain = domain, logFile = os.path.join(path_to_stage_dir, 'nlc.log'))
combOverviews = combineOverviews.combineOverviews(overviewSimuFile = 'eSigmaTable', tableName = 'ag_eSigmaTable', sortKeys = ['norm'])
deKenPrice.plotPopulation = plotPopOne4eachCtry(countryList)
## # Set solver variables
## nXvars = len(xVars.split())
## deKenPrice.I_NP = int(self.params.nPopulation)
## deKenPrice.F_weight = float(self.params.fWeight)
## deKenPrice.F_CR = float(self.params.fCritical)
## deKenPrice.I_D = int(nXvars)
## deKenPrice.lowerBds = np.array([ element[0] for element in domain ], dtype = 'float64')
## deKenPrice.upperBds = np.array([ element[1] for element in domain ], dtype = 'float64')
## deKenPrice.I_itermax = int(self.params.itermax)
## deKenPrice.F_VTR = float(self.params.yConvCrit)
## deKenPrice.I_strategy = int(self.params.optStrategy)
## deKenPrice.I_plotting = int(self.params.makePlots)
## deKenPrice.xConvCrit = float(self.params.xConvCrit)
## deKenPrice.workingDir = path_to_stage_dir
## deKenPrice.verbosity = self.params.solverVerb
plot3dTable = emptyFun
# plot3dTable = combineOverviews.plotTable(tablePath =os.path.join(path_to_stage_dir, 'ag_eSigmaTable'), savePath = os.path.join(path_to_stage_dir, 'scatter3d'))
# plot3dTable.columnNames = ['E', 'sigma', 'norm']
for ctryIndex in ctryIndices:
Ctry1 = countryList[ctryIndex[0]]
Ctry2 = countryList[ctryIndex[1]]
# Set Ctry information for this run.
update_parameter_in_file(os.path.join(baseDir, 'input/markovA.in'), 'Ctry',
0, Ctry1, 'space-separated')
update_parameter_in_file(os.path.join(baseDir, 'input/markovB.in'), 'Ctry',
0, Ctry2, 'space-separated')
# Get the correct Ctry Paras into base dir.
self.getCtryParas(baseDir, Ctry1, Ctry2)
# Copy base dir
ctryBaseDir = os.path.join(path_to_stage_dir, 'base' + Ctry1 + Ctry2)
try:
shutil.copytree(baseDir, ctryBaseDir)
except:
logger.info('%s already exists' % baseDir)
kwargs = extra.copy()
kwargs['output_dir'] = path_to_stage_dir
# yield job
yield (jobname, gParaSearchDriver,
[ self.params.executable, path_to_stage_dir, self.params.architecture,
baseDir, xVars, self.params.initialPop,
self.params.nPopulation, domain, self.params.solverVerb, self.params.problemType,
self.params.pathEmpirical, self.params.itermax, self.params.xConvCrit, self.params.yConvCrit,
self.params.makePlots, self.params.optStrategy, self.params.fWeight, self.params.fCritical, self.params.countryList,
analyzeResults, nlc, plot3dTable, combOverviews
], kwargs)
# Set solver variables
nXvars = len(xVars.split())
deKenPrice.I_NP = int(self.params.nPopulation)
deKenPrice.F_weight = float(self.params.fWeight)
deKenPrice.F_CR = float(self.params.fCritical)
deKenPrice.I_D = int(nXvars)
if self.params.problemType == 'one4eachCtry':
deKenPrice.lowerBds = np.array([xVarsDom[i] for i in range(len(xVarsDom)) if i % 2 == 0], dtype = 'float64')
deKenPrice.upperBds = np.array([xVarsDom[i] for i in range(len(xVarsDom)) if i % 2 == 1], dtype = 'float64')
else:
deKenPrice.lowerBds = np.array([ element[0] for element in domain ], dtype = 'float64')
deKenPrice.upperBds = np.array([ element[1] for element in domain ], dtype = 'float64')
deKenPrice.I_itermax = int(self.params.itermax)
deKenPrice.F_VTR = float(self.params.yConvCrit)
deKenPrice.I_strategy = int(self.params.optStrategy)
deKenPrice.I_plotting = int(self.params.makePlots)
deKenPrice.xConvCrit = float(self.params.xConvCrit)
deKenPrice.workingDir = path_to_stage_dir
deKenPrice.verbosity = self.params.solverVerb
logger.info('done with new_tasks')
