def combinedOwnerSimuPlot():
import copy
logger.debug('starting combinedOwnerSimuPlot')
folders = [
folder for folder in os.listdir(os.getcwd()) if os.path.isdir(folder)
and not folder == 'localBaseDir' and not folder == 'ghousing.jobs'
]
logger.debug('folders are %s ' % folders)
tableList = [(folder,
os.path.join(os.getcwd(), folder, 'output', 'aggregate.out'))
for folder in folders]
tableDicts = dict([(folder,
tableDict.fromTextFile(
table,
width=np.max([len(folder)
for folder in folders]) + 5,
prec=10)) for folder, table in tableList
if os.path.isfile(table)])
tableKeys = tableDicts.keys()
tableKeys.sort()
if tableDicts:
for ixTable, tableKey in enumerate(tableKeys):
table = copy.deepcopy(tableDicts[tableKey])
table.keep(['age', 'owner'])
table.rename('owner', tableKey)
if ixTable == 0:
fullTable = copy.deepcopy(table)
else:
fullTable.merge(table, 'age')
fullTable.drop('_merge')
logger.info(fullTable)
empOwnershipFile = os.path.join(os.getcwd(), 'localBaseDir', 'input',
'PSIDOwnershipProfilealleduc.out')
empOwnershipTable = tableDict.fromTextFile(empOwnershipFile,
width=20,
prec=10)
empOwnershipTable.rename('PrOwnership', 'empOwnership')
fullTable.merge(empOwnershipTable, 'age')
fullTable.drop('_merge')
if fullTable:
logger.info(fullTable)
f = open(os.path.join(os.getcwd(), 'ownerSimu'), 'w')
print >> f, fullTable
f.flush()
else:
logger.info('no owner simus')
logger.debug('done combinedOwnerSimuPlot')
plotSimulation(table=os.path.join(os.getcwd(), 'ownerSimu'),
xVar='age',
yVars=list(fullTable.cols),
yVarRange=(0., 1.),
figureFile=os.path.join(os.getcwd(), 'ownerSimu.png'),
verb='CRITICAL')
def combinedOwnerSimuPlot():
import copy
logger.debug("starting combinedOwnerSimuPlot")
folders = [
folder
for folder in os.listdir(os.getcwd())
if os.path.isdir(folder) and not folder == "localBaseDir" and not folder == "ghousing.jobs"
]
logger.debug("folders are %s " % folders)
tableList = [(folder, os.path.join(os.getcwd(), folder, "output", "aggregate.out")) for folder in folders]
tableDicts = dict(
[
(folder, tableDict.fromTextFile(table, width=np.max([len(folder) for folder in folders]) + 5, prec=10))
for folder, table in tableList
if os.path.isfile(table)
]
)
tableKeys = tableDicts.keys()
tableKeys.sort()
if tableDicts:
for ixTable, tableKey in enumerate(tableKeys):
table = copy.deepcopy(tableDicts[tableKey])
table.keep(["age", "owner"])
table.rename("owner", tableKey)
if ixTable == 0:
fullTable = copy.deepcopy(table)
else:
fullTable.merge(table, "age")
fullTable.drop("_merge")
logger.info(fullTable)
empOwnershipFile = os.path.join(os.getcwd(), "localBaseDir", "input", "PSIDOwnershipProfilealleduc.out")
empOwnershipTable = tableDict.fromTextFile(empOwnershipFile, width=20, prec=10)
empOwnershipTable.rename("PrOwnership", "empOwnership")
fullTable.merge(empOwnershipTable, "age")
fullTable.drop("_merge")
if fullTable:
logger.info(fullTable)
f = open(os.path.join(os.getcwd(), "ownerSimu"), "w")
print >> f, fullTable
f.flush()
else:
logger.info("no owner simus")
logger.debug("done combinedOwnerSimuPlot")
plotSimulation(
table=os.path.join(os.getcwd(), "ownerSimu"),
xVar="age",
yVars=list(fullTable.cols),
yVarRange=(0.0, 1.0),
figureFile=os.path.join(os.getcwd(), "ownerSimu.png"),
verb="CRITICAL",
)
def combineTables():
print 'start combineTables'
tableList = [
os.path.join(os.getcwd(), folder, 'optimalRun')
for folder in os.listdir(os.getcwd())
if os.path.isdir(folder) and not folder == 'localBaseDir'
and not folder == 'idRiskParaSearch.jobs'
]
logger.info(tableList)
a = [table for table in tableList]
print a
tableDicts = [
tableDict.fromTextFile(table, width=20, prec=10) for table in tableList
if os.path.isfile(table)
]
logger.info('tableDicts=')
logger.info(tableDicts)
logger.info('check if tableDicts empty')
if tableDicts:
optimalRuns = tableDicts[0]
for ixTable, table in enumerate(tableDicts):
if ixTable == 0: continue
optimalRuns = optimalRuns.getAppended(table)
#optimalRuns.order(['dy', 'wBarLower'])
#optimalRuns.sort(['dy'])
logger.info(optimalRuns)
f = open(os.path.join(os.getcwd(), 'optimalRuns'), 'w')
print >> f, optimalRuns
f.flush()
def combineTables():
print "start combineTables"
tableList = [
os.path.join(os.getcwd(), folder, "optimalRun")
for folder in os.listdir(os.getcwd())
if os.path.isdir(folder) and not folder == "localBaseDir" and not folder == "idRiskParaSearch.jobs"
]
logger.info(tableList)
a = [table for table in tableList]
print a
tableDicts = [tableDict.fromTextFile(table, width=20, prec=10) for table in tableList if os.path.isfile(table)]
logger.info("tableDicts=")
logger.info(tableDicts)
logger.info("check if tableDicts empty")
if tableDicts:
optimalRuns = tableDicts[0]
for ixTable, table in enumerate(tableDicts):
if ixTable == 0:
continue
optimalRuns = optimalRuns.getAppended(table)
# optimalRuns.order(['dy', 'wBarLower'])
# optimalRuns.sort(['dy'])
logger.info(optimalRuns)
f = open(os.path.join(os.getcwd(), "optimalRuns"), "w")
print >> f, optimalRuns
f.flush()
def combinedThresholdPlot():
import copy
folders = [folder for folder in os.listdir(os.getcwd()) if os.path.isdir(folder) and not folder == 'localBaseDir' and not folder == 'ghousing.jobs']
tableList = [ (folder, os.path.join(os.getcwd(), folder, 'output', 'ownershipThreshold_1.out')) for folder in folders ]
tableDicts = dict([ (folder, tableDict.fromTextFile(table, width = np.max([len(folder) for folder in folders]) + 5, prec = 10)) for folder, table in tableList if os.path.isfile(table)])
tableKeys = tableDicts.keys()
tableKeys.sort()
if tableDicts:
for ixTable, tableKey in enumerate(tableKeys):
# print tableKey
# print tableDicts[tableKey]
table = copy.deepcopy(tableDicts[tableKey])
table.keep(['age', 'yst1'])
table.rename('yst1', tableKey)
if ixTable == 0:
fullTable = copy.deepcopy(table)
else:
fullTable.merge(table, 'age')
if '_merge' in fullTable.cols:
fullTable.drop('_merge')
logger.info(fullTable)
logger.info(fullTable)
f = open(os.path.join(os.getcwd(), 'ownerThresholds'), 'w')
print >> f, fullTable
f.flush()
ax = 3
plotSimulation(table = os.path.join(os.getcwd(), 'ownerThresholds'), xVar = 'age', yVars = list(fullTable.cols), figureFile = os.path.join(os.getcwd(), 'ownerThresholds.png'), verb = 'CRITICAL' )
def combinedOwnerSimuPlot():
import copy
logger.debug('starting combinedOwnerSimuPlot')
folders = [folder for folder in os.listdir(os.getcwd()) if os.path.isdir(folder) and not folder == 'localBaseDir' and not folder == 'ghousing.jobs']
logger.debug('folders are %s ' % folders)
tableList = [ (folder, os.path.join(os.getcwd(), folder, 'output', 'aggregate.out')) for folder in folders ]
tableDicts = dict([ (folder, tableDict.fromTextFile(table, width = np.max([len(folder) for folder in folders]) + 5, prec = 10)) for folder, table in tableList if os.path.isfile(table)])
tableKeys = tableDicts.keys()
tableKeys.sort()
if tableDicts:
for ixTable, tableKey in enumerate(tableKeys):
table = copy.deepcopy(tableDicts[tableKey])
table.keep(['age', 'owner'])
table.rename('owner', tableKey)
if ixTable == 0:
fullTable = copy.deepcopy(table)
else:
fullTable.merge(table, 'age')
fullTable.drop('_merge')
logger.info(fullTable)
empOwnershipFile = os.path.join(os.getcwd(), 'localBaseDir', 'input', 'PSIDOwnershipProfilealleduc.out')
empOwnershipTable = tableDict.fromTextFile(empOwnershipFile, width = 20, prec = 10)
empOwnershipTable.rename('PrOwnership', 'empOwnership')
fullTable.merge(empOwnershipTable, 'age')
fullTable.drop('_merge')
if fullTable:
logger.info(fullTable)
f = open(os.path.join(os.getcwd(), 'ownerSimu'), 'w')
print >> f, fullTable
f.flush()
else:
logger.info('no owner simus')
logger.debug('done combinedOwnerSimuPlot')
plotSimulation(table = os.path.join(os.getcwd(), 'ownerSimu'), xVar = 'age', yVars = list(fullTable.cols), yVarRange = (0., 1.), figureFile = os.path.join(os.getcwd(), 'ownerSimu.png'), verb = 'CRITICAL' )
def next(self, *args):
self.iter += 1
if self.wBarLower_task.execution.returncode == 13:
logger.critical('wBarLower failed. terminating para combo')
self.execution.returncode = 13
return Run.State.TERMINATED
elif self.wBarLower_task.execution.returncode == 0:
wBarTable = tableDict.fromTextFile(os.path.join(self.paraFolder, 'optimwBarLower', 'overviewSimu'), width = 20, prec = 10)
optimalRunTable = wBarTable.getSubset( np.abs(wBarTable['wBarLower'] - self.wBarLower_task.costlyOptimizer.best_x) < 1.e-7 )
optimalRunFile = open(os.path.join(self.paraFolder, 'optimalRun'), 'w')
print >> optimalRunFile, optimalRunTable
return Run.State.TERMINATED
else:
print 'unknown return code'
os._exit
def next(self, *args):
self.iter += 1
if self.wBarLower_task.execution.returncode == 13:
logger.critical('wBarLower failed. terminating para combo')
self.execution.returncode = 13
return Run.State.TERMINATED
elif self.wBarLower_task.execution.returncode == 0:
wBarTable = tableDict.fromTextFile(os.path.join(self.paraFolder, 'optimwBarLower', 'overviewSimu'), width = 20, prec = 10)
optimalRunTable = wBarTable.getSubset( np.abs(wBarTable['wBarLower'] - self.wBarLower_task.costlyOptimizer.best_x) < 1.e-7 )
optimalRunFile = open(os.path.join(self.paraFolder, 'optimalRun'), 'w')
print >> optimalRunFile, optimalRunTable
self.execution.returncode = 0
return Run.State.TERMINATED
else:
print 'unknown return code'
os._exit
def combinedThresholdPlot():
import copy
folders = [
folder
for folder in os.listdir(os.getcwd())
if os.path.isdir(folder) and not folder == "localBaseDir" and not folder == "ghousing.jobs"
]
tableList = [
(folder, os.path.join(os.getcwd(), folder, "output", "ownershipThreshold_1.out")) for folder in folders
]
tableDicts = dict(
[
(folder, tableDict.fromTextFile(table, width=np.max([len(folder) for folder in folders]) + 5, prec=10))
for folder, table in tableList
if os.path.isfile(table)
]
)
tableKeys = tableDicts.keys()
tableKeys.sort()
if tableDicts:
for ixTable, tableKey in enumerate(tableKeys):
# print tableKey
# print tableDicts[tableKey]
table = copy.deepcopy(tableDicts[tableKey])
table.keep(["age", "yst1"])
table.rename("yst1", tableKey)
if ixTable == 0:
fullTable = copy.deepcopy(table)
else:
fullTable.merge(table, "age")
if "_merge" in fullTable.cols:
fullTable.drop("_merge")
logger.info(fullTable)
logger.info(fullTable)
f = open(os.path.join(os.getcwd(), "ownerThresholds"), "w")
print >> f, fullTable
f.flush()
ax = 3
plotSimulation(
table=os.path.join(os.getcwd(), "ownerThresholds"),
xVar="age",
yVars=list(fullTable.cols),
figureFile=os.path.join(os.getcwd(), "ownerThresholds.png"),
verb="CRITICAL",
)
def combinedThresholdPlot():
import copy
folders = [
folder for folder in os.listdir(os.getcwd()) if os.path.isdir(folder)
and not folder == 'localBaseDir' and not folder == 'ghousing.jobs'
]
tableList = [(folder,
os.path.join(os.getcwd(), folder, 'output',
'ownershipThreshold_1.out'))
for folder in folders]
tableDicts = dict([(folder,
tableDict.fromTextFile(
table,
width=np.max([len(folder)
for folder in folders]) + 5,
prec=10)) for folder, table in tableList
if os.path.isfile(table)])
tableKeys = tableDicts.keys()
tableKeys.sort()
if tableDicts:
for ixTable, tableKey in enumerate(tableKeys):
# print tableKey
# print tableDicts[tableKey]
table = copy.deepcopy(tableDicts[tableKey])
table.keep(['age', 'yst1'])
table.rename('yst1', tableKey)
if ixTable == 0:
fullTable = copy.deepcopy(table)
else:
fullTable.merge(table, 'age')
if '_merge' in fullTable.cols:
fullTable.drop('_merge')
logger.info(fullTable)
logger.info(fullTable)
f = open(os.path.join(os.getcwd(), 'ownerThresholds'), 'w')
print >> f, fullTable
f.flush()
ax = 3
plotSimulation(table=os.path.join(os.getcwd(), 'ownerThresholds'),
xVar='age',
yVars=list(fullTable.cols),
figureFile=os.path.join(os.getcwd(),
'ownerThresholds.png'),
verb='CRITICAL')
def new_tasks(self, extra):
logger.info('entering new_tasks')
# Adjust logging level for stream handler. Could be extracted into a general function.
for handler in logger.handlers:
if not isinstance(handler, logbook.FileHandler):
logger.handlers.remove(handler)
mySH = logbook.StreamHandler(stream = sys.stdout, level = self.params.solverVerb.upper(), format_string = '{record.message}', bubble = True)
logger.handlers.append(mySH)
baseDir = self.params.initial
xVars = self.params.xVars
countryList = self.params.countryList.split()
ctryIndices = getIndex(base = [len(countryList), len(countryList)], restr = 'lowerTr')
for ctryIndex in ctryIndices:
logger.info(countryList[ctryIndex[0]] + countryList[ctryIndex[1]])
# Compute domain
xVarsDom = self.params.xVarsDom.split()
lowerBds = np.array([xVarsDom[i] for i in range(len(xVarsDom)) if i % 2 == 0], dtype = 'float64')
upperBds = np.array([xVarsDom[i] for i in range(len(xVarsDom)) if i % 2 == 1], dtype = 'float64')
domain = zip(lowerBds, upperBds)
# Make problem type specific adjustments.
if self.params.problemType == 'one4all':
gdpTable = tableDict.fromTextFile(fileIn = os.path.join(self.params.pathEmpirical, 'outputInput/momentTable/Gdp/gdpMoments.csv'),
delim = ',', width = 20)
jobname = 'one4all'
logger.info('%s' % self.params.norm)
norm = self.params.norm
try:
norm = int(norm)
except ValueError:
if norm == "np.inf" or norm == "inf":
norm = np.inf
else:
pass # in particular custom norms
logger.info('using norm %s' % (norm))
path_to_stage_dir = os.getcwd()
executable = os.path.basename(self.params.executable)
analyzeResults = anaOne4all(len(list(ctryIndices)), norm = norm)
nlc = nlcOne4all(gdpTable = gdpTable, ctryList = countryList, domain = domain, logFile = os.path.join(path_to_stage_dir, 'nlc.log'))
combOverviews = combineOverviews.combineOverviews(overviewSimuFile = 'eSigmaTable', tableName = 'ag_eSigmaTable', sortKeys = ['norm'])
plot3dTable = combineOverviews.plotTable(tablePath =os.path.join(path_to_stage_dir, 'ag_eSigmaTable'), savePath = os.path.join(path_to_stage_dir, 'scatter3d'))
plot3dTable.columnNames = ['E', 'sigma', 'norm']
for ctryIndex in ctryIndices:
Ctry1 = countryList[ctryIndex[0]]
Ctry2 = countryList[ctryIndex[1]]
# Set Ctry information for this run.
update_parameter_in_file(os.path.join(baseDir, 'input/markovA.in'), 'Ctry',
0, Ctry1, 'space-separated')
update_parameter_in_file(os.path.join(baseDir, 'input/markovB.in'), 'Ctry',
0, Ctry2, 'space-separated')
# Get the correct Ctry Paras into base dir.
self.getCtryParas(baseDir, Ctry1, Ctry2)
# Copy base dir
ctryBaseDir = os.path.join(path_to_stage_dir, 'base' + Ctry1 + Ctry2)
try:
shutil.copytree(baseDir, ctryBaseDir)
except:
logger.info('%s already exists' % baseDir)
kwargs = extra.copy()
kwargs['output_dir'] = path_to_stage_dir
# yield job
yield (jobname, gParaSearchDriver,
[ self.params.executable, path_to_stage_dir, self.params.architecture,
baseDir, self.params.xVars, self.params.initialPop,
self.params.nPopulation, domain, self.params.solverVerb, self.params.problemType,
self.params.pathEmpirical, self.params.itermax, self.params.xConvCrit, self.params.yConvCrit,
self.params.makePlots, self.params.optStrategy, self.params.fWeight, self.params.fCritical, self.params.countryList,
analyzeResults, nlc, plot3dTable, combOverviews
], kwargs)
elif self.params.problemType == 'one4eachPair':
for ctryIndex in ctryIndices:
Ctry1 = countryList[ctryIndex[0]]
Ctry2 = countryList[ctryIndex[1]]
logger.info(Ctry1 + Ctry2)
jobname = Ctry1 + '-' + Ctry2
# set stage dir.
path_to_stage_dir = self.make_directory_path(self.params.output, jobname)
gc3libs.utils.mkdir(path_to_stage_dir)
#path_to_stage_dir = os.path.join(iterationFolder, jobname)
# Get moments table from empirical analysis
gdpTable = tableDict.fromTextFile(fileIn = os.path.join(self.params.pathEmpirical, 'outputInput/momentTable/Gdp/gdpMoments.csv'),
delim = ',', width = 20)
# Set Ctry information for this run.
update_parameter_in_file(os.path.join(baseDir, 'input/markovA.in'), 'Ctry',
0, Ctry1, 'space-separated')
update_parameter_in_file(os.path.join(baseDir, 'input/markovB.in'), 'Ctry',
0, Ctry2, 'space-separated')
# Get the correct Ctry Paras into base dir.
self.getCtryParas(baseDir, Ctry1, Ctry2)
# Copy base dir
ctryBaseDir = os.path.join(path_to_stage_dir, 'base')
try:
shutil.copytree(baseDir, ctryBaseDir)
except:
logger.info('%s already exists' % baseDir)
EA = getParameter(fileIn = os.path.join(baseDir, 'input/parameters.in'), varIn = 'EA',
regexIn = 'bar-separated')
EB = getParameter(fileIn = os.path.join(baseDir, 'input/parameters.in'), varIn = 'EB',
regexIn = 'bar-separated')
sigmaA = getParameter(fileIn = os.path.join(baseDir, 'input/parameters.in'), varIn = 'sigmaA',
regexIn = 'bar-separated')
sigmaB = getParameter(fileIn = os.path.join(baseDir, 'input/parameters.in'), varIn = 'sigmaB',
regexIn = 'bar-separated')
# Pass ctry information to nlc
analyzeResults = anaOne4eachPair
nlc = nlcOne4eachPair(gdpTable = gdpTable, ctryPair = [Ctry1, Ctry2], domain = domain, logFile = os.path.join(path_to_stage_dir, 'nlc.log'))
combOverviews = combineOverviews.combineOverviews(overviewSimuFile = 'overviewSimu', tableName = 'agTable', sortKeys = ['normDev'])
plot3dTable = combineOverviews.plotTable(tablePath =os.path.join(path_to_stage_dir, 'agTable'), savePath = os.path.join(path_to_stage_dir, 'scatter3d'))
plot3dTable.columnNames = ['EA', 'sigmaA', 'normDev']
executable = os.path.basename(self.params.executable)
kwargs = extra.copy()
kwargs['output_dir'] = path_to_stage_dir
# yield job
yield (jobname, gParaSearchDriver,
[ self.params.executable, path_to_stage_dir, self.params.architecture,
ctryBaseDir, self.params.xVars, self.params.initialPop,
self.params.nPopulation, domain, self.params.solverVerb, self.params.problemType,
self.params.pathEmpirical, self.params.itermax, self.params.xConvCrit, self.params.yConvCrit,
self.params.makePlots, self.params.optStrategy, self.params.fWeight, self.params.fCritical, self.params.countryList,
analyzeResults, nlc, plot3dTable, combOverviews
], kwargs)
elif self.params.problemType == 'one4eachCtry':
gdpTable = tableDict.fromTextFile(fileIn = os.path.join(self.params.pathEmpirical, 'outputInput/momentTable/Gdp/gdpMoments.csv'),
delim = ',', width = 20)
if len(countryList) > len(self.params.xVars.split()) / 2 and len(xVars.split()) == 2:
if self.params.xVars[-1:] != " ":
xVars = ( self.params.xVars + ' ' ) * len(countryList)
xVars = xVars[:-1]
else:
xVars = self.params.xVars * len(countryList)
if self.params.xVarsDom[-1:] != " ":
xVarsDom = ( self.params.xVarsDom + ' ' ) * len(countryList)
xVarsDom = xVarsDom[:-1].split()
else:
xVarsDom = self.params.xVarsDom * len(countryList)
xVarsDom = xVarsDom.split()
else:
xVars = self.params.xVars
xVarsDom = self.params.xVarsDom[:-1].split()
jobname = 'one4eachCtry'
lowerBds = np.array([xVarsDom[i] for i in range(len(xVarsDom)) if i % 2 == 0], dtype = 'float64')
upperBds = np.array([xVarsDom[i] for i in range(len(xVarsDom)) if i % 2 == 1], dtype = 'float64')
domain = zip(lowerBds, upperBds)
norm = self.params.norm
try:
norm = int(norm)
except ValueError:
if norm == "np.inf" or norm == "inf":
norm = np.inf
else:
pass # in particular custom norms
logger.info('using norm %s' % (norm))
path_to_stage_dir = os.getcwd()
executable = os.path.basename(self.params.executable)
analyzeResults = anaOne4eachCtry(countryList, len(list(ctryIndices)), norm = norm)
nlc = nlcOne4eachCtry(gdpTable = gdpTable, ctryList = countryList, domain = domain, logFile = os.path.join(path_to_stage_dir, 'nlc.log'))
combOverviews = combineOverviews.combineOverviews(overviewSimuFile = 'eSigmaTable', tableName = 'ag_eSigmaTable', sortKeys = ['norm'])
deKenPrice.plotPopulation = plotPopOne4eachCtry(countryList)
## # Set solver variables
## nXvars = len(xVars.split())
## deKenPrice.I_NP = int(self.params.nPopulation)
## deKenPrice.F_weight = float(self.params.fWeight)
## deKenPrice.F_CR = float(self.params.fCritical)
## deKenPrice.I_D = int(nXvars)
## deKenPrice.lowerBds = np.array([ element[0] for element in domain ], dtype = 'float64')
## deKenPrice.upperBds = np.array([ element[1] for element in domain ], dtype = 'float64')
## deKenPrice.I_itermax = int(self.params.itermax)
## deKenPrice.F_VTR = float(self.params.yConvCrit)
## deKenPrice.I_strategy = int(self.params.optStrategy)
## deKenPrice.I_plotting = int(self.params.makePlots)
## deKenPrice.xConvCrit = float(self.params.xConvCrit)
## deKenPrice.workingDir = path_to_stage_dir
## deKenPrice.verbosity = self.params.solverVerb
plot3dTable = emptyFun
# plot3dTable = combineOverviews.plotTable(tablePath =os.path.join(path_to_stage_dir, 'ag_eSigmaTable'), savePath = os.path.join(path_to_stage_dir, 'scatter3d'))
# plot3dTable.columnNames = ['E', 'sigma', 'norm']
for ctryIndex in ctryIndices:
Ctry1 = countryList[ctryIndex[0]]
Ctry2 = countryList[ctryIndex[1]]
# Set Ctry information for this run.
update_parameter_in_file(os.path.join(baseDir, 'input/markovA.in'), 'Ctry',
0, Ctry1, 'space-separated')
update_parameter_in_file(os.path.join(baseDir, 'input/markovB.in'), 'Ctry',
0, Ctry2, 'space-separated')
# Get the correct Ctry Paras into base dir.
self.getCtryParas(baseDir, Ctry1, Ctry2)
# Copy base dir
ctryBaseDir = os.path.join(path_to_stage_dir, 'base' + Ctry1 + Ctry2)
try:
shutil.copytree(baseDir, ctryBaseDir)
except:
logger.info('%s already exists' % baseDir)
kwargs = extra.copy()
kwargs['output_dir'] = path_to_stage_dir
# yield job
yield (jobname, gParaSearchDriver,
[ self.params.executable, path_to_stage_dir, self.params.architecture,
baseDir, xVars, self.params.initialPop,
self.params.nPopulation, domain, self.params.solverVerb, self.params.problemType,
self.params.pathEmpirical, self.params.itermax, self.params.xConvCrit, self.params.yConvCrit,
self.params.makePlots, self.params.optStrategy, self.params.fWeight, self.params.fCritical, self.params.countryList,
analyzeResults, nlc, plot3dTable, combOverviews
], kwargs)
# Set solver variables
nXvars = len(xVars.split())
deKenPrice.I_NP = int(self.params.nPopulation)
deKenPrice.F_weight = float(self.params.fWeight)
deKenPrice.F_CR = float(self.params.fCritical)
deKenPrice.I_D = int(nXvars)
if self.params.problemType == 'one4eachCtry':
deKenPrice.lowerBds = np.array([xVarsDom[i] for i in range(len(xVarsDom)) if i % 2 == 0], dtype = 'float64')
deKenPrice.upperBds = np.array([xVarsDom[i] for i in range(len(xVarsDom)) if i % 2 == 1], dtype = 'float64')
else:
deKenPrice.lowerBds = np.array([ element[0] for element in domain ], dtype = 'float64')
deKenPrice.upperBds = np.array([ element[1] for element in domain ], dtype = 'float64')
deKenPrice.I_itermax = int(self.params.itermax)
deKenPrice.F_VTR = float(self.params.yConvCrit)
deKenPrice.I_strategy = int(self.params.optStrategy)
deKenPrice.I_plotting = int(self.params.makePlots)
deKenPrice.xConvCrit = float(self.params.xConvCrit)
deKenPrice.workingDir = path_to_stage_dir
deKenPrice.verbosity = self.params.solverVerb
logger.info('done with new_tasks')
def momentPlots(baseName, path, xVar, overlay, conditions = {}, xVarRange = None, figureFile = None, type = 'presentation'):
if os.path.isdir(path):
# if not tableFile:
tableFile = os.path.join(path, 'overviewSimu')
elif os.path.isfile(path):
tableFile = path
else:
logger.critical('path is not a directory or a table file.')
logger.info('\nCreating plot for path %s with conditions %s' % (path, conditions))
# make sure xVar is not conditioned upon -> we want to use it for xaxis.
if xVar in conditions.keys():
conditions.pop(xVar)
# create a file name for emerging plots
moments = ['EP', 'e_rs', 'e_rb', 'sigma_rs', 'sigma_rb']
if not figureFile:
paraMoments = moments + conditions.keys()
# paraMomentVals = map(str, [EP, e_rs, e_rb, sigma_rs, sigma_rb] + conditions.values())
paraMomentVals = map(str, overlay.values() + conditions.values())
paraString = '_'.join(map(''.join, zip(paraMoments, paraMomentVals)))
fileName = baseName + 'xVar' + xVar + '__' + paraString
fileName = fileName.replace('.', '') # kill all dots for latex
figureFile = os.path.join('/mnt/resultsLarge/idRisk/plots', fileName + '.eps')
# create table if not existent already
if not os.path.isfile(tableFile) or not os.path.getsize(tableFile):
tablePath = os.path.dirname(tableFile)
createOverviewTable(tablePath)
# read and adjust table
overviewTab = tableDict.fromTextFile(tableFile, None, 20, 5)
logger.debug('Read the following table: ')
# Rename variables
logger.debug(overviewTab)
overviewTab.rename('eR_a', 'e_rs')
overviewTab.rename('std_R_a(quar)', 'sigma_rs')
overviewTab.rename('eR_b', 'e_rb')
overviewTab.rename('std_R_b(quar)', 'sigma_rb')
overviewTab.rename('rP_ana', 'EP')
# Scale to yearly values
overviewTab['e_rs'] = overviewTab['e_rs'] ** 4
overviewTab['e_rb'] = overviewTab['e_rb'] ** 4
overviewTab['EP'] = overviewTab['e_rs'] - overviewTab['e_rb']
overviewTab['sigma_rs'] = overviewTab['sigma_rs'] * 2
overviewTab['sigma_rb'] = overviewTab['sigma_rb'] * 2
# make consistency check
if not xVar in overviewTab.cols:
print 'xVar %s not found in table' % xVar
sys.exit()
overviewTab.order([xVar] + conditions.keys() + moments)
overviewTab.sort(conditions.keys() + [xVar])
logger.debug('table after order and sort')
logger.debug(overviewTab)
# Copy table and impose conditions
overviewTabSub = copy.deepcopy(overviewTab)
logger.debug('keep only relevant variables')
varsToKeep = ['EP', 'e_rs', 'e_rb', 'sigma_rs', 'sigma_rb']
if not varsToKeep.count(xVar): varsToKeep.insert(0, xVar)
overviewTabSub.keep(varsToKeep)
# Subset y series
for cond in conditions:
overviewTabSub = overviewTabSub.getSubset( (overviewTabSub[cond] == conditions[cond] ))
# Subset x range
if xVarRange:
overviewTabSub.subset( (overviewTabSub[xVar] >= xVarRange[0]) & (overviewTabSub[xVar] <= xVarRange[1]) )
# make sure the table is not empty
if len(overviewTabSub) == 0:
logger.critical('table empty. Returning')
return 1
logger.info('table after imposing conditions:')
logger.info(overviewTabSub)
x = overviewTabSub[xVar]
# Scale variables to percentage points
overviewTabSub['EP'] = overviewTabSub['EP'] * 100
overviewTabSub['e_rs'] = (overviewTabSub['e_rs'] -1 ) * 100
overviewTabSub['e_rb'] = (overviewTabSub['e_rb'] -1 ) * 100
overviewTabSub['sigma_rs'] = overviewTabSub['sigma_rs'] * 100
overviewTabSub['sigma_rb'] = overviewTabSub['sigma_rb'] * 100
logger.info('Final plot data')
logger.info(overviewTabSub)
# make plot
fig = plt.figure()
ax = fig.add_subplot(111)
# set formatting for plots
if type == 'presentation':
colors = {'EP': 'k', 'e_rs': 'b', 'e_rb': 'g', 'sigma_rs': 'r', 'sigma_rb': 'c'}
linewidths = { 'EP': 3, 'e_rs': 3, 'e_rb': 3, 'sigma_rs': 3, 'sigma_rb': 3 }
linestyles = { 'EP': '-', 'e_rs': '-', 'e_rb': '-', 'sigma_rs': '-', 'sigma_rb': '-' }
elif type == 'paper':
colors = {'EP': 'k', 'e_rs': 'b', 'e_rb': 'g', 'sigma_rs': 'r', 'sigma_rb': 'c'}
linewidths = { 'EP': 3, 'e_rs': 3, 'e_rb': 3, 'sigma_rs': 3, 'sigma_rb': 3 }
linestyles = { 'EP': '-', 'e_rs': '-', 'e_rb': '-', 'sigma_rs': '-', 'sigma_rb': '-' }
else:
logger.critical('unknown plot type to create. ')
labels = { 'EP': '$E(R_m-R_f)$', 'e_rs': '$E(R_m)$', 'e_rb': '$E(R_f)$', 'sigma_rs': '$\sigma(R_m)$', 'sigma_rb': '$\sigma(R_f)$' }
yVars = list(varsToKeep)
yVars.remove(xVar)
for yvar in yVars:
ax.plot(x, overviewTabSub[yvar], linewidth = linewidths[yvar], linestyle = linestyles[yvar], color = colors[yvar], antialiased = True, label = labels[yvar])
## if overlay['EP']:
## ax.plot(x, overviewTabSub['EP'] * 100 , linewidths = 3, linestyle = '-', color = 'k', antialiased = True, label = '$E(R_m-R_f)$')
## if overlay['e_rs']:
## ax.plot(x, (overviewTabSub['e_rs'] -1 ) * 100 , linewidth = 3, linestyle = '-', color = 'b', antialiased = True, label = '$E(R_m)$')
## if overlay['e_rb']:
## ax.plot(x, (overviewTabSub['e_rb'] -1 ) * 100 , linewidth = 3, linestyle = '-', color = 'g', antialiased = True, label = '$E(R_f)$')
## if overlay['sigma_rs']:
## ax.plot(x, overviewTabSub['sigma_rs'] * 100 , linewidth = 3, linestyle = '-', color = 'r', antialiased = True, label = '$\sigma(R_m)$')
## if overlay['sigma_rb']:
## ax.plot(x, overviewTabSub['sigma_rb'] * 100 , linewidth = 3, linestyle = '-', marker = '', color = 'c', antialiased = True, label = '$\sigma(R_f)$')
## #ax.plot(x, overviewTabSub['iBar_Shock0Agent0'] * 10 , linewidth = 3, linestyle = '-', marker = '', color = 'c', antialiased = True, label = 'iBar_Shock0Agent0')
if type == 'presentation':
ax.legend(loc = 'lower left')
if xVar == 'dy':
ax.set_xlabel('$\Delta y$', fontsize = 18)
else:
ax.set_xlabel(xVar, fontsize = 18)
ax.set_ylabel(r'%', fontsize = 18)
elif type == 'paper':
ax.legend(loc = 'lower left')
if xVar == 'dy':
ax.set_xlabel('$\Delta y$', fontsize = 10)
else:
ax.set_xlabel(xVar, fontsize = 10)
ax.set_ylabel(r'%', fontsize = 10)
else:
logger.critical('unknown plot type to create. ')
fig.savefig(figureFile)
os.system('chmod 660 ' + figureFile)
logger.info('figure saved to %s' % figureFile)
from plotAggregate import makeAggregatePlot
from pymods.classes.tableDict import tableDict
path2Pymods = os.path.join('../')
if not sys.path.count(path2Pymods):
sys.path.append(path2Pymods)
from pymods.support.support import getParameter
output_dir = '/home/jonen/workspace/housingproj/model/results/newMchain/finePsi/DE/para_ctry=de_psi=0.0380000000_R_UL=1.0600000000_updateCtryParametersFile=1/output'
aggregateOut = os.path.join(output_dir, 'aggregate.out')
empOwnershipFile = os.path.join(os.path.split(output_dir)[0], 'input', 'SOEP' + 'OwnershipProfilealleduc.out')
ownershipTableFile = os.path.join(output_dir, 'ownershipTable.out')
if os.path.exists(aggregateOut):
# make plot of predicted vs empirical ownership profile
aggregateOutTable = tableDict.fromTextFile(aggregateOut, width = 20, prec = 10)
aggregateOutTable.keep(['age', 'owner'])
aggregateOutTable.rename('owner', 'thOwnership')
empOwnershipTable = tableDict.fromTextFile(empOwnershipFile, width = 20, prec = 10)
empOwnershipTable.rename('PrOwnership', 'empOwnership')
print empOwnershipTable
# print empOwnershipTable.cellFormat
ownershipTable = aggregateOutTable.merged(empOwnershipTable, 'age')
print ownershipTable
# print ownershipTable.cellFormat
yVars = ['thOwnership', 'empOwnership']
# add the individual simulations
for profile in [ '1', '2', '3' ]:
profileOwnershipFile = os.path.join(output_dir, 'simulation_' + profile + '.out')
from pymods.classes.tableDict import tableDict
path2Pymods = os.path.join('../')
if not sys.path.count(path2Pymods):
sys.path.append(path2Pymods)
from pymods.support.support import getParameter
output_dir = '/home/jonen/workspace/housingproj/model/results/newMchain/finePsi/DE/para_ctry=de_psi=0.0380000000_R_UL=1.0600000000_updateCtryParametersFile=1/output'
aggregateOut = os.path.join(output_dir, 'aggregate.out')
empOwnershipFile = os.path.join(
os.path.split(output_dir)[0], 'input',
'SOEP' + 'OwnershipProfilealleduc.out')
ownershipTableFile = os.path.join(output_dir, 'ownershipTable.out')
if os.path.exists(aggregateOut):
# make plot of predicted vs empirical ownership profile
aggregateOutTable = tableDict.fromTextFile(aggregateOut, width=20, prec=10)
aggregateOutTable.keep(['age', 'owner'])
aggregateOutTable.rename('owner', 'thOwnership')
empOwnershipTable = tableDict.fromTextFile(empOwnershipFile,
width=20,
prec=10)
empOwnershipTable.rename('PrOwnership', 'empOwnership')
print empOwnershipTable
# print empOwnershipTable.cellFormat
ownershipTable = aggregateOutTable.merged(empOwnershipTable, 'age')
print ownershipTable
# print ownershipTable.cellFormat
yVars = ['thOwnership', 'empOwnership']
# add the individual simulations
print 'cannot find linspace in string'
os._exit()
if __name__ == '__main__':
logger.info('Starting: \n%s' % ' '.join(sys.argv))
idRiskParaSearchScript().run()
logger.debug('combine resulting tables')
tableList = [
os.path.join(os.getcwd(), folder, 'optimalRun')
for folder in os.listdir(os.getcwd())
if os.path.isdir(folder) and not folder == 'localBaseDir'
and not folder == 'idRiskParaSearch.jobs'
]
tableDicts = [
tableDict.fromTextFile(table, width=20, prec=10) for table in tableList
if os.path.isfile(table)
]
if tableDicts:
optimalRuns = tableDicts[0]
for ixTable, table in enumerate(tableDicts):
if ixTable == 0: continue
optimalRuns = optimalRuns.getAppended(table)
#optimalRuns.order(['dy', 'wBarLower'])
#optimalRuns.sort(['dy'])
logger.info(optimalRuns)
f = open(os.path.join(os.getcwd(), 'optimalRuns'), 'w')
print >> f, optimalRuns
f.flush()
#logger.info('Generating plot')
#baseName = 'moments'
(linSpacePart, strIn) = re.match('(\s*linspace\(.*?\)\s*)[,\s*]*(.*)', strIn).groups()
args = re.match('linspace\(([(0-9\.\s-]+),([0-9\.\s-]+),([0-9\.\s-]+)\)', linSpacePart).groups()
args = [ float(arg) for arg in args] # assume we always want float for linspace
linSpaceVec = np.linspace(args[0], args[1], args[2])
return linSpaceVec
else:
print 'cannot find linspace in string'
os._exit()
if __name__ == '__main__':
logger.info('Starting: \n%s' % ' '.join(sys.argv))
idRiskParaSearchScript().run()
logger.debug('combine resulting tables')
tableList = [ os.path.join(os.getcwd(), folder, 'optimalRun') for folder in os.listdir(os.getcwd()) if os.path.isdir(folder) and not folder == 'localBaseDir' and not folder == 'idRiskParaSearch.jobs' ]
tableDicts = [ tableDict.fromTextFile(table, width = 20, prec = 10) for table in tableList if os.path.isfile(table)]
if tableDicts:
optimalRuns = tableDicts[0]
for ixTable, table in enumerate(tableDicts):
if ixTable == 0: continue
optimalRuns = optimalRuns.getAppended(table)
#optimalRuns.order(['dy', 'wBarLower'])
#optimalRuns.sort(['dy'])
logger.info(optimalRuns)
f = open(os.path.join(os.getcwd(), 'optimalRuns'), 'w')
print >> f, optimalRuns
f.flush()
#logger.info('Generating plot')
#baseName = 'moments'
#path = os.getcwd()
# conditions = {}
def new_tasks(self, extra):
logger.info('entering new_tasks')
# Adjust logging level for stream handler. Could be extracted into a general function.
for handler in logger.handlers:
if not isinstance(handler, logbook.FileHandler):
logger.handlers.remove(handler)
mySH = logbook.StreamHandler(stream = sys.stdout, level = self.params.solverVerb.upper(), format_string = '{record.message}', bubble = True)
logger.handlers.append(mySH)
baseDir = self.params.initial
xVars = self.params.xVars
countryList = self.params.countryList.split()
ctryIndices = getIndex(base = [len(countryList), len(countryList)], restr = 'lowerTr')
for ctryIndex in ctryIndices:
logger.info(countryList[ctryIndex[0]] + countryList[ctryIndex[1]])
# Compute domain
xVarsDom = self.params.xVarsDom.split()
lowerBds = np.array([xVarsDom[i] for i in range(len(xVarsDom)) if i % 2 == 0], dtype = 'float64')
upperBds = np.array([xVarsDom[i] for i in range(len(xVarsDom)) if i % 2 == 1], dtype = 'float64')
domain = zip(lowerBds, upperBds)
# Make problem type specific adjustments.
if self.params.problemType == 'one4all':
gdpTable = tableDict.fromTextFile(fileIn = os.path.join(self.params.pathEmpirical, 'outputInput/momentTable/Gdp/gdpMoments.csv'),
delim = ',', width = 20)
jobname = 'one4all'
logger.info('%s' % self.params.norm)
norm = self.params.norm
try:
norm = int(norm)
except ValueError:
if norm == "np.inf" or norm == "inf":
norm = np.inf
else:
pass # in particular custom norms
logger.info('using norm %s' % (norm))
path_to_stage_dir = os.getcwd()
executable = os.path.basename(self.params.executable)
analyzeResults = anaOne4all(len(list(ctryIndices)), norm = norm)
nlc = nlcOne4all(gdpTable = gdpTable, ctryList = countryList, domain = domain, logFile = os.path.join(path_to_stage_dir, 'nlc.log'))
combOverviews = combineOverviews.combineOverviews(overviewSimuFile = 'eSigmaTable', tableName = 'ag_eSigmaTable', sortKeys = ['norm'])
plot3dTable = combineOverviews.plotTable(tablePath =os.path.join(path_to_stage_dir, 'ag_eSigmaTable'), savePath = os.path.join(path_to_stage_dir, 'scatter3d'))
plot3dTable.columnNames = ['E', 'sigma', 'norm']
for ctryIndex in ctryIndices:
Ctry1 = countryList[ctryIndex[0]]
Ctry2 = countryList[ctryIndex[1]]
# Set Ctry information for this run.
update_parameter_in_file(os.path.join(baseDir, 'input/markovA.in'), 'Ctry',
0, Ctry1, 'space-separated')
update_parameter_in_file(os.path.join(baseDir, 'input/markovB.in'), 'Ctry',
0, Ctry2, 'space-separated')
# Get the correct Ctry Paras into base dir.
self.getCtryParas(baseDir, Ctry1, Ctry2)
# Copy base dir
ctryBaseDir = os.path.join(path_to_stage_dir, 'base' + Ctry1 + Ctry2)
try:
shutil.copytree(baseDir, ctryBaseDir)
except:
logger.info('%s already exists' % baseDir)
kwargs = extra.copy()
kwargs['output_dir'] = path_to_stage_dir
# yield job
yield (jobname, gParaSearchDriver,
[ self.params.executable, path_to_stage_dir, self.params.architecture,
baseDir, self.params.xVars, self.params.initialPop,
self.params.nPopulation, domain, self.params.solverVerb, self.params.problemType,
self.params.pathEmpirical, self.params.itermax, self.params.xConvCrit, self.params.yConvCrit,
self.params.makePlots, self.params.optStrategy, self.params.fWeight, self.params.fCritical, self.params.countryList,
analyzeResults, nlc, plot3dTable, combOverviews
], kwargs)
elif self.params.problemType == 'one4eachPair':
for ctryIndex in ctryIndices:
Ctry1 = countryList[ctryIndex[0]]
Ctry2 = countryList[ctryIndex[1]]
logger.info(Ctry1 + Ctry2)
jobname = Ctry1 + '-' + Ctry2
# set stage dir.
path_to_stage_dir = self.make_directory_path(self.params.output, jobname)
gc3libs.utils.mkdir(path_to_stage_dir)
#path_to_stage_dir = os.path.join(iterationFolder, jobname)
# Get moments table from empirical analysis
gdpTable = tableDict.fromTextFile(fileIn = os.path.join(self.params.pathEmpirical, 'outputInput/momentTable/Gdp/gdpMoments.csv'),
delim = ',', width = 20)
# Set Ctry information for this run.
update_parameter_in_file(os.path.join(baseDir, 'input/markovA.in'), 'Ctry',
0, Ctry1, 'space-separated')
update_parameter_in_file(os.path.join(baseDir, 'input/markovB.in'), 'Ctry',
0, Ctry2, 'space-separated')
# Get the correct Ctry Paras into base dir.
self.getCtryParas(baseDir, Ctry1, Ctry2)
# Copy base dir
ctryBaseDir = os.path.join(path_to_stage_dir, 'base')
try:
shutil.copytree(baseDir, ctryBaseDir)
except:
logger.info('%s already exists' % baseDir)
EA = getParameter(fileIn = os.path.join(baseDir, 'input/parameters.in'), varIn = 'EA',
regexIn = 'bar-separated')
EB = getParameter(fileIn = os.path.join(baseDir, 'input/parameters.in'), varIn = 'EB',
regexIn = 'bar-separated')
sigmaA = getParameter(fileIn = os.path.join(baseDir, 'input/parameters.in'), varIn = 'sigmaA',
regexIn = 'bar-separated')
sigmaB = getParameter(fileIn = os.path.join(baseDir, 'input/parameters.in'), varIn = 'sigmaB',
regexIn = 'bar-separated')
# Pass ctry information to nlc
analyzeResults = anaOne4eachPair
nlc = nlcOne4eachPair(gdpTable = gdpTable, ctryPair = [Ctry1, Ctry2], domain = domain, logFile = os.path.join(path_to_stage_dir, 'nlc.log'))
combOverviews = combineOverviews.combineOverviews(overviewSimuFile = 'overviewSimu', tableName = 'agTable', sortKeys = ['normDev'])
plot3dTable = combineOverviews.plotTable(tablePath =os.path.join(path_to_stage_dir, 'agTable'), savePath = os.path.join(path_to_stage_dir, 'scatter3d'))
plot3dTable.columnNames = ['EA', 'sigmaA', 'normDev']
executable = os.path.basename(self.params.executable)
kwargs = extra.copy()
kwargs['output_dir'] = path_to_stage_dir
# yield job
yield (jobname, gParaSearchDriver,
[ self.params.executable, path_to_stage_dir, self.params.architecture,
ctryBaseDir, self.params.xVars, self.params.initialPop,
self.params.nPopulation, domain, self.params.solverVerb, self.params.problemType,
self.params.pathEmpirical, self.params.itermax, self.params.xConvCrit, self.params.yConvCrit,
self.params.makePlots, self.params.optStrategy, self.params.fWeight, self.params.fCritical, self.params.countryList,
analyzeResults, nlc, plot3dTable, combOverviews
], kwargs)
elif self.params.problemType == 'one4eachCtry':
gdpTable = tableDict.fromTextFile(fileIn = os.path.join(self.params.pathEmpirical, 'outputInput/momentTable/Gdp/gdpMoments.csv'),
delim = ',', width = 20)
if len(countryList) > len(self.params.xVars.split()) / 2 and len(xVars.split()) == 2:
if self.params.xVars[-1:] != " ":
xVars = ( self.params.xVars + ' ' ) * len(countryList)
xVars = xVars[:-1]
else:
xVars = self.params.xVars * len(countryList)
if self.params.xVarsDom[-1:] != " ":
xVarsDom = ( self.params.xVarsDom + ' ' ) * len(countryList)
xVarsDom = xVarsDom[:-1].split()
else:
xVarsDom = self.params.xVarsDom * len(countryList)
xVarsDom = xVarsDom.split()
else:
xVars = self.params.xVars
xVarsDom = self.params.xVarsDom[:-1].split()
jobname = 'one4eachCtry'
lowerBds = np.array([xVarsDom[i] for i in range(len(xVarsDom)) if i % 2 == 0], dtype = 'float64')
upperBds = np.array([xVarsDom[i] for i in range(len(xVarsDom)) if i % 2 == 1], dtype = 'float64')
domain = zip(lowerBds, upperBds)
norm = self.params.norm
try:
norm = int(norm)
except ValueError:
if norm == "np.inf" or norm == "inf":
norm = np.inf
else:
pass # in particular custom norms
logger.info('using norm %s' % (norm))
path_to_stage_dir = os.getcwd()
executable = os.path.basename(self.params.executable)
analyzeResults = anaOne4eachCtry(countryList, len(list(ctryIndices)), norm = norm)
nlc = nlcOne4eachCtry(gdpTable = gdpTable, ctryList = countryList, domain = domain, logFile = os.path.join(path_to_stage_dir, 'nlc.log'))
combOverviews = combineOverviews.combineOverviews(overviewSimuFile = 'eSigmaTable', tableName = 'ag_eSigmaTable', sortKeys = ['norm'])
deKenPrice.plotPopulation = plotPopOne4eachCtry(countryList)
## # Set solver variables
## nXvars = len(xVars.split())
## deKenPrice.I_NP = int(self.params.nPopulation)
## deKenPrice.F_weight = float(self.params.fWeight)
## deKenPrice.F_CR = float(self.params.fCritical)
## deKenPrice.I_D = int(nXvars)
## deKenPrice.lowerBds = np.array([ element[0] for element in domain ], dtype = 'float64')
## deKenPrice.upperBds = np.array([ element[1] for element in domain ], dtype = 'float64')
## deKenPrice.I_itermax = int(self.params.itermax)
## deKenPrice.F_VTR = float(self.params.yConvCrit)
## deKenPrice.I_strategy = int(self.params.optStrategy)
## deKenPrice.I_plotting = int(self.params.makePlots)
## deKenPrice.xConvCrit = float(self.params.xConvCrit)
## deKenPrice.workingDir = path_to_stage_dir
## deKenPrice.verbosity = self.params.solverVerb
plot3dTable = emptyFun
# plot3dTable = combineOverviews.plotTable(tablePath =os.path.join(path_to_stage_dir, 'ag_eSigmaTable'), savePath = os.path.join(path_to_stage_dir, 'scatter3d'))
# plot3dTable.columnNames = ['E', 'sigma', 'norm']
for ctryIndex in ctryIndices:
Ctry1 = countryList[ctryIndex[0]]
Ctry2 = countryList[ctryIndex[1]]
# Set Ctry information for this run.
update_parameter_in_file(os.path.join(baseDir, 'input/markovA.in'), 'Ctry',
0, Ctry1, 'space-separated')
update_parameter_in_file(os.path.join(baseDir, 'input/markovB.in'), 'Ctry',
0, Ctry2, 'space-separated')
# Get the correct Ctry Paras into base dir.
self.getCtryParas(baseDir, Ctry1, Ctry2)
# Copy base dir
ctryBaseDir = os.path.join(path_to_stage_dir, 'base' + Ctry1 + Ctry2)
try:
shutil.copytree(baseDir, ctryBaseDir)
except:
logger.info('%s already exists' % baseDir)
kwargs = extra.copy()
kwargs['output_dir'] = path_to_stage_dir
# yield job
yield (jobname, gParaSearchDriver,
[ self.params.executable, path_to_stage_dir, self.params.architecture,
baseDir, xVars, self.params.initialPop,
self.params.nPopulation, domain, self.params.solverVerb, self.params.problemType,
self.params.pathEmpirical, self.params.itermax, self.params.xConvCrit, self.params.yConvCrit,
self.params.makePlots, self.params.optStrategy, self.params.fWeight, self.params.fCritical, self.params.countryList,
analyzeResults, nlc, plot3dTable, combOverviews
], kwargs)
# Set solver variables
nXvars = len(xVars.split())
deKenPrice.I_NP = int(self.params.nPopulation)
deKenPrice.F_weight = float(self.params.fWeight)
deKenPrice.F_CR = float(self.params.fCritical)
deKenPrice.I_D = int(nXvars)
if self.params.problemType == 'one4eachCtry':
deKenPrice.lowerBds = np.array([xVarsDom[i] for i in range(len(xVarsDom)) if i % 2 == 0], dtype = 'float64')
deKenPrice.upperBds = np.array([xVarsDom[i] for i in range(len(xVarsDom)) if i % 2 == 1], dtype = 'float64')
else:
deKenPrice.lowerBds = np.array([ element[0] for element in domain ], dtype = 'float64')
deKenPrice.upperBds = np.array([ element[1] for element in domain ], dtype = 'float64')
deKenPrice.I_itermax = int(self.params.itermax)
deKenPrice.F_VTR = float(self.params.yConvCrit)
deKenPrice.I_strategy = int(self.params.optStrategy)
deKenPrice.I_plotting = int(self.params.makePlots)
deKenPrice.xConvCrit = float(self.params.xConvCrit)
deKenPrice.workingDir = path_to_stage_dir
deKenPrice.verbosity = self.params.solverVerb
logger.info('done with new_tasks')
def momentPlots(baseName,
path,
xVar,
overlay,
conditions={},
xVarRange=None,
figureFile=None,
type='presentation'):
if os.path.isdir(path):
# if not tableFile:
tableFile = os.path.join(path, 'overviewSimu')
elif os.path.isfile(path):
tableFile = path
else:
logger.critical('path is not a directory or a table file.')
logger.info('\nCreating plot for path %s with conditions %s' %
(path, conditions))
# make sure xVar is not conditioned upon -> we want to use it for xaxis.
if xVar in conditions.keys():
conditions.pop(xVar)
# create a file name for emerging plots
moments = ['EP', 'e_rs', 'e_rb', 'sigma_rs', 'sigma_rb']
if not figureFile:
paraMoments = moments + conditions.keys()
# paraMomentVals = map(str, [EP, e_rs, e_rb, sigma_rs, sigma_rb] + conditions.values())
paraMomentVals = map(str, overlay.values() + conditions.values())
paraString = '_'.join(map(''.join, zip(paraMoments, paraMomentVals)))
fileName = baseName + 'xVar' + xVar + '__' + paraString
fileName = fileName.replace('.', '') # kill all dots for latex
figureFile = os.path.join('/mnt/resultsLarge/idRisk/plots',
fileName + '.eps')
# create table if not existent already
if not os.path.isfile(tableFile) or not os.path.getsize(tableFile):
tablePath = os.path.dirname(tableFile)
createOverviewTable(tablePath)
# read and adjust table
overviewTab = tableDict.fromTextFile(tableFile, None, 20, 5)
logger.debug('Read the following table: ')
# Rename variables
logger.debug(overviewTab)
overviewTab.rename('eR_a', 'e_rs')
overviewTab.rename('std_R_a(quar)', 'sigma_rs')
overviewTab.rename('eR_b', 'e_rb')
overviewTab.rename('std_R_b(quar)', 'sigma_rb')
overviewTab.rename('rP_ana', 'EP')
# Scale to yearly values
overviewTab['e_rs'] = overviewTab['e_rs']**4
overviewTab['e_rb'] = overviewTab['e_rb']**4
overviewTab['EP'] = overviewTab['e_rs'] - overviewTab['e_rb']
overviewTab['sigma_rs'] = overviewTab['sigma_rs'] * 2
overviewTab['sigma_rb'] = overviewTab['sigma_rb'] * 2
# make consistency check
if not xVar in overviewTab.cols:
print 'xVar %s not found in table' % xVar
sys.exit()
overviewTab.order([xVar] + conditions.keys() + moments)
overviewTab.sort(conditions.keys() + [xVar])
logger.debug('table after order and sort')
logger.debug(overviewTab)
# Copy table and impose conditions
overviewTabSub = copy.deepcopy(overviewTab)
logger.debug('keep only relevant variables')
varsToKeep = ['EP', 'e_rs', 'e_rb', 'sigma_rs', 'sigma_rb']
if not varsToKeep.count(xVar): varsToKeep.insert(0, xVar)
overviewTabSub.keep(varsToKeep)
# Subset y series
for cond in conditions:
overviewTabSub = overviewTabSub.getSubset(
(overviewTabSub[cond] == conditions[cond]))
# Subset x range
if xVarRange:
overviewTabSub.subset((overviewTabSub[xVar] >= xVarRange[0])
& (overviewTabSub[xVar] <= xVarRange[1]))
# make sure the table is not empty
if len(overviewTabSub) == 0:
logger.critical('table empty. Returning')
return 1
logger.info('table after imposing conditions:')
logger.info(overviewTabSub)
x = overviewTabSub[xVar]
# Scale variables to percentage points
overviewTabSub['EP'] = overviewTabSub['EP'] * 100
overviewTabSub['e_rs'] = (overviewTabSub['e_rs'] - 1) * 100
overviewTabSub['e_rb'] = (overviewTabSub['e_rb'] - 1) * 100
overviewTabSub['sigma_rs'] = overviewTabSub['sigma_rs'] * 100
overviewTabSub['sigma_rb'] = overviewTabSub['sigma_rb'] * 100
logger.info('Final plot data')
logger.info(overviewTabSub)
# make plot
fig = plt.figure()
ax = fig.add_subplot(111)
# set formatting for plots
if type == 'presentation':
colors = {
'EP': 'k',
'e_rs': 'b',
'e_rb': 'g',
'sigma_rs': 'r',
'sigma_rb': 'c'
}
linewidths = {
'EP': 3,
'e_rs': 3,
'e_rb': 3,
'sigma_rs': 3,
'sigma_rb': 3
}
linestyles = {
'EP': '-',
'e_rs': '-',
'e_rb': '-',
'sigma_rs': '-',
'sigma_rb': '-'
}
elif type == 'paper':
colors = {
'EP': 'k',
'e_rs': 'b',
'e_rb': 'g',
'sigma_rs': 'r',
'sigma_rb': 'c'
}
linewidths = {
'EP': 3,
'e_rs': 3,
'e_rb': 3,
'sigma_rs': 3,
'sigma_rb': 3
}
linestyles = {
'EP': '-',
'e_rs': '-',
'e_rb': '-',
'sigma_rs': '-',
'sigma_rb': '-'
}
else:
logger.critical('unknown plot type to create. ')
labels = {
'EP': '$E(R_m-R_f)$',
'e_rs': '$E(R_m)$',
'e_rb': '$E(R_f)$',
'sigma_rs': '$\sigma(R_m)$',
'sigma_rb': '$\sigma(R_f)$'
}
yVars = list(varsToKeep)
yVars.remove(xVar)
for yvar in yVars:
ax.plot(x,
overviewTabSub[yvar],
linewidth=linewidths[yvar],
linestyle=linestyles[yvar],
color=colors[yvar],
antialiased=True,
label=labels[yvar])
## if overlay['EP']:
## ax.plot(x, overviewTabSub['EP'] * 100 , linewidths = 3, linestyle = '-', color = 'k', antialiased = True, label = '$E(R_m-R_f)$')
## if overlay['e_rs']:
## ax.plot(x, (overviewTabSub['e_rs'] -1 ) * 100 , linewidth = 3, linestyle = '-', color = 'b', antialiased = True, label = '$E(R_m)$')
## if overlay['e_rb']:
## ax.plot(x, (overviewTabSub['e_rb'] -1 ) * 100 , linewidth = 3, linestyle = '-', color = 'g', antialiased = True, label = '$E(R_f)$')
## if overlay['sigma_rs']:
## ax.plot(x, overviewTabSub['sigma_rs'] * 100 , linewidth = 3, linestyle = '-', color = 'r', antialiased = True, label = '$\sigma(R_m)$')
## if overlay['sigma_rb']:
## ax.plot(x, overviewTabSub['sigma_rb'] * 100 , linewidth = 3, linestyle = '-', marker = '', color = 'c', antialiased = True, label = '$\sigma(R_f)$')
## #ax.plot(x, overviewTabSub['iBar_Shock0Agent0'] * 10 , linewidth = 3, linestyle = '-', marker = '', color = 'c', antialiased = True, label = 'iBar_Shock0Agent0')
if type == 'presentation':
ax.legend(loc='lower left')
if xVar == 'dy':
ax.set_xlabel('$\Delta y$', fontsize=18)
else:
ax.set_xlabel(xVar, fontsize=18)
ax.set_ylabel(r'%', fontsize=18)
elif type == 'paper':
ax.legend(loc='lower left')
if xVar == 'dy':
ax.set_xlabel('$\Delta y$', fontsize=10)
else:
ax.set_xlabel(xVar, fontsize=10)
ax.set_ylabel(r'%', fontsize=10)
else:
logger.critical('unknown plot type to create. ')
fig.savefig(figureFile)
os.system('chmod 660 ' + figureFile)
logger.info('figure saved to %s' % figureFile)
