def __init__(self):
SessionBasedScript.__init__(
self,
version = '0.2',
# only '.loop' files are considered as valid input
input_filename_pattern = '*.loop',
)
paraLoop_fp.__init__(self, verbosity = 'INFO')
def __init__(self):
SessionBasedScript.__init__(
self,
version='0.2',
# only '.loop' files are considered as valid input
input_filename_pattern='*.loop',
)
paraLoop_fp.__init__(self, verbosity='INFO')
def __init__(self, inParaCombos, iteration, pathToExecutable, pathToStageDir, architecture, baseDir, xVars,
solverVerb, problemType, analyzeResults, ctryList, **extra_args):
'''
Generate a list of tasks and initialize a ParallelTaskCollection with them.
Uses paraLoop class to generate a list of (descriptions, substitutions for the input files). Descriptions are generated from
variable names that are hard coded in this method right now.
Uses method generateTaskList to create a list of GPremiumApplication's which are invoked from a list of inputs (appropriately adjusted input files),
the output directory and some further settings for each run.
inParaCombos: List of tuples defining the parameter combinations.
iteration: Current iteration number.
pathToExecutable: Path to the executable (the external program to be called).
pathToStageDir: Root path. Usually os.getcwd()
architecture: 32 or 64 bit.
baseDir: Directory in which the input files are located.
xVars: Names of the x variables.
solverVerb: Logger verbosity.
problemType: Forward premium specific flag to determine which case to look at.
analyzeResults: Function to use to analyze the emerging output.
ctryList: Forward premium specific list of ctrys to look at.
'''
logger.debug('entering gParaSearchParalell.__init__')
# Set up initial variables and set the correct methods.
self.pathToStageDir = pathToStageDir
self.problemType = problemType
self.executable = pathToExecutable
self.architecture = architecture
self.baseDir = baseDir
self.verbosity = solverVerb.upper()
self.xVars = xVars
self.n = len(self.xVars.split())
self.analyzeResults = analyzeResults
self.ctryList = ctryList
self.iteration = iteration
self.jobname = 'evalSolverGuess' + '-' + extra_args['jobname'] + '-' + str(self.iteration)
self.extra_args = extra_args
tasks = []
# --- createJobs_x ---
# Log activity
cDate = datetime.date.today()
cTime = datetime.datetime.time(datetime.datetime.now())
dateString = '{0:04d}-{1:02d}-{2:02d}-{3:02d}-{4:02d}-{5:02d}'.format(cDate.year, cDate.month, cDate.day, cTime.hour, cTime.minute, cTime.second)
logger.debug('Establishing parallel task on %s' % dateString)
# Enter an iteration specific folder
self.iterationFolder = os.path.join(self.pathToStageDir, 'Iteration-' + str(self.iteration))
try:
os.mkdir(self.iterationFolder)
except OSError:
print '%s already exists' % self.iterationFolder
# save population to file
np.savetxt(os.path.join(self.iterationFolder, 'curPopulation'), inParaCombos, delimiter = ' ')
# Take the list of parameter combinations and translate them in a comma separated list of values for each variable to be fed into paraLoop file.
# This can be done much more elegantly with ','.join() but it works...
vals = []
nVariables = range(len(inParaCombos[0]))
for ixVar in nVariables:
varValString = ''
for ixParaCombo, paraCombo in enumerate(inParaCombos):
### Should make more precise string conversion.
varValString += str(paraCombo[ixVar])
if ixParaCombo < len(inParaCombos) - 1:
varValString += ', '
vals.append( varValString )
# Make problem specific adjustments to the paraLoop file.
if self.problemType == 'one4all':
print 'one4all'
variables = ['Ctry', 'Ctry', 'EA', 'EB', 'sigmaA', 'sigmaB']
groups = [ 0, 0, 1, 1, 1, 1 ]
groupRestrs = [ 'lowerTr', 'lowerTr', 'diagnol', 'diagnol', 'diagnol', 'diagnol' ]
writeVals = [ ", ".join(self.ctryList), ", ".join(self.ctryList), vals[0], vals[0], vals[1], vals[1] ]
self.variables = ['EA','sigmaA']
self.paraCombos = inParaCombos
paraFiles = [ 'input/markovA.in', 'input/markovB.in', 'input/parameters.in', 'input/parameters.in', 'input/parameters.in', 'input/parameters.in' ]
paraFileRegex = [ 'space-separated', 'space-separated', 'bar-separated', 'bar-separated' , 'bar-separated' , 'bar-separated' ]
self.analyzeResults.tablePath = self.iterationFolder
elif self.problemType == 'one4eachPair':
print 'one4eachPair'
# Check if EA or sigmaA are alone in the specified parameters. If so make diagnol adjustments
writeVals = []
if 'EA' in self.xVars and not 'EB' in self.xVars:
variables = [ 'EA', 'EB' ]
groups = [ '0', '0' ]
groupRestrs = [ 'diagnol', 'diagnol' ]
writeVals.append(vals[0])
writeVals.append(vals[0])
paraCombosEA = [ np.append(ele[0], ele[0]) for ele in inParaCombos ]
if 'sigmaA' in self.xVars and not 'sigmaB' in self.xVars:
variables.append( 'sigmaA')
variables.append('sigmaB')
groups.append( '0')
groups.append('0')
groupRestrs.append( 'diagnol')
groupRestrs.append( 'diagnol' )
writeVals.append(vals[1])
writeVals.append(vals[1])
paraCombosSigmaA = [ np.append(ele[1], ele[1]) for ele in inParaCombos ]
# match ctry with val
ctryVals = {}
for ixCtry, ctry in enumerate(ctryList):
ctryVals[ctry] = vals
self.variables = variables
# Prepare paraCombos matching to resulting table. Used in analyzeOverviewTable
# !!! This should be dependent on problem type or on missing variables in xvars. !!!
paraCombos = []
for EA,sA in zip(paraCombosEA, paraCombosSigmaA):
paraCombo = np.append(EA, sA)
paraCombos.append(paraCombo)
self.paraCombos = paraCombos
paraFiles = [ 'input/parameters.in', 'input/parameters.in', 'input/parameters.in', 'input/parameters.in' ]
paraFileRegex = [ 'bar-separated', 'bar-separated' , 'bar-separated' , 'bar-separated' ]
elif self.problemType == 'one4eachCtry':
print 'one4eachCtry'
ctry1List = []
ctry2List = []
EAList = []
EBList = []
sigmaAList = []
sigmaBList = []
self.paraCombos = []
ctryIndices = getIndex([len(ctryList), len(ctryList)], 'lowerTr')
for ixCombo in range(len(inParaCombos)):
ctry1ListCombo = []
ctry2ListCombo = []
EAListCombo = []
EBListCombo = []
sigmaAListCombo = []
sigmaBListCombo = []
for ctryIndex in ctryIndices:
ctry1ListCombo.append(ctryList[ctryIndex[0]])
ctry2ListCombo.append(ctryList[ctryIndex[1]])
EAListCombo.append(inParaCombos[ixCombo][0 + 2 * ctryIndex[0]])
sigmaAListCombo.append(inParaCombos[ixCombo][1 + 2 * ctryIndex[0]])
EBListCombo.append(inParaCombos[ixCombo][0 + 2 *ctryIndex[1]])
sigmaBListCombo.append(inParaCombos[ixCombo][1 + 2 * ctryIndex[1]])
self.paraCombos.append(zip(ctry1ListCombo, ctry2ListCombo, EAListCombo, sigmaAListCombo, EBListCombo, sigmaBListCombo))
ctry1List.extend(ctry1ListCombo)
ctry2List.extend(ctry2ListCombo)
EAList.extend(map(str, EAListCombo))
EBList.extend(map(str, EBListCombo))
sigmaAList.extend(map(str, sigmaAListCombo))
sigmaBList.extend(map(str, sigmaBListCombo))
variables = ['Ctry', 'Ctry', 'EA', 'EB', 'sigmaA', 'sigmaB']
groups = [ 0, 0, 0, 0, 0, 0 ]
groupRestrs = [ 'diagnol', 'diagnol', 'diagnol', 'diagnol', 'diagnol', 'diagnol' ]
writeVals = [ ", ".join(ctry1List), ", ".join(ctry2List), ", ".join(EAList), ", ".join(EBList), ", ".join(sigmaAList),", ".join(sigmaBList)]
paraFiles = [ 'input/markovA.in', 'input/markovB.in', 'input/parameters.in', 'input/parameters.in', 'input/parameters.in', 'input/parameters.in' ]
paraFileRegex = [ 'space-separated', 'space-separated', 'bar-separated', 'bar-separated' , 'bar-separated' , 'bar-separated' ]
#self.paraCombos = inParaCombos
self.analyzeResults.tablePath = self.iterationFolder
# variable list passed to analyzeOverviewTables
self.variables = ['EA', 'sigmaA', 'EB', 'sigmaB']
print 'Done setting up one4eachCtry. '
# Write a para.loop file to generate grid jobs
para_loop = self.writeParaLoop(variables = variables,
groups = groups,
groupRestrs = groupRestrs,
vals = writeVals,
desPath = os.path.join(self.iterationFolder, 'para.loopTmp'),
paraFiles = paraFiles,
paraFileRegex = paraFileRegex)
paraLoop_fp.__init__(self, verbosity = self.verbosity)
tasks = self.generateTaskList(para_loop, self.iterationFolder)
ParallelTaskCollection.__init__(self, self.jobname, tasks)
def __init__(self, inParaCombos, iteration, pathToExecutable, pathToStageDir, architecture, baseDir, xVars,
solverVerb, problemType, analyzeResults, ctryList, **extra_args):
'''
Generate a list of tasks and initialize a ParallelTaskCollection with them.
Uses paraLoop class to generate a list of (descriptions, substitutions for the input files). Descriptions are generated from
variable names that are hard coded in this method right now.
Uses method generateTaskList to create a list of GPremiumApplication's which are invoked from a list of inputs (appropriately adjusted input files),
the output directory and some further settings for each run.
inParaCombos: List of tuples defining the parameter combinations.
iteration: Current iteration number.
pathToExecutable: Path to the executable (the external program to be called).
pathToStageDir: Root path. Usually os.getcwd()
architecture: 32 or 64 bit.
baseDir: Directory in which the input files are located.
xVars: Names of the x variables.
solverVerb: Logger verbosity.
problemType: Forward premium specific flag to determine which case to look at.
analyzeResults: Function to use to analyze the emerging output.
ctryList: Forward premium specific list of ctrys to look at.
'''
logger.debug('entering gParaSearchParalell.__init__')
# Set up initial variables and set the correct methods.
self.pathToStageDir = pathToStageDir
self.problemType = problemType
self.executable = pathToExecutable
self.architecture = architecture
self.baseDir = baseDir
self.verbosity = solverVerb.upper()
self.xVars = xVars
self.n = len(self.xVars.split())
self.analyzeResults = analyzeResults
self.ctryList = ctryList
self.iteration = iteration
self.jobname = 'evalSolverGuess' + '-' + extra_args['jobname'] + '-' + str(self.iteration)
self.extra_args = extra_args
tasks = []
# --- createJobs_x ---
# Log activity
cDate = datetime.date.today()
cTime = datetime.datetime.time(datetime.datetime.now())
dateString = '{0:04d}-{1:02d}-{2:02d}-{3:02d}-{4:02d}-{5:02d}'.format(cDate.year, cDate.month, cDate.day, cTime.hour, cTime.minute, cTime.second)
logger.debug('Establishing parallel task on %s' % dateString)
# Enter an iteration specific folder
self.iterationFolder = os.path.join(self.pathToStageDir, 'Iteration-' + str(self.iteration))
try:
os.mkdir(self.iterationFolder)
except OSError:
print '%s already exists' % self.iterationFolder
# save population to file
np.savetxt(os.path.join(self.iterationFolder, 'curPopulation'), inParaCombos, delimiter = ' ')
# Take the list of parameter combinations and translate them in a comma separated list of values for each variable to be fed into paraLoop file.
# This can be done much more elegantly with ','.join() but it works...
vals = []
nVariables = range(len(inParaCombos[0]))
for ixVar in nVariables:
varValString = ''
for ixParaCombo, paraCombo in enumerate(inParaCombos):
### Should make more precise string conversion.
varValString += str(paraCombo[ixVar])
if ixParaCombo < len(inParaCombos) - 1:
varValString += ', '
vals.append( varValString )
# Make problem specific adjustments to the paraLoop file.
if self.problemType == 'one4all':
print 'one4all'
variables = ['Ctry', 'Ctry', 'EA', 'EB', 'sigmaA', 'sigmaB']
groups = [ 0, 0, 1, 1, 1, 1 ]
groupRestrs = [ 'lowerTr', 'lowerTr', 'diagnol', 'diagnol', 'diagnol', 'diagnol' ]
writeVals = [ ", ".join(self.ctryList), ", ".join(self.ctryList), vals[0], vals[0], vals[1], vals[1] ]
self.variables = ['EA','sigmaA']
self.paraCombos = inParaCombos
paraFiles = [ 'input/markovA.in', 'input/markovB.in', 'input/parameters.in', 'input/parameters.in', 'input/parameters.in', 'input/parameters.in' ]
paraFileRegex = [ 'space-separated', 'space-separated', 'bar-separated', 'bar-separated' , 'bar-separated' , 'bar-separated' ]
self.analyzeResults.tablePath = self.iterationFolder
elif self.problemType == 'one4eachPair':
print 'one4eachPair'
# Check if EA or sigmaA are alone in the specified parameters. If so make diagnol adjustments
writeVals = []
if 'EA' in self.xVars and not 'EB' in self.xVars:
variables = [ 'EA', 'EB' ]
groups = [ '0', '0' ]
groupRestrs = [ 'diagnol', 'diagnol' ]
writeVals.append(vals[0])
writeVals.append(vals[0])
paraCombosEA = [ np.append(ele[0], ele[0]) for ele in inParaCombos ]
if 'sigmaA' in self.xVars and not 'sigmaB' in self.xVars:
variables.append( 'sigmaA')
variables.append('sigmaB')
groups.append( '0')
groups.append('0')
groupRestrs.append( 'diagnol')
groupRestrs.append( 'diagnol' )
writeVals.append(vals[1])
writeVals.append(vals[1])
paraCombosSigmaA = [ np.append(ele[1], ele[1]) for ele in inParaCombos ]
# match ctry with val
ctryVals = {}
for ixCtry, ctry in enumerate(ctryList):
ctryVals[ctry] = vals
self.variables = variables
# Prepare paraCombos matching to resulting table. Used in analyzeOverviewTable
# !!! This should be dependent on problem type or on missing variables in xvars. !!!
paraCombos = []
for EA,sA in zip(paraCombosEA, paraCombosSigmaA):
paraCombo = np.append(EA, sA)
paraCombos.append(paraCombo)
self.paraCombos = paraCombos
paraFiles = [ 'input/parameters.in', 'input/parameters.in', 'input/parameters.in', 'input/parameters.in' ]
paraFileRegex = [ 'bar-separated', 'bar-separated' , 'bar-separated' , 'bar-separated' ]
elif self.problemType == 'one4eachCtry':
print 'one4eachCtry'
ctry1List = []
ctry2List = []
EAList = []
EBList = []
sigmaAList = []
sigmaBList = []
self.paraCombos = []
ctryIndices = getIndex([len(ctryList), len(ctryList)], 'lowerTr')
for ixCombo in range(len(inParaCombos)):
ctry1ListCombo = []
ctry2ListCombo = []
EAListCombo = []
EBListCombo = []
sigmaAListCombo = []
sigmaBListCombo = []
for ctryIndex in ctryIndices:
ctry1ListCombo.append(ctryList[ctryIndex[0]])
ctry2ListCombo.append(ctryList[ctryIndex[1]])
EAListCombo.append(inParaCombos[ixCombo][0 + 2 * ctryIndex[0]])
sigmaAListCombo.append(inParaCombos[ixCombo][1 + 2 * ctryIndex[0]])
EBListCombo.append(inParaCombos[ixCombo][0 + 2 *ctryIndex[1]])
sigmaBListCombo.append(inParaCombos[ixCombo][1 + 2 * ctryIndex[1]])
self.paraCombos.append(zip(ctry1ListCombo, ctry2ListCombo, EAListCombo, sigmaAListCombo, EBListCombo, sigmaBListCombo))
ctry1List.extend(ctry1ListCombo)
ctry2List.extend(ctry2ListCombo)
EAList.extend(map(str, EAListCombo))
EBList.extend(map(str, EBListCombo))
sigmaAList.extend(map(str, sigmaAListCombo))
sigmaBList.extend(map(str, sigmaBListCombo))
variables = ['Ctry', 'Ctry', 'EA', 'EB', 'sigmaA', 'sigmaB']
groups = [ 0, 0, 0, 0, 0, 0 ]
groupRestrs = [ 'diagnol', 'diagnol', 'diagnol', 'diagnol', 'diagnol', 'diagnol' ]
writeVals = [ ", ".join(ctry1List), ", ".join(ctry2List), ", ".join(EAList), ", ".join(EBList), ", ".join(sigmaAList),", ".join(sigmaBList)]
paraFiles = [ 'input/markovA.in', 'input/markovB.in', 'input/parameters.in', 'input/parameters.in', 'input/parameters.in', 'input/parameters.in' ]
paraFileRegex = [ 'space-separated', 'space-separated', 'bar-separated', 'bar-separated' , 'bar-separated' , 'bar-separated' ]
#self.paraCombos = inParaCombos
self.analyzeResults.tablePath = self.iterationFolder
# variable list passed to analyzeOverviewTables
self.variables = ['EA', 'sigmaA', 'EB', 'sigmaB']
print 'Done setting up one4eachCtry. '
# Write a para.loop file to generate grid jobs
para_loop = self.writeParaLoop(variables = variables,
groups = groups,
groupRestrs = groupRestrs,
vals = writeVals,
desPath = os.path.join(self.iterationFolder, 'para.loopTmp'),
paraFiles = paraFiles,
paraFileRegex = paraFileRegex)
paraLoop_fp.__init__(self, verbosity = self.verbosity)
tasks = self.generateTaskList(para_loop, self.iterationFolder)
ParallelTaskCollection.__init__(self, self.jobname, tasks)
