def validate(self, discretization, parallelization):
# get parallelization configuration
xpesize, ypesize, zpesize = parallelization.reshape(3)
# check if number of cells in not smaller than block size
if discretization['NX'] < self.bs * xpesize:
message = 'mesh resolution NX / xpesize is smaller than block size'
details = '%d / %d < %d.' % (discretization['NX'], xpesize,
self.bs)
helpers.error(message, details)
if discretization['NY'] < self.bs * ypesize:
message = 'mesh resolution NY / ypesize is smaller than block size'
details = '%d / %d < %d.' % (discretization['NY'], ypesize,
self.bs)
helpers.error(message, details)
if discretization['NZ'] < self.bs * zpesize:
message = 'mesh resolution NZ / zpesize is smaller than block size'
details = '%d / %d < %d.' % (discretization['NZ'], zpesize,
self.bs)
helpers.error(message, details)
# check if number of blocks is not smaller than available threads
blocks_x = discretization['NX'] / (self.bs * xpesize)
blocks_y = discretization['NY'] / (self.bs * ypesize)
blocks_z = discretization['NZ'] / (self.bs * zpesize)
blocks = blocks_x * blocks_y * blocks_z
if blocks < parallelization.threads:
message = 'number of blocks is smaller than available threads: %d < %d.' % (
blocks, parallelization.threads)
details = 'Discretization: %s' % str(discretization)
advice = 'Parallelization: %s ranks, %d threads' % (str(
parallelization.reshape(3)), parallelization.threads)
helpers.warning(message, details, advice)
helpers.query('Continue with sub-optimal parallelization?')
def execute (self, cmd, directory='.'):
# report command
if self.params.verbose >= 1:
print
print '=== EXECUTE ==='
print 'DIR: ' + directory
print 'CMD: ' + cmd
print '==='
print
# set stdout and stderr based on verbosity level
if self.params.verbose >= 2:
stdout = None
stderr = None
else:
stdout = subprocess.PIPE#open (os.devnull, 'w')
stderr = subprocess.PIPE#subprocess.STDOUT
# execute command
if not self.params.simulate:
process = subprocess.Popen (cmd, cwd=directory, stdout=stdout, stderr=stderr, shell=True, env=os.environ.copy())
output = process.communicate()
failed = process.poll()
if failed:
message = 'Submission failed'
helpers.warning (message, details=str(process.stdout))
def report(self):
if not self.available:
helpers.warning('ERRORS not available')
return
print
print ' :: ERRORS: (normalized to %s [~%.1e])' % (helpers.scif(
self.normalization), self.normalization)
print ' :'
print ' : LEVEL :' + ' '.join(
[' ' + helpers.intf(level, table=1) for level in self.levels])
print ' :----------' + '-'.join(
[helpers.scif(None, table=1, bar=1) for level in self.levels])
print ' : ERROR :',
for level in self.levels:
print helpers.scif(self.relative_error[level], table=1),
print
print ' :'
print ' : Total sampling error : %s [~%.1e]' % (helpers.scif(
self.total_relative_error), self.total_relative_error)
print ' : Deterministic error (bias) : %s [~%.1e]' % (helpers.scif(
self.relative_bias), self.relative_bias)
if numpy.isnan(self.total_relative_error) or numpy.isinf(
self.total_relative_error):
self.available = 0
def speedup(self, indicators, counts, forecast=False):
if not self.available or self.total_error == 0:
helpers.warning(
'Speedup can not be estimated since total sampling error is not available'
)
self.speedup_mlmc = None
self.speedup_ocv = None
return
if forecast:
counts = counts.loaded + counts.additional
else:
counts = counts.loaded
error = self.total(
self.errors(indicators.variance_diff_opt['infered'], counts))
# compute MLMC vs. MC speedup
FINEST = numpy.max(
[level for level in self.levels if counts[level] > 0])
work_mlmc = sum([
indicators.pairworks[level] * counts[level]
for level in self.levels
])
#variance_mc = numpy.max ( [ indicators.variance [0] ['infered'] [level] for level in self.levels [0 : FINEST + 1] ] )
variance_mc = indicators.variance[0]['infered'][FINEST]
samples_mc = numpy.ceil(variance_mc / error**2)
work_mc = indicators.works[FINEST] * samples_mc
self.speedup_mlmc = work_mc / work_mlmc
# avoid round-off errors for pure MC runs
if len(self.levels) == 1:
self.speedup_mlmc = 1.0
samples_mc = counts[0]
# report
print
if forecast: print ' :: FORECAST'
print ' :: SPEEDUP (MLMC vs. MC): %.1f' % self.speedup_mlmc + (
' [finest level: %d]' % FINEST if FINEST != self.L else '')
print ' : -> MLMC budget : %s CPU hours' % helpers.intf(
numpy.ceil(work_mlmc))
print ' : -> MC budget : %s CPU hours' % helpers.intf(
numpy.ceil(work_mc))
print ' : -> MC samples: %s' % helpers.intf(samples_mc)
# compute and report OCV MLMC vs. PLAIN MLMC speedup
# REMARK: since samples were optimized for OSV, this is not an accurate measure for speedup
# REMARK: speedup of the variance reduction cost functional is already reported in indicators.optimize()
'''
def manual(self):
if helpers.query('Do you want to manually adjust samples?',
exit=0) != 'y':
modified = False
return modified
else:
message = 'specify the required additional number of samples (separated by spaces)'
hint = 'press ENTER to make no changes'
default = str(self.counts.additional)
parsed = 0
while not parsed:
samples = helpers.query(message,
hint=hint,
type=str,
default=default,
exit=0)
modified = samples != default
if modified:
samples = samples.split(' ')
if len(samples) != len(self.levels):
helpers.warning(
'Input not recognized, please try again:')
continue
else:
parsed = 1
for level in self.levels:
self.counts.additional[level] = int(samples[level])
self.available = 1
else:
parsed = 1
return modified
def env (self, var):
try:
return os.environ [var]
except:
helpers.warning ('environment variable %s not set' % var, details = 'Using: path = None')
return None
def evaluate(self, samples, indices=None, qois=None):
# statistic is initially not available
self.available = 0
# compute indices, if not explicitely specified
if indices == None:
indices = [
index for index, sample in enumerate(samples) if sample != None
]
# check if at least one sample is available
if len(indices) == 0:
print ' %-30s[%s]' % (self.name, 'unavailable')
return
# check if sufficiently many samples are available
if len(indices) < self.limit:
print ' %-30s[%s]' % (self.name, 'insufficient')
return
# statistic will be available
self.available = 1
# copy data class from the first valid sample
self.estimate = copy.deepcopy(samples[indices[0]])
# resize estimate according to statistics size
if not self.online:
self.estimate.resize(self.size)
# quantities of interest to be assembled
if qois == None:
names = self.estimate.qois
extents = [None for name in names]
else:
names = qois.keys()
extents = qois.values()
# use progress indicator, report current statistic each time
prefix = ' %-30s' % self.name
steps = len(names) * len(indices)
progress = helpers.Progress(prefix=prefix, steps=steps, length=20)
progress.init()
failed = []
# compute sample statistics for each qoi
for i, (qoi, extent) in enumerate(zip(names, extents)):
# check if qoi is available
if qoi not in self.estimate.data:
failed.append(qoi)
progress.update((i + 1) * len(indices))
continue
# templated online estimation is supported by statistic
if self.online:
# initialize statistics
self.init()
# process all samples
for index in indices:
# update estimate with a sample
self.update(samples[index][qoi], extent)
# update progress
progress.update(i * len(indices) + index + 1)
# store estimated statistic
self.estimate[qoi] = self.result()
# templated online estimation is NOT supported by statistic
else:
# compute sample statistics for each element in a flattened array
for element in xrange(len(self.estimate.serialize(qoi))):
# create an ensemble of sample values
ensemble = [
sample.serialize(qoi)[element]
for index, sample in enumerate(samples)
if index in indices
]
# compute statistic
self.estimate.serialize(qoi)[element] = self.compute(
ensemble, extent)
# update progress
progress.update((i + 1) * len(indices))
# finalize progress indicator
progress.finalize()
# report missing qois
if failed != []:
helpers.warning('Missing QoIs: %s' % ' '.join(failed))
for word in text[start : currWord+1]:
addWordOrNum(word, counttarget, countall)
targettopost += 1
prevPostEnd = min(len(text), currWord + AROUND + 1)
for word in text[currWord+1 : prevPostEnd]:
addWordOrNum(word, countpostfix, countall)
currWord = prevPostEnd - 1
for j in range(0, len(thingsFound)):
thingsFound[j] = 0
break
currWord += 1
if prevPostEnd == 0:
warning("None of \"" + (",".join([" ".join(t) for t in things])) + "\" found in " + person["name"] + " abstract.")
backtoback += len(text) - prevPostEnd
for word in text[prevPostEnd : len(text)]:
addWordOrNum(word, countbackground)
countall = sorted(countall.items(), key=operator.itemgetter(1), reverse=True)[:DISTINCT_WORDS]
words = [w[0] for w in countall]
states = [countbackground, countprefix, counttarget, countpostfix]
def stateToBCol(state, words):
col = [state[word] for word in words]
for word in words:
del state[word]
col.append(sum(state.values()))
normalization = sum(col)
return [float(count) / normalization for count in col]
b = [stateToBCol(state, words) for state in states]
def infer(self,
indicator,
degree=1,
log=0,
exp=0,
offset=0,
factor=1,
critical=0,
min=None,
max=None):
# check if inference is not enforced
if not self.enforce:
# proceed only in case some indicators are not available
if not numpy.isnan(indicator['measured'][indicator.start:]).any():
# simply copy all values
indicator['infered'] = indicator['measured'][:]
# do not proceed with the inference
return
# inference will be performed
self.infered = 1
# simply copy all values before 'start'
indicator['infered'][:indicator.
start] = indicator['measured'][:indicator.start]
# filter out invalid entries
levels = numpy.array(
self.levels[indicator.start:]
)[~numpy.isnan(indicator['measured'][indicator.start:])]
values = numpy.abs(indicator['measured'][levels])
if not numpy.isnan(indicator['accuracy'][levels]).any():
weights = 1.0 / indicator['accuracy'][levels]
else:
weights = None
# add offset, if specified
values += offset
# use factor, if specified
values *= factor
# check if sufficiently many measurements are available for inference
if (not exp and len(values) < degree + 1) or (exp and len(values) < 3):
if critical:
self.available = 0
helpers.warning('Inference of indicator \'%s\' not possible!' %
indicator.name)
return
'''
# if only one measurement is available, assign it to all infered level values
if len (values) == 1:
indicator ['infered'] [indicator.start:] = values [0]
return
'''
# nonlinear fit for y = a * exp (b * x) + c
if exp:
# not yet implemented
helpers.warning(
'Inference of indicator \'%s\' using nonlinear fit not implemented!'
% indicator.name)
return
# linear fit for y = a * x + b or y = a * exp (b * x)
else:
# use log-coordinates, if specified
if log:
values = numpy.log(values)
# fit a linear polynomial to absolute valus in log-scale using linear least squares, weighted by data uncertainties
#if not numpy.isnan ( indicator ['accuracy'] [levels] ) .any ():
# line = numpy.polyfit ( levels, values, degree, w = weights )
#else:
line = numpy.polyfit(levels, values, degree)
# get infered maximum likelihood values
infered = numpy.polyval(line, self.levels)
# transform back to linear coordinates
if log:
infered = numpy.exp(infered)
# use factor, if specified
infered /= factor
# subtract offset, if specified
infered -= offset
# respect envelope specifications
if min != None:
infered = numpy.maximum(min, infered)
if max != None:
infered = numpy.minimum(max, infered)
# if inference is enforced, update all indicator values to the infered values
if self.enforce:
indicator['infered'][indicator.start:] = infered[indicator.start:]
# otherwise, update only invalid entries
else:
indicator['infered'][indicator.start:] = indicator['measured'][
indicator.start:]
levels = numpy.array(self.levels[indicator.start:])[numpy.isnan(
indicator['measured'][indicator.start:])]
indicator['infered'][levels] = infered[levels]
def compute(self, mcs, indices, L0):
print
print ' :: Computing INDICATORS...',
sys.stdout.flush()
self.L0 = L0
self.available = 1
# === VALUES and DISTANCES
# evaluates indicators for each level, type and sample for the specified indices
values = self.values(mcs, indices)
# evaluate distances between indicators for every two consecute levels of each sample for the specified indices
distances = self.distances(mcs, indices)
# === MEAN, VARIANCE & DEVIATION indicators
self.mean = [
Indicator('MEAN FINE', self.levels),
Indicator('MEAN COARSE', self.levels, start=self.L0 + 1)
]
self.variance = [
Indicator('VARIANCE FINE', self.levels),
Indicator('VARIANCE COARSE', self.levels, start=self.L0 + 1)
]
self.deviation = [
Indicator('DEVIATION FINE', self.levels),
Indicator('DEVIATION COARSE', self.levels, start=self.L0 + 1)
]
# compute mean, variance and deviation for all levels and types
for level, type in self.levels_types:
#self.mean [type] ['weights'] [level] = numpy.sqrt ( values [level] [type] .size )
#self.variance [type] ['weights'] [level] = numpy.sqrt ( values [level] [type] .size )
self.mean[type]['measured'][level] = numpy.mean(
numpy.abs(values[level][type]))
self.variance[type]['measured'][level] = numpy.var(
values[level][type],
ddof=1) if len(values[level][type]) > 1 else float('nan')
self.deviation[type]['measured'][level] = numpy.std(
values[level][type],
ddof=1) if len(values[level][type]) > 1 else float('nan')
# compute accuracy of mean, variance and deviations indicators for all levels and types
for level, type in self.levels_types:
size = values[level][type].size
self.mean[type]['accuracy'][level] = self.accuracy_mean(
self.deviation[type]['measured'][level], size)
self.variance[type]['accuracy'][level] = self.accuracy_variance(
self.variance[type]['measured'][level], size)
self.deviation[type]['accuracy'][level] = self.accuracy_deviation(
self.deviation[type]['measured'][level], size)
# set the normalization
if numpy.isnan(self.mean[self.FINE]['measured'][self.L0]):
self.normalization = 1
helpers.warning(
'Defaulting \'normalization\' to 1.0 for indicators')
else:
self.normalization = numpy.abs(
self.mean[self.FINE]['measured'][self.L0])
# least squares inference of indicator level values based on the magnitudes of measured level values
self.infer(self.mean[self.FINE],
degree=1,
log=False,
critical=False,
min=0)
self.infer(self.mean[self.COARSE],
degree=1,
log=False,
critical=False,
min=0)
self.infer(self.variance[self.FINE],
degree=1,
log=False,
critical=True,
min=0)
self.infer(self.variance[self.COARSE],
degree=1,
log=False,
critical=True,
min=0)
# === MEAN DIFF and VARIANCE DIFF level distance indicators
# (WITHOUT optimal control variate coefficients computed below)
self.mean_diff = Indicator('MEAN DIFF',
self.levels,
start=self.L0 + 1)
self.variance_diff = Indicator('VARIANCE DIFF',
self.levels,
start=self.L0 + 1)
self.deviation_diff = Indicator('DEVIATION DIFF',
self.levels,
start=self.L0 + 1)
# compute level distances
for level in self.levels:
#self.mean_diff ['weights'] [level] = numpy.sqrt ( distances [level] .size )
#self.variance_diff ['weights'] [level] = numpy.sqrt ( distances [level] .size )
self.mean_diff['measured'][level] = numpy.mean(
numpy.abs(distances[level]))
self.variance_diff['measured'][level] = numpy.var(
distances[level],
ddof=1) if len(distances[level]) > 1 else float('nan')
self.deviation_diff['measured'][level] = numpy.std(
distances[level],
ddof=1) if len(distances[level]) > 1 else float('nan')
# compute accuracy of mean diff and variance diff indicators for all levels and types
for level, type in self.levels_types:
size = distances[level].size
self.mean_diff['accuracy'][level] = self.accuracy_mean(
self.deviation_diff['measured'][level], size)
self.variance_diff['accuracy'][level] = self.accuracy_variance(
self.variance_diff['measured'][level], size)
self.deviation_diff['accuracy'][level] = self.accuracy_deviation(
self.deviation_diff['measured'][level], size)
# least squares inference of indicator level values based on the magnitudes of measured level values
# REMARK: in such case, 'infered' and 'optimal' values are inconsistent (differ even if all coeffs = 1), due to inconsistent computation
#self.infer (self.mean_diff, degree=1, log=True, critical = False, min = 0)
# compute 'mean diff' from infered 'mean'
# REMARK: not very useful, since decay to zero is usually lost in such case (unless inference order > 0)
self.mean_diff['infered'][self.L0] = self.mean[self.FINE]['infered'][
self.L0]
self.mean_diff['infered'][self.L0 + 1:] = numpy.abs(
self.mean[self.FINE]['infered'][self.L0 + 1:] -
self.mean[self.COARSE]['infered'][self.L0 + 1:])
# least squares inference of 'variance diff' indicator level values based on the magnitides of measured level values
if self.inference == 'diffs':
self.infer(self.variance_diff,
degree=1,
log=True,
critical=True,
min=0)
# infered value of VARIANCE DIFF is always by default the infered value of VARIANCE [FINE]
# REMARK: this needs to be after inference, which overwrites non-infered values from the measured ones
self.variance_diff['infered'][self.L0] = self.variance[
self.FINE]['infered'][self.L0]
# === COVARIANCES and CORRELATIONS
self.covariance = Indicator('COVARIANCE',
self.levels,
start=self.L0 + 1)
self.correlation = Indicator('CORRELATION',
self.levels,
start=self.L0 + 1)
# compute covariance and correlation (measured)
# remark: computing covariances and correlations from 'values' leads to inconsistent estimations and should be avoided
for level in self.levels[self.L0 + 1:]:
self.covariance['measured'][level] = 0.5 * (
self.variance[self.FINE]['measured'][level] +
self.variance[self.COARSE]['measured'][level] -
self.variance_diff['measured'][level])
self.correlation['measured'][
level] = self.covariance['measured'][level] / numpy.sqrt(
self.variance[self.FINE]['measured'][level] *
self.variance[self.COARSE]['measured'][level])
# for 'diffs' inference, correlations and covariances are computed from infered 'variance_diff'
if self.inference == 'diffs':
# compute covariance and correlation (infered)
for level in self.levels[self.L0 + 1:]:
self.covariance['infered'][level] = 0.5 * (
self.variance[self.FINE]['infered'][level] +
self.variance[self.COARSE]['infered'][level] -
self.variance_diff['infered'][level])
max_covariance_magnitude = numpy.sqrt(
self.variance[self.FINE]['infered'][level] *
self.variance[self.COARSE]['infered'][level])
if numpy.abs(self.covariance['infered']
[level]) > max_covariance_magnitude:
self.covariance['infered'][level] = numpy.sign(
self.covariance['infered']
[level]) * max_covariance_magnitude
self.correlation['infered'][
level] = self.covariance['infered'][level] / numpy.sqrt(
self.variance[self.FINE]['infered'][level] *
self.variance[self.COARSE]['infered'][level])
#self.correlation ['infered'] [level] = numpy.minimum ( 1, self.correlation ['infered'] [level] )
#self.correlation ['infered'] [level] = numpy.maximum ( -1, self.correlation ['infered'] [level] )
# for 'correlations' inference, 'correlations' is infered and variance diffs with covariances and computed from it
elif self.inference == 'correlations':
# least squares inference of indicator level values based on the magnitides of measured level values
self.infer(self.correlation,
degree=1,
log=True,
offset=-1,
factor=-1,
critical=True,
min=-1.0,
max=1.0)
# compute covariance and variance diffs (infered)
for level in self.levels[self.L0 + 1:]:
self.covariance['infered'][
level] = self.correlation['infered'][level] * numpy.sqrt(
self.variance[self.FINE]['infered'][level] *
self.variance[self.COARSE]['infered'][level])
self.variance_diff['infered'][level] = self.variance[
self.FINE]['infered'][level] + self.variance[
self.COARSE]['infered'][
level] - 2 * self.covariance['infered'][level]
self.variance_diff['infered'][level] = numpy.maximum(
0, self.variance_diff['infered'][level])
# report non-available inference modes
elif self.inference != None:
helpers.error('Inference mode \'%s\' is not implemented' %
self.inference)
print 'done.'
def assemble(self, stats, qois=None, clip=True):
# check if at least one sample at some level
self.available = [
count for count in self.config.samples.counts.loaded if count > 0
] != []
if not self.available:
helpers.error('Statistics can not be assembled')
print
print ' :: ASSEMBLING:'
# report quantities of interest to be assembled
if qois != None:
print ' : Specified qois:',
for qoi in qois.keys():
print qoi,
print
# assemble MC estimates on all levels and types for each statistic
print ' : MC estimates...'
for mc in self.mcs:
# bugfix when self.L0 != 0
if mc.config.level == self.L0 and mc.config.type == self.config.COARSE:
indices = []
else:
indices = self.config.samples.indices.loaded[mc.config.level]
# assemble MC estimate
mc.assemble(stats, indices, qois)
# assemble differences of MC estimates between type = 0 and type = 1 on all levels for each statistic
print ' : Differences of MC estimates...'
self.diffs = [copy.deepcopy(stats) for level in self.config.levels]
# coarsest level difference is just a plain MC estimate (for all statistics)
# TODO: what if this estimate is missing?
for index, stat in enumerate(
self.mcs[self.config.pick[self.L0][self.config.FINE]].stats):
if stat.available:
self.diffs[self.L0][index].estimate = stat.estimate
self.diffs[self.L0][index].available = stat.available
# assemble differences of the remaining levels
for level in self.config.levels[self.L0 + 1:]:
# assemble all statistics
for index, stat in enumerate(self.diffs[level]):
# check if both required components of the difference are available
if not self.mcs[self.config.pick[level][
self.config.FINE]].stats[index].available:
stat.available = 0
continue
if not self.mcs[self.config.pick[level][
self.config.COARSE]].stats[index].available:
stat.available = 0
continue
stat.available = 1
# assemble the FINE term
stat.estimate = self.indicators.coefficients.values[
level] * self.mcs[self.config.pick[level][
self.config.FINE]].stats[index].estimate
# assemble the COARSE term
stat.estimate -= self.indicators.coefficients.values[
level - 1] * self.mcs[self.config.pick[level][
self.config.COARSE]].stats[index].estimate
# assemble MLMC estimates (sum of differences for each statistic)
print ' : MLMC estimates...'
self.stats = copy.deepcopy(stats)
# check if all stats are available for the coarsest level
available = [
stat for stat in self.diffs[self.L0] if not stat.available
] == []
# report if a level is missing
if not available:
helpers.warning(
'Some statistics are not available for the coarsest level %d in MLMC assembly!'
% self.L0)
self.available = 0
return
# copy coarsest difference for each statistic
for index, stat in enumerate(self.stats):
stat.estimate = copy.deepcopy(self.diffs[self.L0][index].estimate)
stat.available = self.diffs[self.L0][index].available
# add remaining differences
for level in self.config.levels[self.L0 + 1:]:
# check if all stats are available
available = [
stat for stat in self.diffs[level] if not stat.available
] == []
# report if a level is missing
if not available:
helpers.warning(
'Level %d is missing in MLMC assembly, leading to an increase in bias!'
% level)
continue
# add up statistics
for index, stat in enumerate(self.stats):
stat.estimate += self.diffs[level][index].estimate
print ' : DONE'
# clip MLMC estimates, if specified
if clip: self.clip()
def update(self):
# loop over simulation iterations
while True:
# load MLMC simulation
self.load()
# deterministic simulations are not suppossed to be updated
if self.config.deterministic:
return
# compute and report error indicators
self.indicators.compute(self.mcs,
self.config.samples.indices.loaded,
self.L0)
self.indicators.report()
# optimize and report error indicators
self.indicators.optimize(self.mcs,
self.config.samples.indices.loaded,
self.L0)
# save error indicators
self.indicators.save(self.config.iteration)
# save coefficients
self.indicators.coefficients.save(self.config.iteration)
# query for progress
helpers.query('Continue?')
# compute, report, and save errors
self.errors.compute(self.indicators, self.config.samples.counts)
self.errors.report()
self.errors.save(self.config.iteration)
# report speedup (MLMC vs MC)
self.errors.speedup(self.indicators, self.config.samples.counts)
# query for progress
helpers.query('Continue?')
# check if we are on the same machine
if self.status.list['cluster'] != local.name:
warning = 'Simulation machine [%s] differs from this machine [%s]' % (
self.status.list['cluster'], local.name)
message = 'Skip update phase?'
if helpers.query(message, warning=warning, exit=0) == 'y':
return
# recursively query user for input for automated optimal sample adjustments
while True:
# return if the simulation is already finished
if self.config.samples.finished(self.errors):
# report final number of samples
self.config.samples.strip()
self.config.samples.report()
print
print ' :: Simulation finished.'
return
# optimize and report error indicators
self.indicators.optimize(self.mcs,
self.config.samples.indices.loaded,
self.L0,
forecast=True)
# update, report, and validate the computed/required/pending number of samples
if self.errors.available and self.indicators.available:
self.config.samples.update(self.errors, self.indicators)
self.config.samples.report()
self.config.samples.validate()
# if samples can not be updated, display warning and skip user input query
else:
helpers.warning(
'indicators or errors not available - samples can not be updated'
)
break
# report forecasted speedup (MLMC vs MC)
self.errors.speedup(self.indicators,
self.config.samples.counts,
forecast=True)
# for interactive sessions
if self.params.query:
# query user for input
modified = self.query()
# if no changes were requested, proceed
if not modified:
break
# for non-interactive sessions, proceed immediately
else:
break
# recursively query user for manual sample specification or adjustment
while True:
# for interactive sessions
if self.params.query:
# query user for input
manual = self.config.samples.manual()
# always report manual sample specification or adjustment and the resulting speedup
if manual:
self.config.samples.available = 1
self.config.samples.report()
self.config.samples.validate()
self.errors.speedup(self.indicators,
self.config.samples.counts,
forecast=True)
# proceed if no specification or adjustment were requested
if not manual:
break
# for non-interactive sessions, proceed immediately
else:
break
# check if samples are available
if not self.config.samples.available:
helpers.warning(
'samples not available -> exiting simulation...')
return
# make indices for the required number of samples
self.config.samples.make()
# distribute required samples
self.config.scheduler.distribute()
# query for progress
if self.params.simulate:
helpers.query(
'Simulate additional job submission (no actual submissions)?',
critical=1)
else:
helpers.query('Submit additional jobs?', critical=1)
# increment iteration
self.config.iteration += 1
# compute required samples
self.run()
# save status of MLMC simulation
self.save()
# for clusters: if non-interactive session -> exit
if local.cluster and not self.params.interactive:
print
print ' :: INFO: Non-interactive mode specified -> exiting.'
print ' : -> Run PyMLMC with \'-i\' option for an interactive mode.'
print
sys.exit()
# otherwise, sleep a bit
else:
time.sleep(5)
def optimize (self, indicators, samples=[]):
# no optimization if only one level is present
if len (self.levels) == 1:
return
# a-priori (work-weighted) optimization
if samples == []:
pairworks = indicators.pairworks / indicators.pairworks [0]
factors = numpy.array (pairworks)
# a-posteriori (sample-weighted) optimization
else:
factors = 1.0 / numpy.maximum ( 1, numpy.array (samples) )
# cost of plain (non-optimized) estimator
cost_plain = self.cost (indicators, factors)
# === if recycling is enabled, coefficients can be computed explicitly
if self.recycle:
# each coefficient can be computed independently
for level in self.levels [ : -1 ]:
self.values [level] = indicators.correlation ['infered'] [level] * numpy.sqrt ( indicators.variance [0] ['infered'] [self.L] / indicators.variance [0] ['infered'] [level] )
# === if recycling is disabled, coefficients are obtained by solving a linear system of equations
else:
# optimization problem as a linear system
A = numpy.zeros ( [self.L, self.L] )
b = numpy.zeros (self.L)
# assemble matrix from indicators
for level in range (self.L):
if level != 0:
A [level] [level - 1] = - factors [level] * indicators.covariance ['infered'] [level]
A [level] [level] = factors [level ] * indicators.variance [0] ['infered'] [level ]
A [level] [level] += factors [level + 1] * indicators.variance [1] ['infered'] [level + 1]
if level != self.L - 1:
A [level] [level + 1] = - factors [level + 1] * indicators.covariance ['infered'] [level + 1]
# assemble right hand side
b [-1] = factors [self.L] * indicators.covariance ['infered'] [self.L]
# solve linear system
self.values [ : -1 ] = numpy.linalg.solve (A, b)
# cost of OCV estimator
cost_ocv = self.cost (indicators, factors)
# compute optimization factor
self.optimization = cost_plain / cost_ocv
# if the result is 'fishy', revert to default values
if self.optimization < 1 or numpy.isnan (self.values).any() or (self.values > 10).any() or (self.values < -10).any():
message = 'Invalid values of optimized coefficients or failed optimization - resetting all to 1.0'
details = ' '.join ( [ helpers.scif (value) for value in self.values ] ) + ', optimization = %.2f' % self.optimization
helpers.warning (message, details=details)
self.values = numpy.ones (self.L+1)
self.optimization = None
