def cost_func( vector ):
"""
framework: cost function used by minimizer
input: numpy.ndarray
output: scalar
"""
start_time = time.time()
#set up prior/background and observed data
bg_physical = user_driver.get_background()
bg_unknown = transform( bg_physical, d.UnknownData )
observed = user_driver.get_observed()
unknown = d.UnknownData( vector )
physical = transform( unknown, d.PhysicalData )
has_skipped = False
if ( data_access.allow_fwd_skip is True and
np.array_equal( vector, data_access.prev_vector ) ):
try:
model_out = d.ModelOutputData()
logger.debug( 'Skipping repeated fwd run.' )
has_skipped = True
except AssertionError:
logger.debug( 'Tried and failed to skip fwd run.' )
if has_skipped is False:
model_in = transform( physical, d.ModelInputData )
model_out = transform( model_in, d.ModelOutputData )
data_access.prev_vector = vector.copy()
simulated = transform( model_out, d.ObservationData )
residual = d.ObservationData.get_residual( observed, simulated )
w_residual = d.ObservationData.error_weight( residual )
bg_vector = bg_unknown.get_vector()
un_vector = unknown.get_vector()
bg_cost = 0.5 * np.sum( ( un_vector - bg_vector )**2 )
res_vector = residual.get_vector()
wres_vector = w_residual.get_vector()
ob_cost = 0.5 * np.sum( res_vector * wres_vector )
cost = bg_cost + ob_cost
unknown.cleanup()
physical.cleanup()
if data_access.allow_fwd_skip is False:
#don't cleanup CMAQ files if we want to reuse them
model_in.cleanup()
if ( archive_defn.iter_model_output is False
and archive_defn.iter_obs_lite is False ):
model_out.cleanup()
simulated.cleanup()
residual.cleanup()
w_residual.cleanup()
end_time = time.time()
logger.info( 'cost = {:} in {:}s'.format( cost, int(end_time-start_time) ) )
return cost
def partial_adjoint(vector):
unknown = d.UnknownData(vector)
physical = transform(unknown, d.PhysicalData)
model_input = transform(physical, d.ModelInputData)
model_output = fwd_no_transport(model_input)
adjoint_forcing = make_forcing()
sensitivity = bwd_no_transport(adjoint_forcing)
phys_adjoint = transform(sensitivity, d.PhysicalAdjointData)
unk_adjoint = transform(phys_adjoint, d.UnknownData)
vec_adjoint = unk_adjoint.get_vector()
return vec_adjoint
def make_perturbation(init_vector, scale):
"""Increase block of values by uncertainty*scale at:
- The first time-step of PhysicalData
- Only on the surface layer
- In the middle of the domain
- For every species
"""
unknown = d.UnknownData(init_vector)
p = transform(unknown, d.PhysicalData)
#edit physical (p)
r, c = int(p.nrows / 3), int(p.ncols / 3)
for spc in p.spcs:
p.emis[spc][0, 0, r:2 * r,
c:2 * c] += scale * p.emis_unc[spc][0, 0, r:2 * r, c:2 * c]
#convert perturbed phys back into vector
pert_unknown = transform(p, d.UnknownData)
return pert_unknown.get_vector()
def get_answer():
"""
framework: run the minimizer & display results from user_driver module
input: None
output: None (user_driver.display should print/save output as desired)
"""
#set up background unknowns
bg_physical = user_driver.get_background()
bg_unknown = transform( bg_physical, d.UnknownData )
user_driver.setup()
start_vector = bg_unknown.get_vector()
min_output = user_driver.minim( cost_func, gradient_func, start_vector )
out_vector = min_output[0]
out_unknown = d.UnknownData( out_vector )
out_physical = transform( out_unknown, d.PhysicalData )
user_driver.post_process( out_physical, min_output[1:] )
out_unknown.cleanup()
out_physical.cleanup()
user_driver.cleanup()
return None
def callback_func( current_vector ):
"""
extension: called once for every iteration of minimizer
input: np.array
output: None
"""
global iter_num
iter_num += 1
current_unknown = d.UnknownData( current_vector )
current_physical = transform( current_unknown, d.PhysicalData )
current_physical.archive( 'iter{:04}.ncf'.format( iter_num ) )
if archive_defn.iter_model_output is True:
current_model_output = d.ModelOutputData()
current_model_output.archive( 'conc_iter{:04}.ncf'.format(iter_num) )
if archive_defn.iter_obs_lite is True:
current_model_output = d.ModelOutputData()
current_obs = transform( current_model_output, d.ObservationData )
current_obs.archive( 'obs_lite_iter{:04}.pic.gz'.format( iter_num ),
force_lite=True )
logger.info( 'iter_num = {}'.format( iter_num ) )
return None
def finite_diff(scale):
prior_phys = user.get_background()
unknowns = transform(prior_phys, d.UnknownData)
init_vector = unknowns.get_vector()
init_gradient = partial_adjoint(init_vector)
d.ModelOutputData().archive('init_conc')
pert_vector = make_perturbation(init_vector, scale)
pert_gradient = partial_adjoint(pert_vector)
d.ModelOutputData().archive('pert_conc')
d.AdjointForcingData().archive('force')
eps = 1e-6
if abs(pert_gradient -
init_gradient).sum() > eps * abs(pert_gradient).sum():
print "WARNING: pert & init gradients differ."
print "init gradient norm = {:}".format(np.linalg.norm(init_gradient))
print "pert gradient norm = {:}".format(np.linalg.norm(pert_gradient))
pert_diff = pert_vector - init_vector
sense_score = .5 * ((pert_diff * init_gradient).sum() +
(pert_diff * pert_gradient).sum())
force_score = 0.
iconc_file = os.path.join(archive_path, 'init_conc',
archive_defn.conc_file)
pconc_file = os.path.join(archive_path, 'pert_conc',
archive_defn.conc_file)
force_file = os.path.join(archive_path, 'force', archive_defn.force_file)
for date in dt.get_datelist():
iconc = ncf.get_variable(dt.replace_date(iconc_file, date), spcs_list)
pconc = ncf.get_variable(dt.replace_date(pconc_file, date), spcs_list)
force = ncf.get_variable(dt.replace_date(force_file, date), spcs_list)
c_diff = {s: pconc[s] - iconc[s] for s in spcs_list}
force_score += sum([(c_diff[s] * force[s]).sum() for s in spcs_list])
return sense_score, force_score
archive_defn.experiment = 'pert_pert_test' archive_defn.description = """This is a pert-pert test. A "true" prior is assumed and a "true" set of observations calculated from it. A normally distributed perturbation is applied to both the true-prior and the true-obs based of their respected uncertainties. The perterbed prior and perturbed observations are then run through the minimizer to test its ability to converge to the expected cost value of n/2 (where n is the number of observations. """ #create the true and perturbed input data prior_true_archive = 'prior_true.nc' prior_pert_archive = 'prior_pert.nc' obs_true_archive = 'obs_true.pic.gz' obs_pert_archive = 'obs_pert.pic.gz' phys_true = user.get_background() obs_orig = user.get_observed() model_input = transform(phys_true, d.ModelInputData) model_output = transform(model_input, d.ModelOutputData) obs_true = transform(model_output, d.ObservationData) o_val = obs_true.get_vector() o_unc = np.array(d.ObservationData.uncertainty) obs_pert = d.ObservationData(np.random.normal(o_val, o_unc)) unk = transform(phys_true, d.UnknownData) unk_pert = d.UnknownData(np.random.normal(unk.get_vector(), 1.0)) phys_pert = transform(unk_pert, d.PhysicalData) phys_true.archive(prior_true_archive) phys_pert.archive(prior_pert_archive) obs_true.archive(obs_true_archive) obs_pert.archive(obs_pert_archive)
def gradient_func( vector ):
"""
framework: gradient function used by minimizer
input: numpy.ndarray
output: numpy.ndarray
"""
start_time = time.time()
#set up prior/background and observed data
bg_physical = user_driver.get_background()
bg_unknown = transform( bg_physical, d.UnknownData )
observed = user_driver.get_observed()
unknown = d.UnknownData( vector )
physical = transform( unknown, d.PhysicalData )
has_skipped = False
if ( data_access.allow_fwd_skip is True and
np.array_equal( vector, data_access.prev_vector ) ):
try:
model_out = d.ModelOutputData()
logger.debug( 'Skipping repeated fwd run.' )
has_skipped = True
except AssertionError:
logger.debug( 'Tried and failed to skip fwd run.' )
if has_skipped is False:
model_in = transform( physical, d.ModelInputData )
model_out = transform( model_in, d.ModelOutputData )
data_access.prev_vector = vector.copy()
simulated = transform( model_out, d.ObservationData )
residual = d.ObservationData.get_residual( observed, simulated )
w_residual = d.ObservationData.error_weight( residual )
adj_forcing = transform( w_residual, d.AdjointForcingData )
sensitivity = transform( adj_forcing, d.SensitivityData )
phys_sense = transform( sensitivity, d.PhysicalAdjointData )
un_gradient = transform( phys_sense, d.UnknownData )
bg_vector = bg_unknown.get_vector()
un_vector = unknown.get_vector()
bg_grad = un_vector - bg_vector
gradient = bg_grad + un_gradient.get_vector()
unknown.cleanup()
physical.cleanup()
if data_access.allow_fwd_skip is False:
#don't cleanup CMAQ files if we want to reuse them
model_in.cleanup()
if ( archive_defn.iter_model_output is False
and archive_defn.iter_obs_lite is False ):
model_out.cleanup()
simulated.cleanup()
residual.cleanup()
w_residual.cleanup()
adj_forcing.cleanup()
sensitivity.cleanup()
phys_sense.cleanup()
un_gradient.cleanup()
end_time = time.time()
logger.info( 'gradient norm = {:} in {:}s'.format( np.linalg.norm(gradient),
int(end_time-start_time) ) )
return np.array( gradient )
st = time.time()
observed = user.get_observed()
print 'completed in {}s'.format(int(time.time() - st))
observed.archive('observed.pickle')
print 'archived.'
print 'get prior in PhysicalData format'
st = time.time()
prior_phys = user.get_background()
print 'completed in {}s'.format(int(time.time() - st))
prior_phys.archive('prior.ncf')
print 'archived.'
print 'convert prior into UnknownData format'
st = time.time()
prior_unknown = transform(prior_phys, d.UnknownData)
print 'completed in {}s'.format(int(time.time() - st))
print 'get unknowns in vector form.'
st = time.time()
prior_vector = prior_unknown.get_vector()
print 'completed in {}s'.format(int(time.time() - st))
print 'perturb vector to produce mock input for gradient_func'
st = time.time()
test_vector = prior_vector + np.random.normal(0.0, 1.0, prior_vector.shape)
print 'completed in {}s'.format(int(time.time() - st))
print '\ncopy logic of gradient function.\n'
print 'convert input vector into UnknownData format'
st = time.time()
observed = user.get_observed()
print 'completed in {}s'.format(int(time.time() - st))
observed.archive('observed.pickle')
print 'archived.'
print 'get prior in PhysicalData format'
st = time.time()
prior_phys = user.get_background()
print 'completed in {}s'.format(int(time.time() - st))
prior_phys.archive('prior.ncf')
print 'archived.'
print 'convert prior into UnknownData format'
st = time.time()
prior_unknown = transform(prior_phys, d.UnknownData)
print 'completed in {}s'.format(int(time.time() - st))
print 'get unknowns in vector form.'
st = time.time()
prior_vector = prior_unknown.get_vector()
print 'completed in {}s'.format(int(time.time() - st))
print 'perturb vector to produce mock input for cost_func'
st = time.time()
test_vector = prior_vector + np.random.normal(0.0, 1.0, prior_vector.shape)
print 'completed in {}s'.format(int(time.time() - st))
print '\ncopy logic of least squares cost function.\n'
print 'convert input vector into UnknownData format'
