def cost_func( vector ):
"""
framework: cost function used by minimizer
input: numpy.ndarray
output: scalar
"""
start_time = time.time()
#set up prior/background and observed data
bg_physical = user_driver.get_background()
bg_unknown = transform( bg_physical, d.UnknownData )
observed = user_driver.get_observed()
unknown = d.UnknownData( vector )
physical = transform( unknown, d.PhysicalData )
has_skipped = False
if ( data_access.allow_fwd_skip is True and
np.array_equal( vector, data_access.prev_vector ) ):
try:
model_out = d.ModelOutputData()
logger.debug( 'Skipping repeated fwd run.' )
has_skipped = True
except AssertionError:
logger.debug( 'Tried and failed to skip fwd run.' )
if has_skipped is False:
model_in = transform( physical, d.ModelInputData )
model_out = transform( model_in, d.ModelOutputData )
data_access.prev_vector = vector.copy()
simulated = transform( model_out, d.ObservationData )
residual = d.ObservationData.get_residual( observed, simulated )
w_residual = d.ObservationData.error_weight( residual )
bg_vector = bg_unknown.get_vector()
un_vector = unknown.get_vector()
bg_cost = 0.5 * np.sum( ( un_vector - bg_vector )**2 )
res_vector = residual.get_vector()
wres_vector = w_residual.get_vector()
ob_cost = 0.5 * np.sum( res_vector * wres_vector )
cost = bg_cost + ob_cost
unknown.cleanup()
physical.cleanup()
if data_access.allow_fwd_skip is False:
#don't cleanup CMAQ files if we want to reuse them
model_in.cleanup()
if ( archive_defn.iter_model_output is False
and archive_defn.iter_obs_lite is False ):
model_out.cleanup()
simulated.cleanup()
residual.cleanup()
w_residual.cleanup()
end_time = time.time()
logger.info( 'cost = {:} in {:}s'.format( cost, int(end_time-start_time) ) )
return cost
# replace archive directory name and desciption file archive_defn.experiment = 'pert_pert_test' archive_defn.description = """This is a pert-pert test. A "true" prior is assumed and a "true" set of observations calculated from it. A normally distributed perturbation is applied to both the true-prior and the true-obs based of their respected uncertainties. The perterbed prior and perturbed observations are then run through the minimizer to test its ability to converge to the expected cost value of n/2 (where n is the number of observations. """ #create the true and perturbed input data prior_true_archive = 'prior_true.nc' prior_pert_archive = 'prior_pert.nc' obs_true_archive = 'obs_true.pic.gz' obs_pert_archive = 'obs_pert.pic.gz' phys_true = user.get_background() obs_orig = user.get_observed() model_input = transform(phys_true, d.ModelInputData) model_output = transform(model_input, d.ModelOutputData) obs_true = transform(model_output, d.ObservationData) o_val = obs_true.get_vector() o_unc = np.array(d.ObservationData.uncertainty) obs_pert = d.ObservationData(np.random.normal(o_val, o_unc)) unk = transform(phys_true, d.UnknownData) unk_pert = d.UnknownData(np.random.normal(unk.get_vector(), 1.0)) phys_pert = transform(unk_pert, d.PhysicalData) phys_true.archive(prior_true_archive) phys_pert.archive(prior_pert_archive) obs_true.archive(obs_true_archive)
def gradient_func( vector ):
"""
framework: gradient function used by minimizer
input: numpy.ndarray
output: numpy.ndarray
"""
start_time = time.time()
#set up prior/background and observed data
bg_physical = user_driver.get_background()
bg_unknown = transform( bg_physical, d.UnknownData )
observed = user_driver.get_observed()
unknown = d.UnknownData( vector )
physical = transform( unknown, d.PhysicalData )
has_skipped = False
if ( data_access.allow_fwd_skip is True and
np.array_equal( vector, data_access.prev_vector ) ):
try:
model_out = d.ModelOutputData()
logger.debug( 'Skipping repeated fwd run.' )
has_skipped = True
except AssertionError:
logger.debug( 'Tried and failed to skip fwd run.' )
if has_skipped is False:
model_in = transform( physical, d.ModelInputData )
model_out = transform( model_in, d.ModelOutputData )
data_access.prev_vector = vector.copy()
simulated = transform( model_out, d.ObservationData )
residual = d.ObservationData.get_residual( observed, simulated )
w_residual = d.ObservationData.error_weight( residual )
adj_forcing = transform( w_residual, d.AdjointForcingData )
sensitivity = transform( adj_forcing, d.SensitivityData )
phys_sense = transform( sensitivity, d.PhysicalAdjointData )
un_gradient = transform( phys_sense, d.UnknownData )
bg_vector = bg_unknown.get_vector()
un_vector = unknown.get_vector()
bg_grad = un_vector - bg_vector
gradient = bg_grad + un_gradient.get_vector()
unknown.cleanup()
physical.cleanup()
if data_access.allow_fwd_skip is False:
#don't cleanup CMAQ files if we want to reuse them
model_in.cleanup()
if ( archive_defn.iter_model_output is False
and archive_defn.iter_obs_lite is False ):
model_out.cleanup()
simulated.cleanup()
residual.cleanup()
w_residual.cleanup()
adj_forcing.cleanup()
sensitivity.cleanup()
phys_sense.cleanup()
un_gradient.cleanup()
end_time = time.time()
logger.info( 'gradient norm = {:} in {:}s'.format( np.linalg.norm(gradient),
int(end_time-start_time) ) )
return np.array( gradient )
import fourdvar.user_driver as user
import fourdvar.datadef as d
from fourdvar._transform import transform
import fourdvar.util.archive_handle as archive
import fourdvar.params.archive_defn as archive_defn
import fourdvar.util.cmaq_handle as cmaq
archive_defn.experiment = 'tmp_grad_verbose'
archive_defn.desc_name = ''
archive_path = archive.get_archive_path()
print 'saving results in:\n{}'.format(archive_path)
print 'get observations in ObservationData format'
st = time.time()
observed = user.get_observed()
print 'completed in {}s'.format(int(time.time() - st))
observed.archive('observed.pickle')
print 'archived.'
print 'get prior in PhysicalData format'
st = time.time()
prior_phys = user.get_background()
print 'completed in {}s'.format(int(time.time() - st))
prior_phys.archive('prior.ncf')
print 'archived.'
print 'convert prior into UnknownData format'
st = time.time()
prior_unknown = transform(prior_phys, d.UnknownData)
print 'completed in {}s'.format(int(time.time() - st))