def cost_func( vector ): """ framework: cost function used by minimizer input: numpy.ndarray output: scalar """ start_time = time.time() #set up prior/background and observed data bg_physical = user_driver.get_background() bg_unknown = transform( bg_physical, d.UnknownData ) observed = user_driver.get_observed() unknown = d.UnknownData( vector ) physical = transform( unknown, d.PhysicalData ) has_skipped = False if ( data_access.allow_fwd_skip is True and np.array_equal( vector, data_access.prev_vector ) ): try: model_out = d.ModelOutputData() logger.debug( 'Skipping repeated fwd run.' ) has_skipped = True except AssertionError: logger.debug( 'Tried and failed to skip fwd run.' ) if has_skipped is False: model_in = transform( physical, d.ModelInputData ) model_out = transform( model_in, d.ModelOutputData ) data_access.prev_vector = vector.copy() simulated = transform( model_out, d.ObservationData ) residual = d.ObservationData.get_residual( observed, simulated ) w_residual = d.ObservationData.error_weight( residual ) bg_vector = bg_unknown.get_vector() un_vector = unknown.get_vector() bg_cost = 0.5 * np.sum( ( un_vector - bg_vector )**2 ) res_vector = residual.get_vector() wres_vector = w_residual.get_vector() ob_cost = 0.5 * np.sum( res_vector * wres_vector ) cost = bg_cost + ob_cost unknown.cleanup() physical.cleanup() if data_access.allow_fwd_skip is False: #don't cleanup CMAQ files if we want to reuse them model_in.cleanup() if ( archive_defn.iter_model_output is False and archive_defn.iter_obs_lite is False ): model_out.cleanup() simulated.cleanup() residual.cleanup() w_residual.cleanup() end_time = time.time() logger.info( 'cost = {:} in {:}s'.format( cost, int(end_time-start_time) ) ) return cost
# replace archive directory name and desciption file archive_defn.experiment = 'pert_pert_test' archive_defn.description = """This is a pert-pert test. A "true" prior is assumed and a "true" set of observations calculated from it. A normally distributed perturbation is applied to both the true-prior and the true-obs based of their respected uncertainties. The perterbed prior and perturbed observations are then run through the minimizer to test its ability to converge to the expected cost value of n/2 (where n is the number of observations. """ #create the true and perturbed input data prior_true_archive = 'prior_true.nc' prior_pert_archive = 'prior_pert.nc' obs_true_archive = 'obs_true.pic.gz' obs_pert_archive = 'obs_pert.pic.gz' phys_true = user.get_background() obs_orig = user.get_observed() model_input = transform(phys_true, d.ModelInputData) model_output = transform(model_input, d.ModelOutputData) obs_true = transform(model_output, d.ObservationData) o_val = obs_true.get_vector() o_unc = np.array(d.ObservationData.uncertainty) obs_pert = d.ObservationData(np.random.normal(o_val, o_unc)) unk = transform(phys_true, d.UnknownData) unk_pert = d.UnknownData(np.random.normal(unk.get_vector(), 1.0)) phys_pert = transform(unk_pert, d.PhysicalData) phys_true.archive(prior_true_archive) phys_pert.archive(prior_pert_archive) obs_true.archive(obs_true_archive)
def gradient_func( vector ): """ framework: gradient function used by minimizer input: numpy.ndarray output: numpy.ndarray """ start_time = time.time() #set up prior/background and observed data bg_physical = user_driver.get_background() bg_unknown = transform( bg_physical, d.UnknownData ) observed = user_driver.get_observed() unknown = d.UnknownData( vector ) physical = transform( unknown, d.PhysicalData ) has_skipped = False if ( data_access.allow_fwd_skip is True and np.array_equal( vector, data_access.prev_vector ) ): try: model_out = d.ModelOutputData() logger.debug( 'Skipping repeated fwd run.' ) has_skipped = True except AssertionError: logger.debug( 'Tried and failed to skip fwd run.' ) if has_skipped is False: model_in = transform( physical, d.ModelInputData ) model_out = transform( model_in, d.ModelOutputData ) data_access.prev_vector = vector.copy() simulated = transform( model_out, d.ObservationData ) residual = d.ObservationData.get_residual( observed, simulated ) w_residual = d.ObservationData.error_weight( residual ) adj_forcing = transform( w_residual, d.AdjointForcingData ) sensitivity = transform( adj_forcing, d.SensitivityData ) phys_sense = transform( sensitivity, d.PhysicalAdjointData ) un_gradient = transform( phys_sense, d.UnknownData ) bg_vector = bg_unknown.get_vector() un_vector = unknown.get_vector() bg_grad = un_vector - bg_vector gradient = bg_grad + un_gradient.get_vector() unknown.cleanup() physical.cleanup() if data_access.allow_fwd_skip is False: #don't cleanup CMAQ files if we want to reuse them model_in.cleanup() if ( archive_defn.iter_model_output is False and archive_defn.iter_obs_lite is False ): model_out.cleanup() simulated.cleanup() residual.cleanup() w_residual.cleanup() adj_forcing.cleanup() sensitivity.cleanup() phys_sense.cleanup() un_gradient.cleanup() end_time = time.time() logger.info( 'gradient norm = {:} in {:}s'.format( np.linalg.norm(gradient), int(end_time-start_time) ) ) return np.array( gradient )
import fourdvar.user_driver as user import fourdvar.datadef as d from fourdvar._transform import transform import fourdvar.util.archive_handle as archive import fourdvar.params.archive_defn as archive_defn import fourdvar.util.cmaq_handle as cmaq archive_defn.experiment = 'tmp_grad_verbose' archive_defn.desc_name = '' archive_path = archive.get_archive_path() print 'saving results in:\n{}'.format(archive_path) print 'get observations in ObservationData format' st = time.time() observed = user.get_observed() print 'completed in {}s'.format(int(time.time() - st)) observed.archive('observed.pickle') print 'archived.' print 'get prior in PhysicalData format' st = time.time() prior_phys = user.get_background() print 'completed in {}s'.format(int(time.time() - st)) prior_phys.archive('prior.ncf') print 'archived.' print 'convert prior into UnknownData format' st = time.time() prior_unknown = transform(prior_phys, d.UnknownData) print 'completed in {}s'.format(int(time.time() - st))