def archive(self, dirname=None):
"""
extension: save copy of files to archive/experiment directory
input: string or None
output: None
notes: this will overwrite any clash of namespace.
if input is None file will write to experiment directory
else it will create dirname in experiment directory and save there.
"""
save_path = get_archive_path()
if dirname is not None:
save_path = os.path.join(save_path, dirname)
ensure_path(save_path, inc_file=False)
for record in self.file_data.values():
source = record['actual']
dest = os.path.join(save_path, record['archive'])
ncf.copy_compress(source, dest)
return None
You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and limitations under the License.
"""
import os
import context
from obs_preprocess.obs_defn import ObsInstantRay
from obs_preprocess.model_space import ModelSpace
import fourdvar.util.file_handle as fh
import fourdvar.params.input_defn as input_defn
# save new obs file as fourdvar input file
save_file = input_defn.obs_file
fh.ensure_path(os.path.dirname(save_file))
# define the parameters of the test observations (1 obs in this example)
obs_param = {
'date': 20070610, # YYYYMMSS
'time': 60000, # HHMMSS
'lat': 34.2, # latitude
'lon': -84.0, # longitude
'spc': 'CO2', # species
'val': 400., # concentration (ppm)
'unc': 1., # uncertainty (ppm)
'interp': False
} # interpolate between 2 nearest timesteps
# convert obs_param into needed input units
model_grid = ModelSpace.create_from_fourdvar()
def run_fwd_single(date, is_first):
"""
extension: run cmaq fwd for a single day
input: dt.date, Boolean (is this day the first of the model)
output: None
"""
env_dict = setup_run()
env_dict['PERTCOLS'] = cfg.pertcols
env_dict['PERTROWS'] = cfg.pertrows
env_dict['PERTLEVS'] = cfg.pertlevs
env_dict['PERTSPCS'] = cfg.pertspcs
env_dict['PERTDELT'] = cfg.pertdelt
env_dict['CTM_APPL'] = cfg.fwd_appl
env_dict['CTM_XFIRST_OUT'] = cfg.fwd_xfirst_file
env_dict['LOGFILE'] = cfg.fwd_logfile
env_dict['FLOOR_FILE'] = cfg.floor_file
env_dict['CTM_PROGNAME'] = cfg.fwd_prog
if is_first is True:
env_dict['INIT_GASC_1'] = cfg.icon_file
env_dict['INIT_AERO_1'] = cfg.icon_file
env_dict['INIT_NONR_1'] = cfg.icon_file
env_dict['INIT_TRAC_1'] = cfg.icon_file
env_dict['CTM_XFIRST_IN'] = ''
else:
prev_grid = dt.move_tag(cfg.last_grid_file, -1)
prev_xfirst = dt.move_tag(cfg.fwd_xfirst_file, -1)
env_dict['INIT_GASC_1'] = prev_grid
env_dict['INIT_AERO_1'] = prev_grid
env_dict['INIT_NONR_1'] = prev_grid
env_dict['INIT_TRAC_1'] = prev_grid
env_dict['CTM_XFIRST_IN'] = prev_xfirst
env_dict = parse_env_dict(env_dict, date)
load_env(env_dict)
run_cmd = cfg.cmd_preamble
if int(cfg.npcol) != 1 or int(cfg.nprow) != 1:
#use mpi
run_cmd += 'mpirun -np {:} '.format(int(cfg.npcol) * int(cfg.nprow))
run_cmd += cfg.fwd_prog
stdout_fname = dt.replace_date(cfg.fwd_stdout_log, date)
fh.ensure_path(stdout_fname, inc_file=True)
with open(stdout_fname, 'w') as stdout_file:
msg = 'calling external process:\n{:}> {:}'.format(
cfg.cmd_shell, run_cmd)
logger.debug(msg)
statcode = subprocess.call(run_cmd,
stdout=stdout_file,
stderr=subprocess.STDOUT,
shell=True,
executable=cfg.cmd_shell)
logger.debug('external process finished.')
if statcode != 0:
msg = 'cmaq fwd failed on {}.'.format(date.strftime('%Y%m%d'))
logger.error(msg)
raise AssertionError(msg)
clean_env(env_dict)
return None
def run_bwd_single(date, is_first):
"""
extension: run cmaq bwd for a single day
input: dt.date, Boolean (is this the first time called)
output: None
"""
env_dict = setup_run()
env_dict['CTM_APPL'] = cfg.bwd_appl
env_dict['CTM_XFIRST_OUT'] = cfg.bwd_xfirst_file
env_dict['CTM_XFIRST_IN'] = cfg.fwd_xfirst_file
env_dict['LOGFILE'] = cfg.bwd_logfile
env_dict['CTM_PROGNAME'] = cfg.bwd_prog
env_dict['CHK_PATH'] = cfg.chk_path
env_dict['INIT_GASC_1'] = cfg.last_grid_file + ' -v'
env_dict['INIT_AERO_1'] = cfg.last_grid_file + ' -v'
env_dict['INIT_NONR_1'] = cfg.last_grid_file + ' -v'
env_dict['INIT_TRAC_1'] = cfg.last_grid_file + ' -v'
env_dict['CTM_CONC_FWD'] = cfg.conc_file + ' -v'
env_dict['CTM_CGRID_FWD'] = cfg.last_grid_file + ' -v'
env_dict['ADJ_LGRID'] = cfg.conc_sense_file + ' -v'
env_dict['ADJ_LGRID_EM'] = cfg.emis_sense_file + ' -v'
env_dict['ADJ_LGRID_EM_SF'] = cfg.emis_scale_sense_file + ' -v'
env_dict['ADJ_FORCE'] = cfg.force_file
if cfg.sense_sync is True:
env_dict['ADJ_LGRID_FREQ'] = 'SYNC_STEP'
else:
env_dict['ADJ_LGRID_FREQ'] = 'OUTPUT_STEP'
if str(cfg.force_lays).strip().lower() == 'template':
frclays = int(ncf.get_attr(template.force, 'NLAYS'))
env_dict['NLAYS_FRC'] = str(frclays)
else:
env_dict['NLAYS_FRC'] = str(cfg.force_lays)
if str(cfg.sense_emis_lays).strip().lower() == 'template':
emsensl = int(ncf.get_attr(template.sense_emis, 'NLAYS'))
env_dict['CTM_EMSENSL'] = str(emsensl)
else:
env_dict['CTM_EMSENSL'] = str(cfg.sense_emis_lays)
if is_first is not True:
prev_conc = dt.move_tag(cfg.conc_sense_file, 1)
prev_emis = dt.move_tag(cfg.emis_sense_file, 1)
prev_scale = dt.move_tag(cfg.emis_scale_sense_file, 1)
env_dict['INIT_LGRID_1'] = prev_conc
env_dict['INIT_EM_1'] = prev_emis
env_dict['INIT_EM_SF_1'] = prev_scale
env_dict = parse_env_dict(env_dict, date)
load_env(env_dict)
run_cmd = cfg.cmd_preamble
if int(cfg.npcol) != 1 or int(cfg.nprow) != 1:
#use mpi
run_cmd += 'mpirun -np {:} '.format(int(cfg.npcol) * int(cfg.nprow))
run_cmd += cfg.bwd_prog
stdout_fname = dt.replace_date(cfg.bwd_stdout_log, date)
fh.ensure_path(stdout_fname, inc_file=True)
with open(stdout_fname, 'w') as stdout_file:
msg = 'calling external process:\n{:}> {:}'.format(
cfg.cmd_shell, run_cmd)
logger.debug(msg)
statcode = subprocess.call(run_cmd,
stdout=stdout_file,
stderr=subprocess.STDOUT,
shell=True,
executable=cfg.cmd_shell)
logger.debug('external process finished.')
if statcode != 0:
msg = 'cmaq bwd failed on {}.'.format(date.strftime('%Y%m%d'))
logger.error(msg)
raise AssertionError(msg)
clean_env(env_dict)
return None
if str(cmaq_config.sense_emis_lays).lower() == 'template':
sense_lay = int(ncf.get_attr(icon_file, 'NLAYS'))
cmaq_config.sense_emis_lays = str(sense_lay)
# generate sample files by running 1 day of cmaq (fwd & bwd)
cmaq_handle.wipeout_fwd()
cmaq_handle.run_fwd_single(dt.start_date, is_first=True)
# make force file with same attr as conc and all data zeroed
conc_spcs = ncf.get_attr(conc_file, 'VAR-LIST').split()
conc_data = ncf.get_variable(conc_file, conc_spcs)
force_data = {k: np.zeros(v.shape) for k, v in conc_data.items()}
ncf.create_from_template(conc_file, force_file, force_data)
cmaq_handle.run_bwd_single(dt.start_date, is_first=True)
# create record for icon & emis files
fh.ensure_path(os.path.dirname(template.icon))
ncf.copy_compress(icon_file, template.icon)
for date in dt.get_datelist():
emis_src = dt.replace_date(cmaq_config.emis_file, date)
emis_dst = dt.replace_date(template.emis, date)
fh.ensure_path(os.path.dirname(emis_dst))
ncf.copy_compress(emis_src, emis_dst)
# create template for conc, force & sense files
fh.ensure_path(os.path.dirname(template.conc))
fh.ensure_path(os.path.dirname(template.force))
fh.ensure_path(os.path.dirname(template.sense_emis))
fh.ensure_path(os.path.dirname(template.sense_conc))
ncf.copy_compress(conc_file, template.conc)
ncf.copy_compress(force_file, template.force)
ncf.copy_compress(sense_emis_file, template.sense_emis)