def obs_operator_ensemble(istage):
# assumes that prior ensemble is already linked to advance_temp<i>/wrfout_d01
print('running obs operator on ensemble forecast')
os.chdir(cluster.dartrundir)
if sat_channel:
list_ensemble_truths = []
for iens in range(1, exp.n_ens+1):
print('observation operator for ens #'+str(iens))
# ens members are already linked to advance_temp<i>/wrfout_d01
copy(cluster.dartrundir+'/advance_temp'+str(iens)+'/wrfout_d01',
cluster.dartrundir+'/wrfout_d01')
# DART may need a wrfinput file as well, which serves as a template for dimension sizes
symlink(cluster.dartrundir+'/wrfout_d01', cluster.dartrundir+'/wrfinput_d01')
# add geodata, if istage>0, wrfout is DART output (has coords)
if istage == 0:
wrfout_add_geo.run(cluster.dartrundir+'/geo_em.d01.nc', cluster.dartrundir+'/wrfout_d01')
# run perfect_model obs (forward operator)
os.system('mpirun -np 12 ./perfect_model_obs > /dev/null')
# truth values in obs_seq.out are H(x) values
true, _ = read_truth_obs_obsseq(cluster.dartrundir+'/obs_seq.out')
list_ensemble_truths.append(true)
n_obs = len(list_ensemble_truths[0])
np_array = np.full((exp.n_ens, n_obs), np.nan)
for i in range(exp.n_ens):
np_array[i, :] = list_ensemble_truths[i]
return np_array
else:
raise NotImplementedError()
def run_obsdiag(filepaths, f_out='./obsdiag.nc'):
write_input_filelist(filepaths)
for obserr_iszero in ['.true.', '.false.']:
prepare(obserr_iszero=obserr_iszero)
# run_allinoneplace obs_diag
print('------ running obs_diag program')
os.chdir(rundir_program)
symlink(cluster.dart_srcdir + '/obs_diag',
rundir_program + '/obs_diag')
try:
os.remove(rundir_program + '/obs_seq_to_netcdf')
except:
pass
os.system('./obs_diag >& obs_diag.log'
) # caution, this overwrites obs_seq_to_netcdf
# move output to archive
#outdir = os.path.dirname(f_out) #'/'.join(folder_obs_seq_final.split('/')[:-1])
if obserr_iszero == '.true.':
fout = f_out[:-3] + '_wrt_truth.nc'
elif obserr_iszero == '.false.':
fout = f_out[:-3] + '_wrt_obs.nc'
shutil.move(rundir_program + '/obs_diag_output.nc', fout)
print(fout, 'saved.')
def _link_bundle(self, bundle, undo_log):
utils.log("Linking bundle '%s'" % bundle)
for dirpath, dirnames, filenames, relpath in \
self._walk_bundle(bundle):
for dirname in dirnames:
if self._ignore_match(dirname):
continue
src_dirpath = os.path.join(dirpath, dirname)
dst_dirpath = os.path.join(self.root_path, relpath, dirname)
if self._is_directory_tracked(src_dirpath):
utils.symlink(src_dirpath, dst_dirpath,
dry_run=self.dry_run, undo_log=undo_log)
else:
utils.mkdir(dst_dirpath, dry_run=self.dry_run,
undo_log=undo_log)
for filename in filenames:
if self._ignore_match(filename):
continue
src_filename = os.path.join(dirpath, filename)
dst_filename = os.path.join(self.root_path, relpath, filename)
utils.symlink(src_filename, dst_filename,
dry_run=self.dry_run, undo_log=undo_log)
def install(self, **kwargs):
utils.makedirs(self.install_directory)
utils.makedirs(self.packages_directory)
utils.makedirs(self.bin_directory)
Sentinal = utils.SentinalFactory(
os.path.join(self.install_directory, 'sentinal_'), kwargs)
with Sentinal('install_gsl') as sentinal:
if sentinal.unfinished:
with utils.CurrentDirectory(self.packages_directory):
subprocess.check_call(
['wget', 'ftp://ftp.gnu.org/gnu/gsl/gsl-1.16.tar.gz'])
subprocess.check_call(['tar', '-xzvf', 'gsl-1.16.tar.gz'])
with utils.CurrentDirectory('gsl-1.16'):
subprocess.check_call([
'./configure', '--prefix', self.packages_directory
])
subprocess.check_call(['make'])
subprocess.check_call(['make', 'install'])
with Sentinal('install_clonehd') as sentinal:
if sentinal.unfinished:
with utils.CurrentDirectory(self.packages_directory):
subprocess.check_call(['git', 'clone', self.git_repo_url])
with utils.CurrentDirectory('cloneHD'):
subprocess.check_call(
['git', 'checkout', self.git_tag])
utils.makedirs('build')
with utils.CurrentDirectory('src'):
os.environ['CPLUS_INCLUDE_PATH'] = os.path.join(
self.packages_directory, 'include')
subprocess.check_call([
'make',
'GSL=' + os.path.join(self.packages_directory,
'lib', 'libgsl.a') +
' ' + os.path.join(self.packages_directory,
'lib', 'libgslcblas.a'),
'CC=g++'
])
for binary in ('cloneHD', 'filterHD', 'pre-filter'):
binary_filename = os.path.abspath(
os.path.join('build', binary))
utils.symlink(binary_filename,
link_directory=self.bin_directory)
with Sentinal('download_chrom_info') as sentinal:
if sentinal.unfinished:
with utils.CurrentDirectory(self.data_directory):
subprocess.check_call(
'wget ftp://hgdownload.cse.ucsc.edu/goldenPath/hg19/database/chromInfo.txt.gz',
shell=True)
def run(assim_time, background_init_time, exppath_firstguess):
"""Prepares DART files for running filter
i.e.
- links first guess state to DART first guess filenames
- creates wrfinput_d01 files
- adds geo-reference (xlat,xlon) coords so that DART can deal with the files
- writes txt files so DART knows what input and output is
- removes probably pre-existing files which could lead to problems
"""
print('prepare prior state estimate')
for iens in range(1, exp.n_ens + 1):
#wrfout_run = cluster.wrf_rundir(iens) + time.strftime('/wrfout_d01_%Y-%m-%d_%H:%M:%S')
print('link wrfout file to DART background file')
wrfout_run = exppath_firstguess+background_init_time.strftime('/%Y-%m-%d_%H:%M/') \
+str(iens)+assim_time.strftime('/wrfout_d01_%Y-%m-%d_%H:%M:%S')
dart_ensdir = cluster.dartrundir + '/advance_temp' + str(iens)
wrfout_dart = dart_ensdir + '/wrfout_d01'
os.makedirs(dart_ensdir, exist_ok=True)
print('linking', wrfout_run, 'to', wrfout_dart)
symlink(wrfout_run, wrfout_dart)
symlink(wrfout_dart, dart_ensdir + '/wrfinput_d01')
# this seems to be necessary (else wrong level selection)
wrfout_add_geo.run(cluster.dartrundir + '/geo_em.d01.nc', wrfout_dart)
fpath = cluster.dartrundir + '/input_list.txt'
print('writing', fpath)
os.remove(fpath)
with open(fpath, 'w') as f:
for iens in range(1, exp.n_ens + 1):
f.write('./advance_temp' + str(iens) + '/wrfout_d01')
f.write('\n')
fpath = cluster.dartrundir + '/output_list.txt'
print('writing', fpath)
os.remove(fpath)
with open(fpath, 'w') as f:
for iens in range(1, exp.n_ens + 1):
f.write('./filter_restart_d01.' + str(iens).zfill(4))
f.write('\n')
print('removing preassim and filter_restart')
os.system('rm -rf ' + cluster.dartrundir + '/preassim_*')
os.system('rm -rf ' + cluster.dartrundir + '/filter_restart*')
os.system('rm -rf ' + cluster.dartrundir + '/output_mean*')
os.system('rm -rf ' + cluster.dartrundir + '/output_sd*')
os.system('rm -rf ' + cluster.dartrundir + '/perfect_output_*')
os.system('rm -rf ' + cluster.dartrundir + '/obs_seq.fina*')
def save_checkpoint(self,
out_dir,
filename_tmpl='epoch_{}.pth',
save_optimizer=True,
meta=None):
if meta is None:
meta = dict(epoch=self.epoch + 1, iter=self.iter)
else:
meta.update(epoch=self.epoch + 1, iter=self.iter)
filename = osp.join(out_dir, filename_tmpl.format(self.epoch + 1))
linkname = osp.join(out_dir, 'latest.pth')
optimizer = self.optimizer if save_optimizer else None
save_checkpoint(self.model, filename, optimizer=optimizer, meta=meta)
symlink(filename, linkname)
def note_symlink(name, source, alias_dir):
alias_path = os.path.join(alias_dir, name)
if os.path.exists(alias_path):
if os.readlink(alias_path) == source:
logger.debug('not creating symlink "{0}" because it exists'.format(alias_path))
elif not os.path.islink(alias_path):
logger.warning('{0} is not a symbolic link. doing nothing'.format(alias_path))
else:
os.remove(alias_path)
logger.debug('removed stale symlink to {0}'.format(alias_path))
symlink(alias_path, source)
logger.debug('created link from {0} to {1}'.format(source, name))
else:
symlink(alias_path, source)
logger.debug('created link from {0} to {1}'.format(source, name))
def _create_link(input_fn, output_fn):
out_dirname = os.path.dirname(output_fn)
if out_dirname != '' and not os.path.exists(out_dirname):
os.makedirs(out_dirname)
if os.path.islink(output_fn):
os.remove(output_fn)
elif os.path.isdir(output_fn):
abort('[{0}]: {1} exists and is a directory'.format('link', output_fn))
elif os.path.exists(output_fn):
abort('[{0}]: could not create a symlink at {1}.'.format('link', output_fn))
out_base = os.path.basename(output_fn)
if out_base == "":
abort('[{0}]: could not create a symlink at {1}.'.format('link', output_fn))
else:
symlink(out_base, input_fn)
os.rename(out_base, output_fn)
puts('[{0}] created symbolic link pointing to "{1}" named "{2}"'.format('symlink', input_fn, out_base))
def init_fabric(buildsystem, conf_file):
fab_dir = 'fabfile'
if os.path.islink(fab_dir):
os.remove(fab_dir)
elif os.path.isdir(fab_dir):
rmtree(fab_dir)
symlink('fabfile', os.path.join(buildsystem, 'fabfile'))
symlink('fabfile', os.path.join(buildsystem, 'fabfile'))
symlink(name=os.path.join(buildsystem, 'bin', 'docs_meta.yaml'),
target=os.path.join(os.getcwd(), conf_file))
symlink(name=os.path.join(buildsystem, 'fabfile', 'utils.py'),
target=os.path.join(os.path.abspath(buildsystem), 'bin', 'utils.py'))
symlink(name=os.path.join(buildsystem, 'fabfile', 'docs_meta.py'),
target=os.path.join(os.path.abspath(buildsystem), 'bin', 'docs_meta.py'))
def track(self, path, bundle=None):
"""Track a file or a directory."""
# We don't use kwarg default, because None represents default to
# callers
bundle = bundle or 'Default'
src_path = utils.truepath(path)
is_directory = os.path.isdir(src_path)
if self.root_path not in src_path:
raise Exception('Cannot track files outside of root path')
bundle_path = os.path.join(self.repo_path, bundle)
dst_path = os.path.join(
bundle_path, utils.relpath(self.root_path, src_path))
undo_log = []
dst_dir = os.path.dirname(dst_path)
if not os.path.exists(dst_dir):
utils.makedirs(dst_dir, dry_run=self.dry_run, undo_log=undo_log)
try:
utils.rename(src_path, dst_path, dry_run=self.dry_run,
undo_log=undo_log)
try:
utils.symlink(dst_path, src_path, dry_run=self.dry_run)
except:
utils.undo_operations(undo_log)
raise
except:
utils.undo_operations(undo_log)
raise
self.git.add(dst_path)
if is_directory:
marker = self._track_directory(dst_path)
self.git.add(marker)
self.git.commit(message="Tracking '%s'" % path)
print "loading transcripts..."
transcripts = Transcripts()
ts_all, ts_train, ts_test = transcripts.split(limit=DEBUG_LIMIT,
add_all=add_all)
print "loading transcripts (%d train, %d test) ...done." % (len(ts_train),
len(ts_test))
#
# create work_dir structure
#
utils.mkdirs('%s/lexicon' % data_dir)
utils.mkdirs('%s/local/dict' % data_dir)
utils.mkdirs(wav16_dir)
utils.mkdirs(mfcc_dir)
utils.symlink('%s/egs/wsj/s5/steps' % kaldi_root, '%s/steps' % work_dir)
utils.symlink('%s/egs/wsj/s5/utils' % kaldi_root, '%s/utils' % work_dir)
#
# kaldi data part
#
def export_kaldi_data(destdirfn, tsdict):
global wav16_dir
print "Exporting to %s..." % destdirfn
utils.mkdirs(destdirfn)
def link_nature_to_dart_truth(time):
# get wrfout_d01 from nature run
shutil.copy(time.strftime(cluster.nature_wrfout),
cluster.dartrundir+'/wrfout_d01')
# DART may need a wrfinput file as well, which serves as a template for dimension sizes
symlink(cluster.dartrundir+'/wrfout_d01', cluster.dartrundir+'/wrfinput_d01')
import os, sys
from config.cfg import exp, cluster
from utils import symlink
joinp = os.path.join
dart_bin_dir = '/home/fs71386/lkugler/DART/DART_WRF_RTTOV_early_access/models/wrf/work/'
rttov_dir = '/home/fs71386/lkugler/RTTOV/'
# DART executables
bins = ['perfect_model_obs', 'filter', 'obs_diag']
for b in bins:
symlink(joinp(dart_bin_dir, b), joinp(cluster.dartrundir, b))
# DART RTTOV capability
symlink(
'/home/fs71386/lkugler/DART/DART_WRF_RTTOV_early_access/' +
'observations/forward_operators/rttov_sensor_db.csv',
joinp(cluster.dartrundir, 'rttov_sensor_db.csv'))
# Basic MSG4 Seviri
files = [
'rtcoef_rttov12/rttov9pred54L/rtcoef_msg_4_seviri.dat',
'rtcoef_rttov12/cldaer_visir/sccldcoef_msg_4_seviri.dat',
]
for f in files:
symlink(joinp(rttov_dir, f), joinp(cluster.dartrundir,
os.path.basename(f)))
# MFASIS
deff = True
if deff:
def install(self, **kwargs):
utils.makedirs(self.install_directory)
utils.makedirs(self.packages_directory)
utils.makedirs(self.bin_directory)
Sentinal = utils.SentinalFactory(
os.path.join(self.install_directory, 'sentinal_'), kwargs)
r_packages = [
'foreach',
'argparse',
'yaml',
]
for pkg in r_packages:
with Sentinal('install_r_' + pkg) as sentinal:
if sentinal.unfinished:
subprocess.check_call(
'R -q -e "install.packages(\'{0}\', repos=\'http://cran.us.r-project.org\')"'
.format(pkg),
shell=True)
bioconductor_packages = [
'HMMcopy',
'IRanges',
'Rsamtools',
'GenomeInfoDb',
'doMC',
'TitanCNA',
]
for pkg in bioconductor_packages:
with Sentinal('install_r_' + pkg) as sentinal:
if sentinal.unfinished:
subprocess.check_call(
'R -q -e "source(\'http://bioconductor.org/biocLite.R\'); biocLite(\'{0}\')"'
.format(pkg),
shell=True)
with Sentinal('clone_titan') as sentinal:
if sentinal.unfinished:
with utils.CurrentDirectory(self.packages_directory):
subprocess.check_call(
'git clone https://github.com/gavinha/TitanCNA',
shell=True)
with utils.CurrentDirectory('TitanCNA'):
subprocess.check_call(
'git checkout 30fceb911b99a281ccbe3fac29d154f567127410',
shell=True)
subprocess.check_call('R CMD INSTALL TitanCNA', shell=True)
with Sentinal('install_titan_tools') as sentinal:
if sentinal.unfinished:
with utils.CurrentDirectory(self.packages_directory):
subprocess.check_call(
'git clone https://github.com/gavinha/TitanCNA-utils',
shell=True)
with utils.CurrentDirectory('TitanCNA-utils'):
subprocess.check_call(
'git checkout 4cfd6155e620dade4090322d71f45f5a39cb688e',
shell=True)
utils.symlink(
'titan_scripts/createTITANsegmentfiles.pl',
link_directory=self.bin_directory)
with Sentinal('download_chrom_info') as sentinal:
if sentinal.unfinished:
with utils.CurrentDirectory(self.data_directory):
subprocess.check_call(
'wget ftp://hgdownload.cse.ucsc.edu/goldenPath/hg19/database/chromInfo.txt.gz',
shell=True)
import os, sys, shutil
import datetime as dt
from config.cfg import exp, cluster
from utils import symlink, copy, link_contents
import prepare_namelist
for iens in range(1, exp.n_ens+1):
print('preparing ens', iens)
input_prof = (cluster.input_profile).replace('<iens>', str(iens).zfill(3))
rundir = cluster.wrf_rundir(iens)
os.makedirs(rundir, exist_ok=True)
link_contents(cluster.srcdir, rundir)
print('linking ideal and wrf.exe:')
symlink(cluster.ideal, rundir+'/ideal.exe')
symlink(cluster.wrfexe, rundir+'/wrf.exe')
# time not important, but general settings
prepare_namelist.run(iens, begin=dt.datetime(2008, 7, 30, 6, 0),
end=dt.datetime(2008, 7, 30, 6, 30),
archive=False)
symlink(input_prof, rundir+'/input_sounding')
print('finished.')
raise RuntimeError("Can't rename: %s -> %s" % (tmp_name, filepath))
relpath = os.path.relpath(os.path.abspath(path),
os.path.abspath(lnks_path))
new_path = utils.norm_path(relpath, audio.name)
lnk_fmt = "%%0%dd=%%s" % len(str(total))
lnk = lnk_fmt % (i + 1, audio.name)
if sys.platform.startswith('win'):
lnk = ''.join(lnk.rpartition('.')[:-1] + ('lnk',))
lnk_path = utils.norm_path(lnks_path, lnk)
try:
utils.symlink(new_path, lnk_path)
except Exception as e:
raise RuntimeError("Can't create link: %s -> %s"
% (lnk_path, new_path))
logging.debug("[#%d] %s -> %s", i + 1, lnk_path, new_path)
def audio_elem(audio, elem):
t = audio.find(elem).text
return utils.valid_filename(t) if t else u''
def get_id_name(audio):
return utils.basename(audio.find("url").text)
def checkout_and_build_libmecab(basedir):
from os import environ
from os.path import isdir, isfile, join as p_join, exists
from sys import stderr
from time import sleep
from utils import get_parallel_jobs, run, chdir, mkdir_p, touch, symlink
if basedir:
chdir(basedir)
if environ.get('LIBMECAB_DIR'):
# This lets you point at an extracted tarball
run("ls", "-l", environ.get("LIBMECAB_DIR"))
assert exists(
p_join(environ.get("LIBMECAB_DIR"), "mecab", "aclocal.m4"))
assert not exists(LIBMECAB_DIR)
run("rsync", "-avP", "--delete-during",
environ.get('LIBMECAB_DIR') + "/", LIBMECAB_DIR)
run("ls", "-l", LIBMECAB_DIR)
else:
mkdir_p(LIBMECAB_DIR)
if exists(p_join(LIBMECAB_DIR, "mecab", "aclocal.m4")):
# assume already checked out/extracted
chdir(p_join(LIBMECAB_DIR, "mecab"))
else:
chdir(LIBMECAB_DIR)
# You would think git clone would have an option to clone an
# arbitrary <treeish>, but you would be wrong.
run("git", "init")
run("git", "remote", "add", "origin", LIBMECAB_REPO)
run("git", "fetch", "origin", "--depth", "1", LIBMECAB_REV)
run("git", "reset", "--hard", "FETCH_HEAD")
chdir("mecab")
CC, CXX, CFLAGS, CXXFLAGS, MDT = compile_flags_for_platform()
stderr.write(
"+ compile_flags_for_platform\n"
"Building MeCab with:\n {}\n {}\n {}\n {}\n".format(
CC, CXX, CFLAGS, CXXFLAGS))
if MDT is not None:
stderr.write(" MACOSX_DEPLOYMENT_TARGET={}\n".format(MDT))
environ["MACOSX_DEPLOYMENT_TARGET"] = MDT
if not isfile("mecab-config"):
# Not yet configured.
# Adjust time stamps to make sure that Make doesn't think it
# needs to re-run autoconf or automake.
for f in [
"aclocal.m4", "config.h.in", "configure", "Makefile.in",
"src/Makefile.in"
]:
touch(f)
sleep(1)
# We build with the default charset set to UTF-8, but we don't
# disable support for EUC-JP or Shift-JIS.
run("./configure", "--enable-static", "--disable-shared",
"--with-charset=utf8", CC, CXX)
# Only build the actual library, not the utilities.
chdir("src")
run("make", "-j{}".format(get_parallel_jobs()), "libmecab.la", CC, CXX,
CFLAGS, CXXFLAGS)
# Bypass libtool.
if not isfile("libmecab.a"):
symlink(".libs/libmecab.a", "libmecab.a")
import os, sys, shutil
import datetime as dt
from config.cfg import exp, cluster
from utils import symlink, copy
time = dt.datetime.strptime(sys.argv[1], '%Y-%m-%d_%H:%M')
# get wrfout_d01 from nature run
shutil.copy(time.strftime(cluster.nature_wrfout),
cluster.dartrundir+'/wrfout_d01')
import wrfout_add_geo
wrfout_add_geo.run(cluster.dartrundir+'/geo_em.d01.nc', cluster.dartrundir+'/wrfout_d01')
# DART may need a wrfinput file as well, which serves as a template for dimension sizes
symlink(cluster.dartrundir+'/wrfout_d01', cluster.dartrundir+'/wrfinput_d01')