Exemple #1
0
def obs_operator_ensemble(istage):
    # assumes that prior ensemble is already linked to advance_temp<i>/wrfout_d01
    print('running obs operator on ensemble forecast')
    os.chdir(cluster.dartrundir)

    if sat_channel:
        list_ensemble_truths = []

        for iens in range(1, exp.n_ens+1):
            print('observation operator for ens #'+str(iens))
            # ens members are already linked to advance_temp<i>/wrfout_d01
            copy(cluster.dartrundir+'/advance_temp'+str(iens)+'/wrfout_d01',
                 cluster.dartrundir+'/wrfout_d01')
            # DART may need a wrfinput file as well, which serves as a template for dimension sizes
            symlink(cluster.dartrundir+'/wrfout_d01', cluster.dartrundir+'/wrfinput_d01')
            
            # add geodata, if istage>0, wrfout is DART output (has coords)
            if istage == 0:
                wrfout_add_geo.run(cluster.dartrundir+'/geo_em.d01.nc', cluster.dartrundir+'/wrfout_d01')

            # run perfect_model obs (forward operator)
            os.system('mpirun -np 12 ./perfect_model_obs > /dev/null')

            # truth values in obs_seq.out are H(x) values
            true, _ = read_truth_obs_obsseq(cluster.dartrundir+'/obs_seq.out')
            list_ensemble_truths.append(true)
        
        n_obs = len(list_ensemble_truths[0])
        np_array = np.full((exp.n_ens, n_obs), np.nan)
        for i in range(exp.n_ens):
            np_array[i, :] = list_ensemble_truths[i]
        return np_array
    else:
        raise NotImplementedError()
Exemple #2
0
def run_obsdiag(filepaths, f_out='./obsdiag.nc'):
    write_input_filelist(filepaths)

    for obserr_iszero in ['.true.', '.false.']:
        prepare(obserr_iszero=obserr_iszero)

        # run_allinoneplace obs_diag
        print('------ running obs_diag program')
        os.chdir(rundir_program)
        symlink(cluster.dart_srcdir + '/obs_diag',
                rundir_program + '/obs_diag')
        try:
            os.remove(rundir_program + '/obs_seq_to_netcdf')
        except:
            pass
        os.system('./obs_diag >& obs_diag.log'
                  )  # caution, this overwrites obs_seq_to_netcdf

        # move output to archive
        #outdir = os.path.dirname(f_out)  #'/'.join(folder_obs_seq_final.split('/')[:-1])
        if obserr_iszero == '.true.':
            fout = f_out[:-3] + '_wrt_truth.nc'
        elif obserr_iszero == '.false.':
            fout = f_out[:-3] + '_wrt_obs.nc'
        shutil.move(rundir_program + '/obs_diag_output.nc', fout)
        print(fout, 'saved.')
Exemple #3
0
    def _link_bundle(self, bundle, undo_log):
        utils.log("Linking bundle '%s'" % bundle)

        for dirpath, dirnames, filenames, relpath in \
                self._walk_bundle(bundle):

            for dirname in dirnames:
                if self._ignore_match(dirname):
                    continue
                src_dirpath = os.path.join(dirpath, dirname)
                dst_dirpath = os.path.join(self.root_path, relpath, dirname)
                if self._is_directory_tracked(src_dirpath):
                    utils.symlink(src_dirpath, dst_dirpath,
                                  dry_run=self.dry_run, undo_log=undo_log)
                else:
                    utils.mkdir(dst_dirpath, dry_run=self.dry_run,
                                undo_log=undo_log)

            for filename in filenames:
                if self._ignore_match(filename):
                    continue
                src_filename = os.path.join(dirpath, filename)
                dst_filename = os.path.join(self.root_path, relpath, filename)
                utils.symlink(src_filename, dst_filename,
                              dry_run=self.dry_run, undo_log=undo_log)
Exemple #4
0
    def install(self, **kwargs):

        utils.makedirs(self.install_directory)
        utils.makedirs(self.packages_directory)
        utils.makedirs(self.bin_directory)

        Sentinal = utils.SentinalFactory(
            os.path.join(self.install_directory, 'sentinal_'), kwargs)

        with Sentinal('install_gsl') as sentinal:
            if sentinal.unfinished:
                with utils.CurrentDirectory(self.packages_directory):
                    subprocess.check_call(
                        ['wget', 'ftp://ftp.gnu.org/gnu/gsl/gsl-1.16.tar.gz'])
                    subprocess.check_call(['tar', '-xzvf', 'gsl-1.16.tar.gz'])
                    with utils.CurrentDirectory('gsl-1.16'):
                        subprocess.check_call([
                            './configure', '--prefix', self.packages_directory
                        ])
                        subprocess.check_call(['make'])
                        subprocess.check_call(['make', 'install'])

        with Sentinal('install_clonehd') as sentinal:
            if sentinal.unfinished:
                with utils.CurrentDirectory(self.packages_directory):
                    subprocess.check_call(['git', 'clone', self.git_repo_url])
                    with utils.CurrentDirectory('cloneHD'):
                        subprocess.check_call(
                            ['git', 'checkout', self.git_tag])
                        utils.makedirs('build')
                        with utils.CurrentDirectory('src'):
                            os.environ['CPLUS_INCLUDE_PATH'] = os.path.join(
                                self.packages_directory, 'include')
                            subprocess.check_call([
                                'make',
                                'GSL=' + os.path.join(self.packages_directory,
                                                      'lib', 'libgsl.a') +
                                ' ' + os.path.join(self.packages_directory,
                                                   'lib', 'libgslcblas.a'),
                                'CC=g++'
                            ])
                        for binary in ('cloneHD', 'filterHD', 'pre-filter'):
                            binary_filename = os.path.abspath(
                                os.path.join('build', binary))
                            utils.symlink(binary_filename,
                                          link_directory=self.bin_directory)

        with Sentinal('download_chrom_info') as sentinal:
            if sentinal.unfinished:
                with utils.CurrentDirectory(self.data_directory):
                    subprocess.check_call(
                        'wget ftp://hgdownload.cse.ucsc.edu/goldenPath/hg19/database/chromInfo.txt.gz',
                        shell=True)
Exemple #5
0
def run(assim_time, background_init_time, exppath_firstguess):
    """Prepares DART files for running filter 
    i.e.
    - links first guess state to DART first guess filenames
    - creates wrfinput_d01 files
    - adds geo-reference (xlat,xlon) coords so that DART can deal with the files
    - writes txt files so DART knows what input and output is
    - removes probably pre-existing files which could lead to problems
    """

    print('prepare prior state estimate')
    for iens in range(1, exp.n_ens + 1):
        #wrfout_run = cluster.wrf_rundir(iens) + time.strftime('/wrfout_d01_%Y-%m-%d_%H:%M:%S')

        print('link wrfout file to DART background file')
        wrfout_run = exppath_firstguess+background_init_time.strftime('/%Y-%m-%d_%H:%M/')  \
                    +str(iens)+assim_time.strftime('/wrfout_d01_%Y-%m-%d_%H:%M:%S')
        dart_ensdir = cluster.dartrundir + '/advance_temp' + str(iens)
        wrfout_dart = dart_ensdir + '/wrfout_d01'

        os.makedirs(dart_ensdir, exist_ok=True)
        print('linking', wrfout_run, 'to', wrfout_dart)
        symlink(wrfout_run, wrfout_dart)
        symlink(wrfout_dart, dart_ensdir + '/wrfinput_d01')

        # this seems to be necessary (else wrong level selection)
        wrfout_add_geo.run(cluster.dartrundir + '/geo_em.d01.nc', wrfout_dart)

    fpath = cluster.dartrundir + '/input_list.txt'
    print('writing', fpath)
    os.remove(fpath)
    with open(fpath, 'w') as f:
        for iens in range(1, exp.n_ens + 1):
            f.write('./advance_temp' + str(iens) + '/wrfout_d01')
            f.write('\n')

    fpath = cluster.dartrundir + '/output_list.txt'
    print('writing', fpath)
    os.remove(fpath)
    with open(fpath, 'w') as f:
        for iens in range(1, exp.n_ens + 1):
            f.write('./filter_restart_d01.' + str(iens).zfill(4))
            f.write('\n')

    print('removing preassim and filter_restart')
    os.system('rm -rf ' + cluster.dartrundir + '/preassim_*')
    os.system('rm -rf ' + cluster.dartrundir + '/filter_restart*')
    os.system('rm -rf ' + cluster.dartrundir + '/output_mean*')
    os.system('rm -rf ' + cluster.dartrundir + '/output_sd*')
    os.system('rm -rf ' + cluster.dartrundir + '/perfect_output_*')
    os.system('rm -rf ' + cluster.dartrundir + '/obs_seq.fina*')
Exemple #6
0
    def save_checkpoint(self,
                        out_dir,
                        filename_tmpl='epoch_{}.pth',
                        save_optimizer=True,
                        meta=None):
        if meta is None:
            meta = dict(epoch=self.epoch + 1, iter=self.iter)
        else:
            meta.update(epoch=self.epoch + 1, iter=self.iter)

        filename = osp.join(out_dir, filename_tmpl.format(self.epoch + 1))
        linkname = osp.join(out_dir, 'latest.pth')
        optimizer = self.optimizer if save_optimizer else None
        save_checkpoint(self.model, filename, optimizer=optimizer, meta=meta)
        symlink(filename, linkname)
Exemple #7
0
def note_symlink(name, source, alias_dir):
    alias_path = os.path.join(alias_dir, name)

    if os.path.exists(alias_path):
        if os.readlink(alias_path) == source:
            logger.debug('not creating symlink "{0}" because it exists'.format(alias_path))
        elif not os.path.islink(alias_path):
            logger.warning('{0} is not a symbolic link. doing nothing'.format(alias_path))
        else:
            os.remove(alias_path)
            logger.debug('removed stale symlink to {0}'.format(alias_path))
            symlink(alias_path, source)
            logger.debug('created link from {0} to {1}'.format(source, name))
    else:
        symlink(alias_path, source)
        logger.debug('created link from {0} to {1}'.format(source, name))
Exemple #8
0
def _create_link(input_fn, output_fn):
    out_dirname = os.path.dirname(output_fn)
    if out_dirname != '' and not os.path.exists(out_dirname):
        os.makedirs(out_dirname)

    if os.path.islink(output_fn):
        os.remove(output_fn)
    elif os.path.isdir(output_fn):
        abort('[{0}]: {1} exists and is a directory'.format('link', output_fn))
    elif os.path.exists(output_fn):
        abort('[{0}]: could not create a symlink at {1}.'.format('link', output_fn))

    out_base = os.path.basename(output_fn)
    if out_base == "":
       abort('[{0}]: could not create a symlink at {1}.'.format('link', output_fn))
    else:
        symlink(out_base, input_fn)
        os.rename(out_base, output_fn)
        puts('[{0}] created symbolic link pointing to "{1}" named "{2}"'.format('symlink', input_fn, out_base))
def init_fabric(buildsystem, conf_file):
    fab_dir = 'fabfile'

    if os.path.islink(fab_dir):
        os.remove(fab_dir)
    elif os.path.isdir(fab_dir):
        rmtree(fab_dir)

    symlink('fabfile', os.path.join(buildsystem, 'fabfile'))

    symlink('fabfile', os.path.join(buildsystem, 'fabfile'))

    symlink(name=os.path.join(buildsystem, 'bin', 'docs_meta.yaml'),
            target=os.path.join(os.getcwd(), conf_file))

    symlink(name=os.path.join(buildsystem, 'fabfile', 'utils.py'),
            target=os.path.join(os.path.abspath(buildsystem), 'bin', 'utils.py'))

    symlink(name=os.path.join(buildsystem, 'fabfile', 'docs_meta.py'),
            target=os.path.join(os.path.abspath(buildsystem), 'bin', 'docs_meta.py'))
Exemple #10
0
    def track(self, path, bundle=None):
        """Track a file or a directory."""
        # We don't use kwarg default, because None represents default to
        # callers
        bundle = bundle or 'Default'
        src_path = utils.truepath(path)
        is_directory = os.path.isdir(src_path)

        if self.root_path not in src_path:
            raise Exception('Cannot track files outside of root path')

        bundle_path = os.path.join(self.repo_path, bundle)
        dst_path = os.path.join(
            bundle_path, utils.relpath(self.root_path, src_path))

        undo_log = []

        dst_dir = os.path.dirname(dst_path)
        if not os.path.exists(dst_dir):
            utils.makedirs(dst_dir, dry_run=self.dry_run, undo_log=undo_log)

        try:
            utils.rename(src_path, dst_path, dry_run=self.dry_run,
                         undo_log=undo_log)
            try:
                utils.symlink(dst_path, src_path, dry_run=self.dry_run)
            except:
                utils.undo_operations(undo_log)
                raise
        except:
            utils.undo_operations(undo_log)
            raise

        self.git.add(dst_path)

        if is_directory:
            marker = self._track_directory(dst_path)
            self.git.add(marker)

        self.git.commit(message="Tracking '%s'" % path)
Exemple #11
0
print "loading transcripts..."
transcripts = Transcripts()
ts_all, ts_train, ts_test = transcripts.split(limit=DEBUG_LIMIT,
                                              add_all=add_all)
print "loading transcripts (%d train, %d test) ...done." % (len(ts_train),
                                                            len(ts_test))
#
# create work_dir structure
#

utils.mkdirs('%s/lexicon' % data_dir)
utils.mkdirs('%s/local/dict' % data_dir)
utils.mkdirs(wav16_dir)
utils.mkdirs(mfcc_dir)

utils.symlink('%s/egs/wsj/s5/steps' % kaldi_root, '%s/steps' % work_dir)
utils.symlink('%s/egs/wsj/s5/utils' % kaldi_root, '%s/utils' % work_dir)

#
# kaldi data part
#


def export_kaldi_data(destdirfn, tsdict):

    global wav16_dir

    print "Exporting to %s..." % destdirfn

    utils.mkdirs(destdirfn)
Exemple #12
0
def link_nature_to_dart_truth(time):
    # get wrfout_d01 from nature run
    shutil.copy(time.strftime(cluster.nature_wrfout),
                cluster.dartrundir+'/wrfout_d01')
    # DART may need a wrfinput file as well, which serves as a template for dimension sizes
    symlink(cluster.dartrundir+'/wrfout_d01', cluster.dartrundir+'/wrfinput_d01')
Exemple #13
0
import os, sys
from config.cfg import exp, cluster
from utils import symlink
joinp = os.path.join

dart_bin_dir = '/home/fs71386/lkugler/DART/DART_WRF_RTTOV_early_access/models/wrf/work/'
rttov_dir = '/home/fs71386/lkugler/RTTOV/'

# DART executables
bins = ['perfect_model_obs', 'filter', 'obs_diag']
for b in bins:
    symlink(joinp(dart_bin_dir, b), joinp(cluster.dartrundir, b))

# DART RTTOV capability
symlink(
    '/home/fs71386/lkugler/DART/DART_WRF_RTTOV_early_access/' +
    'observations/forward_operators/rttov_sensor_db.csv',
    joinp(cluster.dartrundir, 'rttov_sensor_db.csv'))

# Basic MSG4 Seviri
files = [
    'rtcoef_rttov12/rttov9pred54L/rtcoef_msg_4_seviri.dat',
    'rtcoef_rttov12/cldaer_visir/sccldcoef_msg_4_seviri.dat',
]
for f in files:
    symlink(joinp(rttov_dir, f), joinp(cluster.dartrundir,
                                       os.path.basename(f)))

# MFASIS
deff = True
if deff:
Exemple #14
0
    def install(self, **kwargs):

        utils.makedirs(self.install_directory)
        utils.makedirs(self.packages_directory)
        utils.makedirs(self.bin_directory)

        Sentinal = utils.SentinalFactory(
            os.path.join(self.install_directory, 'sentinal_'), kwargs)

        r_packages = [
            'foreach',
            'argparse',
            'yaml',
        ]

        for pkg in r_packages:
            with Sentinal('install_r_' + pkg) as sentinal:
                if sentinal.unfinished:
                    subprocess.check_call(
                        'R -q -e "install.packages(\'{0}\', repos=\'http://cran.us.r-project.org\')"'
                        .format(pkg),
                        shell=True)

        bioconductor_packages = [
            'HMMcopy',
            'IRanges',
            'Rsamtools',
            'GenomeInfoDb',
            'doMC',
            'TitanCNA',
        ]

        for pkg in bioconductor_packages:
            with Sentinal('install_r_' + pkg) as sentinal:
                if sentinal.unfinished:
                    subprocess.check_call(
                        'R -q -e "source(\'http://bioconductor.org/biocLite.R\'); biocLite(\'{0}\')"'
                        .format(pkg),
                        shell=True)

        with Sentinal('clone_titan') as sentinal:
            if sentinal.unfinished:
                with utils.CurrentDirectory(self.packages_directory):
                    subprocess.check_call(
                        'git clone https://github.com/gavinha/TitanCNA',
                        shell=True)
                    with utils.CurrentDirectory('TitanCNA'):
                        subprocess.check_call(
                            'git checkout 30fceb911b99a281ccbe3fac29d154f567127410',
                            shell=True)
                    subprocess.check_call('R CMD INSTALL TitanCNA', shell=True)

        with Sentinal('install_titan_tools') as sentinal:
            if sentinal.unfinished:
                with utils.CurrentDirectory(self.packages_directory):
                    subprocess.check_call(
                        'git clone https://github.com/gavinha/TitanCNA-utils',
                        shell=True)
                    with utils.CurrentDirectory('TitanCNA-utils'):
                        subprocess.check_call(
                            'git checkout 4cfd6155e620dade4090322d71f45f5a39cb688e',
                            shell=True)
                        utils.symlink(
                            'titan_scripts/createTITANsegmentfiles.pl',
                            link_directory=self.bin_directory)

        with Sentinal('download_chrom_info') as sentinal:
            if sentinal.unfinished:
                with utils.CurrentDirectory(self.data_directory):
                    subprocess.check_call(
                        'wget ftp://hgdownload.cse.ucsc.edu/goldenPath/hg19/database/chromInfo.txt.gz',
                        shell=True)
Exemple #15
0
import os, sys, shutil
import datetime as dt
from config.cfg import exp, cluster
from utils import symlink, copy, link_contents

import prepare_namelist

for iens in range(1, exp.n_ens+1):
    print('preparing ens', iens)
    input_prof = (cluster.input_profile).replace('<iens>', str(iens).zfill(3))

    rundir = cluster.wrf_rundir(iens)
    os.makedirs(rundir, exist_ok=True)
    link_contents(cluster.srcdir, rundir)
    print('linking ideal and wrf.exe:')
    symlink(cluster.ideal, rundir+'/ideal.exe')
    symlink(cluster.wrfexe, rundir+'/wrf.exe')

    # time not important, but general settings
    prepare_namelist.run(iens, begin=dt.datetime(2008, 7, 30, 6, 0),
                         end=dt.datetime(2008, 7, 30, 6, 30),
			 archive=False)

    symlink(input_prof, rundir+'/input_sounding')
print('finished.')
Exemple #16
0
            raise RuntimeError("Can't rename: %s -> %s" % (tmp_name, filepath))

    relpath = os.path.relpath(os.path.abspath(path),
                              os.path.abspath(lnks_path))
    new_path = utils.norm_path(relpath, audio.name)

    lnk_fmt = "%%0%dd=%%s" % len(str(total))
    lnk = lnk_fmt % (i + 1, audio.name)

    if sys.platform.startswith('win'):
        lnk = ''.join(lnk.rpartition('.')[:-1] + ('lnk',))

    lnk_path = utils.norm_path(lnks_path, lnk)

    try:
        utils.symlink(new_path, lnk_path)
    except Exception as e:
        raise RuntimeError("Can't create link: %s -> %s"
                           % (lnk_path, new_path))
    logging.debug("[#%d] %s -> %s", i + 1, lnk_path, new_path)


def audio_elem(audio, elem):
    t = audio.find(elem).text
    return utils.valid_filename(t) if t else u''


def get_id_name(audio):
    return utils.basename(audio.find("url").text)

Exemple #17
0
def checkout_and_build_libmecab(basedir):
    from os import environ
    from os.path import isdir, isfile, join as p_join, exists
    from sys import stderr
    from time import sleep
    from utils import get_parallel_jobs, run, chdir, mkdir_p, touch, symlink

    if basedir:
        chdir(basedir)

    if environ.get('LIBMECAB_DIR'):
        # This lets you point at an extracted tarball
        run("ls", "-l", environ.get("LIBMECAB_DIR"))
        assert exists(
            p_join(environ.get("LIBMECAB_DIR"), "mecab", "aclocal.m4"))
        assert not exists(LIBMECAB_DIR)
        run("rsync", "-avP", "--delete-during",
            environ.get('LIBMECAB_DIR') + "/", LIBMECAB_DIR)
        run("ls", "-l", LIBMECAB_DIR)
    else:
        mkdir_p(LIBMECAB_DIR)

    if exists(p_join(LIBMECAB_DIR, "mecab", "aclocal.m4")):
        # assume already checked out/extracted
        chdir(p_join(LIBMECAB_DIR, "mecab"))

    else:
        chdir(LIBMECAB_DIR)
        # You would think git clone would have an option to clone an
        # arbitrary <treeish>, but you would be wrong.
        run("git", "init")
        run("git", "remote", "add", "origin", LIBMECAB_REPO)
        run("git", "fetch", "origin", "--depth", "1", LIBMECAB_REV)
        run("git", "reset", "--hard", "FETCH_HEAD")
        chdir("mecab")

    CC, CXX, CFLAGS, CXXFLAGS, MDT = compile_flags_for_platform()

    stderr.write(
        "+ compile_flags_for_platform\n"
        "Building MeCab with:\n       {}\n      {}\n   {}\n {}\n".format(
            CC, CXX, CFLAGS, CXXFLAGS))

    if MDT is not None:
        stderr.write(" MACOSX_DEPLOYMENT_TARGET={}\n".format(MDT))
        environ["MACOSX_DEPLOYMENT_TARGET"] = MDT

    if not isfile("mecab-config"):
        # Not yet configured.
        # Adjust time stamps to make sure that Make doesn't think it
        # needs to re-run autoconf or automake.
        for f in [
                "aclocal.m4", "config.h.in", "configure", "Makefile.in",
                "src/Makefile.in"
        ]:
            touch(f)
            sleep(1)

        # We build with the default charset set to UTF-8, but we don't
        # disable support for EUC-JP or Shift-JIS.
        run("./configure", "--enable-static", "--disable-shared",
            "--with-charset=utf8", CC, CXX)

    # Only build the actual library, not the utilities.
    chdir("src")

    run("make", "-j{}".format(get_parallel_jobs()), "libmecab.la", CC, CXX,
        CFLAGS, CXXFLAGS)

    # Bypass libtool.
    if not isfile("libmecab.a"):
        symlink(".libs/libmecab.a", "libmecab.a")
Exemple #18
0
import os, sys, shutil
import datetime as dt
from config.cfg import exp, cluster
from utils import symlink, copy

time = dt.datetime.strptime(sys.argv[1], '%Y-%m-%d_%H:%M')

# get wrfout_d01 from nature run
shutil.copy(time.strftime(cluster.nature_wrfout),
            cluster.dartrundir+'/wrfout_d01')

import wrfout_add_geo
wrfout_add_geo.run(cluster.dartrundir+'/geo_em.d01.nc', cluster.dartrundir+'/wrfout_d01')

# DART may need a wrfinput file as well, which serves as a template for dimension sizes
symlink(cluster.dartrundir+'/wrfout_d01', cluster.dartrundir+'/wrfinput_d01')