def __init__(self, drs, drs_tree): self.drs_tree = drs_tree self.drs = drs self.state = None self._todo = [] self.versions = {} self.latest = 0 self._vtrans = make_translator(drs_tree.drs_root) self._cmortrans = make_translator(drs_tree.drs_root, with_version=False) from drslib.drs_tree_check import default_checkers self._checkers = default_checkers[:] self._checker_failures = [] #!TODO: calling internal method. Make this method public. ensemble = self.drs._encode_ensemble() self.pub_dir = os.path.join(self.drs_tree.drs_root, self.drs.product, self.drs.institute, self.drs.model, self.drs.experiment, self.drs.frequency, self.drs.realm, self.drs.table, ensemble) self.deduce_state()
def check_listing(listing_file, output='output1'): prefix = os.path.join(tmpdir, 'reg_ncc') filenames = open(os.path.join(test_dir, listing_file)).readlines() write_listing_seq(prefix, filenames) trans = make_translator(prefix) for filename in filenames: drs = trans.filename_to_drs(filenames[0]) status = pc1.find_product(drs.variable, drs.table, drs.experiment, drs.model, prefix) assert status if output: assert pc1.product == output
def test_regression_4(): listing_file = 'esmControl.ls' prefix = os.path.join(tmpdir, 'reg_4') write_listing(prefix, os.path.join(os.path.dirname(__file__), listing_file)) trans = make_translator(prefix) for filename in os.listdir(prefix): drs = trans.filename_to_drs(filename) status = pc3.find_product(drs.variable, drs.table, drs.experiment, drs.model, prefix, startyear=drs.subset[0][0]) assert status
def test_regression_2(): prefix = os.path.join(tmpdir, 'reg_2') filenames = [ 'areacella_fx_HadGEM2-ES_rcp85_r1i1p1.nc', 'orog_fx_HadGEM2-ES_rcp85_r1i1p1.nc', 'sftlf_fx_HadGEM2-ES_rcp85_r1i1p1.nc', ] write_listing_seq(prefix, filenames) trans = make_translator(prefix) for filename in filenames: drs = trans.filename_to_drs(filename) status = pc1.find_product(drs.variable, drs.table, drs.experiment, drs.model, prefix) assert status assert pc1.product=='output1'
def trans_files(cmip3_path, cmip5_path): cmip3_t = cmip3.make_translator(cmip3_path) cmip5_t = cmip5.make_translator(cmip5_path) log.info('Dry run is %s' % dry_run) log.info('Copying is %s' % copy_trans) for dirpath, filenames in walk_cmip3(cmip3_path): log.info('Processing directory %s' % dirpath) try: drs = cmip3_t.path_to_drs(dirpath) path = cmip5_t.drs_to_path(drs) except TranslationError, e: log.error('Failed to translate path %s: %s' % (dirpath, e)) continue except:
def write_eg(prefix, seq): """ Create a test directory tree under prefix. Files are created in <prefix>/output in CMOR's output directory structure """ trans = cmip5.make_translator(prefix, with_version=False) for drs in seq: path = trans.drs_to_filepath(drs) if not op.exists(op.dirname(path)): os.makedirs(op.dirname(path)) else: if op.exists(path): raise RuntimeError("%s exists" % path) write_eg_file(path)
def test_regression_3(): prefix = os.path.join(tmpdir, 'reg_3') filenames = [ 'ps_6hrLev_NorESM1-M_rcp45_r1i1p1_2181010100-2190123118.nc', 'ps_6hrLev_NorESM1-M_rcp45_r1i1p1_2191010100-2200123118.nc' ] write_listing_seq(prefix, filenames) trans = make_translator(prefix) for filename in filenames: drs = trans.filename_to_drs(filenames[0]) status = pc1.find_product(drs.variable, drs.table, drs.experiment, drs.model, prefix, startyear=drs.subset[0][0]) assert status print '%s startyear=%d product=%s' % (filename, drs.subset[0][0], pc1.product) assert pc1.product=='output2'
def __init__(self, drs_root, table_store=None): """ :param drs_root: The path to the DRS *activity* directory. :param table_store: Override the default table store. This can be used to select the TAMIP tables. """ self.drs_root = os.path.normpath(os.path.abspath(drs_root)) self.pub_trees = {} self._vtrans = make_translator(self.drs_root, table_store=table_store) self.incoming = DRSList() self.incomplete = DRSList() self._p_cmip5 = None self._move_cmd = config.move_cmd if not os.path.isdir(self.drs_root): raise Exception('DRS root "%s" is not a directory' % self.drs_root)
def test_regression_1(): """ Bug identified during ingest. """ prefix = os.path.join(tmpdir, 'reg_1') filenames = [ 'clt_day_HadGEM2-ES_piControl_r1i1p1_19791201-19891130.nc', 'clt_day_HadGEM2-ES_piControl_r1i1p1_19891201-19991130.nc', 'clt_day_HadGEM2-ES_piControl_r1i1p1_19991201-20091130.nc', ] write_listing_seq(prefix, filenames) trans = make_translator(prefix) for filename in filenames: drs = trans.filename_to_drs(filename) status = pc1.find_product(drs.variable, drs.table, drs.experiment, drs.model, prefix, startyear=1979) assert status assert pc1.product=='output1'
from drslib.cmip5 import make_translator from drslib import p_cmip5 from drslib import drs_tree import gen_drs import os, shutil import metaconfig import tempfile LISTING = 'multi_product.ls' config = metaconfig.get_config('drslib') shelve_dir = config.get('p_cmip5', 'shelve-dir') drs_root = '/cmip5' vtrans = make_translator(drs_root) def setup_module(): global p_cmip5, listing, tmpdir, dt tmpdir = tempfile.mkdtemp(prefix='drs_tree-product-') print 'TMPDIR ',tmpdir shelves = p_cmip5.init._find_shelves(shelve_dir) config_file = os.path.join(os.path.dirname(__file__), 'ukmo_sample.ini') listing = os.path.join(os.path.dirname(__file__), LISTING) gen_drs.write_listing(tmpdir, listing) p_cmip5 = p_cmip5.product.cmip5_product(mip_table_shelve=shelves['stdo_mip'], template=shelves['template'],
# All rights reserved. # # See the LICENSE file in the source distribution of this software for # the full license text. """ Tests compatible with nosetests """ import os from drslib import cmip5 from drslib.drs import CmipDRS translator = cmip5.make_translator('cmip5') translator_noversion = cmip5.make_translator('cmip5', with_version=False) def get_drs1(): return translator.filename_to_drs('tas_Amon_HadCM3_historicalNat_r1i1p1_185001-200512.nc') def test_1(): """ Demonstrate succesfully parsing a filename """ drs = get_drs1() print drs.__dict__ def test_2():
def __init__(self): from drslib import cmip5 self._trans = cmip5.make_translator('cmip5')
import logging log = logging.getLogger(__name__) THREDDS_NS = 'http://www.unidata.ucar.edu/namespaces/thredds/InvCatalog/v1.0' XLINK_NS = 'http://www.w3.org/1999/xlink' usage = """%prog [options] thredds ... thredds: A thredds url or file path of a dataset published by esgpublish.\ """ trans = make_translator('') drs_prop_map = {'dataset_version': 'version', 'project': 'activity', 'experiment': 'experiment', 'product': 'product', 'model': 'model', 'time_frequency': 'frequency', 'realm': 'realm', 'cmor_table': 'table', 'ensemble': 'ensemble', 'institute': 'institute' } class InvalidThreddsException(Exception): """
# # See the LICENSE file in the source distribution of this software for # the full license text. """ Tests compatible with nosetests """ import os from drslib import cmip3, cmip5 translator = cmip3.make_translator('/') cmip5_translator = cmip5.make_translator('cmip5') def test_1(): p = '/20c3m/atm/da/rsus/gfdl_cm2_0/run1/rsus_A2.19610101-19651231.nc' p2 = convert(p) assert p2 == 'cmip5/output/GFDL/CM2/20c3m/day/atmos/A2/r1/v1/rsus/rsus_A2_CM2_20c3m_r1_19610101-19651231.nc' def test_listing(): """ Test successful conversion of part of the CMIP3 listing """ fh = open(os.path.join(os.path.dirname(__file__), 'cmip3_test_ls')) for line in fh: yield convert, line.strip()
def setup(): global cmip5_trans cmip5_trans = cmip5.make_translator('/cmip5', with_version=False)
# BSD Licence # Copyright (c) 2010, Science & Technology Facilities Council (STFC) # All rights reserved. # # See the LICENSE file in the source distribution of this software for # the full license text. import os from drslib import cmip5, mip_table TAMIP_TEST_PATH = os.path.join(os.path.dirname(__file__), 'TAMIP') table_store = mip_table.MIPTableStore('%s/TAMIP_*' % TAMIP_TEST_PATH) translator = cmip5.make_translator('tamip', table_store=table_store) def test_gen_1(): fh = open(os.path.join(TAMIP_TEST_PATH, 'tamip.ls')) for filename in fh: yield do, filename def do(filename): drs = translator.filename_to_drs(filename) drs.version = 1 drs.product = 'output' drs.activity == 'tamip' print drs assert drs.is_complete()