def __init__(self, drs, drs_tree):
self.drs_tree = drs_tree
self.drs = drs
self.state = None
self._todo = []
self.versions = {}
self.latest = 0
self._vtrans = make_translator(drs_tree.drs_root)
self._cmortrans = make_translator(drs_tree.drs_root, with_version=False)
from drslib.drs_tree_check import default_checkers
self._checkers = default_checkers[:]
self._checker_failures = []
#!TODO: calling internal method. Make this method public.
ensemble = self.drs._encode_ensemble()
self.pub_dir = os.path.join(self.drs_tree.drs_root,
self.drs.product,
self.drs.institute,
self.drs.model,
self.drs.experiment,
self.drs.frequency,
self.drs.realm,
self.drs.table,
ensemble)
self.deduce_state()
def check_listing(listing_file, output='output1'):
prefix = os.path.join(tmpdir, 'reg_ncc')
filenames = open(os.path.join(test_dir, listing_file)).readlines()
write_listing_seq(prefix, filenames)
trans = make_translator(prefix)
for filename in filenames:
drs = trans.filename_to_drs(filenames[0])
status = pc1.find_product(drs.variable, drs.table, drs.experiment,
drs.model, prefix)
assert status
if output:
assert pc1.product == output
def test_regression_4():
listing_file = 'esmControl.ls'
prefix = os.path.join(tmpdir, 'reg_4')
write_listing(prefix, os.path.join(os.path.dirname(__file__),
listing_file))
trans = make_translator(prefix)
for filename in os.listdir(prefix):
drs = trans.filename_to_drs(filename)
status = pc3.find_product(drs.variable, drs.table, drs.experiment,
drs.model, prefix,
startyear=drs.subset[0][0])
assert status
def test_regression_2():
prefix = os.path.join(tmpdir, 'reg_2')
filenames = [
'areacella_fx_HadGEM2-ES_rcp85_r1i1p1.nc',
'orog_fx_HadGEM2-ES_rcp85_r1i1p1.nc',
'sftlf_fx_HadGEM2-ES_rcp85_r1i1p1.nc',
]
write_listing_seq(prefix, filenames)
trans = make_translator(prefix)
for filename in filenames:
drs = trans.filename_to_drs(filename)
status = pc1.find_product(drs.variable, drs.table, drs.experiment,
drs.model, prefix)
assert status
assert pc1.product=='output1'
def trans_files(cmip3_path, cmip5_path):
cmip3_t = cmip3.make_translator(cmip3_path)
cmip5_t = cmip5.make_translator(cmip5_path)
log.info('Dry run is %s' % dry_run)
log.info('Copying is %s' % copy_trans)
for dirpath, filenames in walk_cmip3(cmip3_path):
log.info('Processing directory %s' % dirpath)
try:
drs = cmip3_t.path_to_drs(dirpath)
path = cmip5_t.drs_to_path(drs)
except TranslationError, e:
log.error('Failed to translate path %s: %s' % (dirpath, e))
continue
except:
def write_eg(prefix, seq):
"""
Create a test directory tree under prefix.
Files are created in <prefix>/output in CMOR's
output directory structure
"""
trans = cmip5.make_translator(prefix, with_version=False)
for drs in seq:
path = trans.drs_to_filepath(drs)
if not op.exists(op.dirname(path)):
os.makedirs(op.dirname(path))
else:
if op.exists(path):
raise RuntimeError("%s exists" % path)
write_eg_file(path)
def test_regression_3():
prefix = os.path.join(tmpdir, 'reg_3')
filenames = [
'ps_6hrLev_NorESM1-M_rcp45_r1i1p1_2181010100-2190123118.nc',
'ps_6hrLev_NorESM1-M_rcp45_r1i1p1_2191010100-2200123118.nc'
]
write_listing_seq(prefix, filenames)
trans = make_translator(prefix)
for filename in filenames:
drs = trans.filename_to_drs(filenames[0])
status = pc1.find_product(drs.variable, drs.table, drs.experiment,
drs.model, prefix,
startyear=drs.subset[0][0])
assert status
print '%s startyear=%d product=%s' % (filename, drs.subset[0][0], pc1.product)
assert pc1.product=='output2'
def __init__(self, drs_root, table_store=None):
"""
:param drs_root: The path to the DRS *activity* directory.
:param table_store: Override the default table store. This can be used
to select the TAMIP tables.
"""
self.drs_root = os.path.normpath(os.path.abspath(drs_root))
self.pub_trees = {}
self._vtrans = make_translator(self.drs_root, table_store=table_store)
self.incoming = DRSList()
self.incomplete = DRSList()
self._p_cmip5 = None
self._move_cmd = config.move_cmd
if not os.path.isdir(self.drs_root):
raise Exception('DRS root "%s" is not a directory' % self.drs_root)
def test_regression_1():
"""
Bug identified during ingest.
"""
prefix = os.path.join(tmpdir, 'reg_1')
filenames = [
'clt_day_HadGEM2-ES_piControl_r1i1p1_19791201-19891130.nc',
'clt_day_HadGEM2-ES_piControl_r1i1p1_19891201-19991130.nc',
'clt_day_HadGEM2-ES_piControl_r1i1p1_19991201-20091130.nc',
]
write_listing_seq(prefix, filenames)
trans = make_translator(prefix)
for filename in filenames:
drs = trans.filename_to_drs(filename)
status = pc1.find_product(drs.variable, drs.table, drs.experiment,
drs.model, prefix,
startyear=1979)
assert status
assert pc1.product=='output1'
from drslib.cmip5 import make_translator
from drslib import p_cmip5
from drslib import drs_tree
import gen_drs
import os, shutil
import metaconfig
import tempfile
LISTING = 'multi_product.ls'
config = metaconfig.get_config('drslib')
shelve_dir = config.get('p_cmip5', 'shelve-dir')
drs_root = '/cmip5'
vtrans = make_translator(drs_root)
def setup_module():
global p_cmip5, listing, tmpdir, dt
tmpdir = tempfile.mkdtemp(prefix='drs_tree-product-')
print 'TMPDIR ',tmpdir
shelves = p_cmip5.init._find_shelves(shelve_dir)
config_file = os.path.join(os.path.dirname(__file__), 'ukmo_sample.ini')
listing = os.path.join(os.path.dirname(__file__), LISTING)
gen_drs.write_listing(tmpdir, listing)
p_cmip5 = p_cmip5.product.cmip5_product(mip_table_shelve=shelves['stdo_mip'],
template=shelves['template'],
# All rights reserved.
#
# See the LICENSE file in the source distribution of this software for
# the full license text.
"""
Tests compatible with nosetests
"""
import os
from drslib import cmip5
from drslib.drs import CmipDRS
translator = cmip5.make_translator('cmip5')
translator_noversion = cmip5.make_translator('cmip5', with_version=False)
def get_drs1():
return translator.filename_to_drs('tas_Amon_HadCM3_historicalNat_r1i1p1_185001-200512.nc')
def test_1():
"""
Demonstrate succesfully parsing a filename
"""
drs = get_drs1()
print drs.__dict__
def test_2():
def __init__(self):
from drslib import cmip5
self._trans = cmip5.make_translator('cmip5')
import logging
log = logging.getLogger(__name__)
THREDDS_NS = 'http://www.unidata.ucar.edu/namespaces/thredds/InvCatalog/v1.0'
XLINK_NS = 'http://www.w3.org/1999/xlink'
usage = """%prog [options] thredds ...
thredds:
A thredds url or file path of a dataset published by esgpublish.\
"""
trans = make_translator('')
drs_prop_map = {'dataset_version': 'version',
'project': 'activity',
'experiment': 'experiment',
'product': 'product',
'model': 'model',
'time_frequency': 'frequency',
'realm': 'realm',
'cmor_table': 'table',
'ensemble': 'ensemble',
'institute': 'institute'
}
class InvalidThreddsException(Exception):
"""
#
# See the LICENSE file in the source distribution of this software for
# the full license text.
"""
Tests compatible with nosetests
"""
import os
from drslib import cmip3, cmip5
translator = cmip3.make_translator('/')
cmip5_translator = cmip5.make_translator('cmip5')
def test_1():
p = '/20c3m/atm/da/rsus/gfdl_cm2_0/run1/rsus_A2.19610101-19651231.nc'
p2 = convert(p)
assert p2 == 'cmip5/output/GFDL/CM2/20c3m/day/atmos/A2/r1/v1/rsus/rsus_A2_CM2_20c3m_r1_19610101-19651231.nc'
def test_listing():
"""
Test successful conversion of part of the CMIP3 listing
"""
fh = open(os.path.join(os.path.dirname(__file__), 'cmip3_test_ls'))
for line in fh:
yield convert, line.strip()
def setup():
global cmip5_trans
cmip5_trans = cmip5.make_translator('/cmip5', with_version=False)
# BSD Licence
# Copyright (c) 2010, Science & Technology Facilities Council (STFC)
# All rights reserved.
#
# See the LICENSE file in the source distribution of this software for
# the full license text.
import os
from drslib import cmip5, mip_table
TAMIP_TEST_PATH = os.path.join(os.path.dirname(__file__), 'TAMIP')
table_store = mip_table.MIPTableStore('%s/TAMIP_*' % TAMIP_TEST_PATH)
translator = cmip5.make_translator('tamip', table_store=table_store)
def test_gen_1():
fh = open(os.path.join(TAMIP_TEST_PATH, 'tamip.ls'))
for filename in fh:
yield do, filename
def do(filename):
drs = translator.filename_to_drs(filename)
drs.version = 1
drs.product = 'output'
drs.activity == 'tamip'
print drs
assert drs.is_complete()
