class TestEg4(TestEg3):
__test__ = True
def _cmor1(self):
gen_drs.write_eg4_1(self.tmpdir)
self.dt = DRSTree(self.tmpdir)
self.dt.discover(self.incoming, activity='cmip5',
product='output1', institute='MOHC', model='HadCM3')
(self.pt, ) = self.dt.pub_trees.values()
def _cmor2(self):
gen_drs.write_eg4_2(self.tmpdir)
self.dt.discover_incoming(self.incoming, activity='cmip5',
product='output1')
def test_1(self):
self._cmor1()
self.pt.do_version(20100101)
self._cmor2()
self.pt.do_version(20100102)
assert self._exists('files')
assert self._exists('files/tas_20100102')
assert self._exists('v20100102/tas')
def test_2(self):
self._cmor1()
self.pt.do_version(20100101)
self._cmor2()
self.pt.do_version(20100102)
assert len(self._listdir('files/tas_20100101')) == 3
assert len(self._listdir('files/tas_20100102')) == 2
assert len(self._listdir('v20100101/tas')) == 3
assert len(self._listdir('v20100102/tas')) == 5
# Do test_3 from superclass
# Do test_4 from superclass
def test_6(self):
# Test differencing 2 versions
self._cmor1()
self.pt.do_version(20100101)
self._cmor2()
v1 = []
todo = []
for state, path1, path2 in self.pt.diff_version(20100101):
if state == self.pt.DIFF_V1_ONLY:
v1.append(path1)
elif state == self.pt.DIFF_V2_ONLY:
todo.append(path2)
assert len(v1) == 3
assert len(todo) == 2
class TestRepair2(TestRepair):
genfuncs = (gen_drs.write_eg3_1, gen_drs.write_eg3_2)
def setUp(self):
TestEg.setUp(self)
self._cmor1()
self.pt.do_version(20100101)
self._cmor2()
self.pt.do_version(20100102)
assert self.pt.state == self.pt.STATE_VERSIONED
self.breakme()
def _cmor1(self):
genfunc = self.genfuncs[0]
genfunc(self.tmpdir)
self.dt = DRSTree(self.tmpdir)
self.dt.discover(self.incoming, activity='cmip5',
product='output1', institute='MOHC', model='HadCM3')
(self.pt, ) = self.dt.pub_trees.values()
def _cmor2(self):
genfunc = self.genfuncs[1]
genfunc(self.tmpdir)
self.dt.discover_incoming(self.incoming, activity='cmip5',
product='output1')
class TestEg5(TestEg4):
__test__ = False
def _cmor1(self):
gen_drs.write_eg5_1(self.tmpdir)
self.dt = DRSTree(self.tmpdir)
self.dt.discover(self.incoming, activity='cmip5',
product='output1', institute='MOHC', model='HadCM3')
(self.pt, ) = self.dt.pub_trees.values()
def _cmor2(self):
gen_drs.write_eg5_2(self.tmpdir)
self.dt.discover_incoming(self.incoming, activity='cmip5',
product='output1')
# Do test1 from superclass
def test_2(self):
self._cmor1()
self.pt.do_version(20100101)
self._cmor2()
self.pt.do_version(20100102)
assert len(self._listdir('files/tas_20100101')) == 5
assert len(self._listdir('files/tas_20100102')) == 2
assert len(self._listdir('v20100101/tas')) == 5
assert len(self._listdir('v20100102/tas')) == 5
# Do test_3 from superclass
# Do test_4 from superclass
def test_6(self):
# Test differencing 2 versions
self._cmor1()
self.pt.do_version(20100101)
self._cmor2()
v1 = []
todo = []
diff = []
same = []
for state, path1, path2 in self.pt.diff_version(20100101):
if state == self.pt.DIFF_V1_ONLY:
v1.append(path1)
elif state == self.pt.DIFF_V2_ONLY:
todo.append(path2)
elif state == self.pt.DIFF_SIZE:
diff.append(path1)
elif state == self.pt.DIFF_NONE:
same.append(path1)
#!TODO: not same? This test needs reviewing.
assert len(v1) == 3
assert len(same) == 2
class TestEg5_1(TestEg5):
__test__ = False
def _cmor2(self):
gen_drs.write_eg5_2(self.tmpdir)
self.dt2 = DRSTree(self.tmpdir)
self.dt2.discover_incoming(self.incoming, activity='cmip5',
product='output1')
(self.pt, ) = self.dt2.pub_trees.values()
class TestThreeway(TestEg):
__test__ = True
listing_files = ['threeway_1.ls', 'threeway_2.ls', 'threeway_3.ls']
def setUp(self):
super(TestThreeway, self).setUp()
self.drs_fs = CMIP5FileSystem(self.tmpdir)
self.dt = DRSTree(self.drs_fs)
self.listing_iter = self._iterSetUpListings()
def _iterSetUpListings(self):
for listing_file in self.listing_files:
listing_path = os.path.join(test_dir, listing_file)
gen_drs.write_listing(self.tmpdir, listing_path)
yield listing_path
def _discover(self):
self.dt.discover_incoming(self.incoming, activity='cmip5',
product='output1',
institute='MOHC',
model='HadGEM2-ES')
def _do_version(self, pt, next_version):
assert next_version not in pt.versions.keys()
pt.do_version(next_version)
assert next_version in pt.versions.keys()
def _check_version(self, pt, version):
for path, drs in pt.versions[version]:
assert os.path.islink(path)
# link is relative
real_path = os.path.realpath(os.path.join(os.path.dirname(path),
os.readlink(path)))
assert os.path.isfile(real_path)
# Check variables match
mo = re.search(r'/files/(.*?)_\d+/(.*?)_', real_path)
assert mo.group(1) == mo.group(2)
def test1(self):
v = 1
for listing_path in self.listing_iter:
print 'Doing version %d' % v
self._discover()
assert len(self.dt.pub_trees) == 1
pt = self.dt.pub_trees.values()[0]
self._do_version(pt, v)
self._check_version(pt, v)
v += 1
class TestEg3_1(TestEg3):
"""Use a separate DRSTree instance for the upgrade to test
TestEg3 still works in this scenario.
"""
__test__ = True
def _cmor2(self):
gen_drs.write_eg3_2(self.tmpdir)
self.dt2 = DRSTree(self.tmpdir)
self.dt2.discover_incoming(self.incoming, activity='cmip5',
product='output1')
(self.pt, ) = self.dt2.pub_trees.values()
def test_2(self):
"""Test incremental discovery without calling discover() first."""
dt = DRSTree(self.tmpdir)
components = dict(activity='cmip5',
product='output1', institute='MOHC', model='HadCM3')
assert len(dt.pub_trees) == 0
# Discover ocean realm
dt.discover_incoming(self.tmpdir, realm='ocean', **components)
assert len(dt.pub_trees) == 1
# Discover atmos realm
dt.discover_incoming(self.tmpdir, realm='atmos', **components)
assert len(dt.pub_trees) == 2
assert set([x.drs.realm for x in dt.pub_trees.values()]) == set(['atmos', 'ocean'])
class TestEg6(TestEg):
__test__ = True
deliveries = [
['clt_day_HadGEM2-ES_rcp26_r1i1p1_20051201-20151130.nc',
'clt_day_HadGEM2-ES_rcp26_r1i1p1_20151201-20251130.nc'],
['huss_day_HadGEM2-ES_rcp26_r1i1p1_20991201-21091130.nc'],
['hur_day_HadGEM2-ES_rcp26_r1i1p1_20991201-20991230.nc',
'hus_day_HadGEM2-ES_rcp26_r1i1p1_20991201-20991230.nc'],
]
def setUp(self):
super(TestEg6, self).setUp()
self.setupIncoming()
self.drs_fs = CMIP5FileSystem(self.tmpdir)
self.dt = DRSTree(self.drs_fs)
for i, delivery in enumerate(self.deliveries):
self.dt.discover_incoming(op.join(self.incoming, str(i)),
activity='cmip5', product='output1', institute='MOHC')
for drs_id, pt in self.dt.pub_trees.items():
pt.do_version(i)
def setupIncoming(self):
# Create incoming files
self.incoming = op.join(self.tmpdir, 'incoming')
os.mkdir(self.incoming)
for i, delivery in enumerate(self.deliveries):
os.mkdir(op.join(self.incoming, str(i)))
for filename in delivery:
gen_drs.write_eg_file(op.join(self.incoming, str(i), filename))
def test_1(self):
assert len(self.dt.pub_trees) == 1
pt = self.dt.pub_trees.values()[0]
file_counts = set((k, len(v)) for (k, v) in pt.versions.items())
print file_counts
assert file_counts == set([(0, 2), (1, 3), (2, 5)])
class TestEg3(TestEg):
__test__ = True
def _cmor1(self):
gen_drs.write_eg3_1(self.tmpdir)
self.dt = DRSTree(self.tmpdir)
self.dt.discover(self.incoming, activity='cmip5',
product='output1', institute='MOHC', model='HadCM3')
(self.pt, ) = self.dt.pub_trees.values()
def _cmor2(self):
gen_drs.write_eg3_2(self.tmpdir)
self.dt.discover_incoming(self.incoming, activity='cmip5',
product='output1')
def _exists(self, x):
return os.path.exists(os.path.join(self.pt.pub_dir, x))
def _listdir(self, x):
return os.listdir(os.path.join(self.pt.pub_dir, x))
def _listlinks(self, x):
links = glob('%s/*' % os.path.join(self.pt.pub_dir, x))
return [os.readlink(lnk) for lnk in links if os.path.islink(lnk)]
def test_01(self):
self._cmor1()
assert len(self.pt.drs_tree.incoming) > 0
self.pt.do_version()
assert len(self.pt.drs_tree.incoming) == 0
assert self.pt.count_todo() == 0
assert len(list(self.pt.list_todo())) == 0
def test_1(self):
self._cmor1()
self.pt.do_version(20100101)
self._cmor2()
self.pt.do_version(20100102)
assert len(self.pt.drs_tree.incoming) == 0
assert self._exists('files')
assert self._exists('files/rsus_20100102')
assert not self._exists('files/rsus_20100101')
assert self._exists('v20100101/tas')
assert self._exists('v20100101/pr')
assert not self._exists('v20100101/rsus')
assert self._exists('v20100102/rsus')
def test_2(self):
self._cmor1()
self.pt.do_version(20100101)
self._cmor2()
self.pt.do_version(20100102)
assert self._exists('v20100102/pr/pr_day_HadCM3_1pctto4x_r1i1p1_2000010100-2001123114.nc')
def test_3(self):
self._cmor1()
assert self.pt.state == self.pt.STATE_INITIAL
self.pt.do_version()
assert self.pt.state == self.pt.STATE_VERSIONED
self._cmor2()
assert self.pt.state == self.pt.STATE_VERSIONED_TRANS
self.pt.do_version()
assert self.pt.state == self.pt.STATE_VERSIONED
def test_4(self):
# Check all links are to the "files" branch
self._cmor1()
self.pt.do_version()
self._cmor2()
self.pt.do_version()
links = self._listlinks('v2/tas/r1i1p1')
for link in links:
assert '/files/' in link
def test_5(self):
self._cmor1()
self.pt.do_version(20100101)
latest = os.readlink(os.path.join(self.pt.pub_dir, 'latest'))
assert latest == 'v20100101'
self._cmor2()
self.pt.do_version(20100102)
latest = os.readlink(os.path.join(self.pt.pub_dir, 'latest'))
assert latest == 'v20100102'
def test_6(self):
# Test differencing 2 versions
self._cmor1()
self.pt.do_version(20100101)
self._cmor2()
v1 = []
todo = []
for state, path1, path2 in self.pt.diff_version(20100101):
if state == self.pt.DIFF_V1_ONLY:
assert not 'rsus' in path1
v1.append(path1)
elif state == self.pt.DIFF_V2_ONLY:
assert 'rsus' in path2
todo.append(path2)
assert len(v1) == 10
assert len(todo) == 5