def setUp(self):
self.test_write_root=TEST_INVARIANTS['test_write_root']
self.sample_data_root=self.test_write_root
self.sm0 = StorageManager(root_path=self.sample_data_root)
self.locroot0='rootloc0'
self.locroot0_path=os.path.abspath(os.path.join(self.sample_data_root, self.locroot0))
self.sep0=self.sm0.sublocation_separator
self.loc01='rootloc0%s1'%self.sep0
self.loc01_path=os.path.abspath(os.path.join(self.sample_data_root, self.loc01))
self.loc02='rootloc0%s2%sa%sbb'%(self.sep0, self.sep0, self.sep0)
self.loc02_path=os.path.abspath(os.path.join(self.sample_data_root, self.loc02))
self.loc02_sublocs=('rootloc0%s2'%self.sep0, 'rootloc0%s2%sa'%(self.sep0, self.sep0))
def test_openReport1(self):
sm = StorageManager(root_path=self.test_write_root)
rep1 = Reporter(self.refparams1, **self.reportparams1)
rep1.initialize(sm, self.resloc1, self.addata1)
rep1.openReport(self.reploc1, self.reportcontent1)
rep1.finalize()
self.assertTrue(os.path.exists(self.reppath1))
self.assertTrue(1, len(os.listdir(self.repdir1)))
def test_finalize1(self):
sm = StorageManager(root_path=self.test_write_root)
rep1 = Reporter(self.refparams1, **self.reportparams1)
rep1.initialize(sm, self.resloc1, self.addata1)
self.assertEqual(0, len(rep1.getReports()))
rep1.finalize()
with self.assertRaises(OSError):
os.listdir(self.repdir1)
def test_init2(self):
sm = StorageManager(root_path=self.sample_data_root)
self.assertEqual(1, len(sm.locations))
self.assertRegexpMatches(sm.name, "[0-9a-f]{16}")
self.assertRegexpMatches(sm.locations.keys()[0], 'ROOT_[0-9a-f]{16}')
# self.assertRegexpMatches(sm.locations.values()[0], os.path.abspath(self.sample_data_root))
self.assertEqual(sm.locations.values()[0], os.path.abspath(self.sample_data_root))
self.assertFalse(os.path.exists(os.path.join(self.sample_data_root, '%s.root.db'%SYSTEM_NAME_LC)))
def setUp(self):
self.test_data_root = TEST_INVARIANTS['test_data_root']
self.test_write_root = TEST_INVARIANTS['test_write_root']
self.data_range = range(1, 6)
self.skiprows = 1
self.data_usecols = range(1, 11)
self.labels_usecols = (1, )
self.data = list()
for d in self.data_range:
data_path = os.path.abspath(
os.path.join(self.test_data_root, 'l1l2_t2_data%d.txt' % d))
self.data.append(
numpy.loadtxt(data_path,
skiprows=self.skiprows,
usecols=self.data_usecols,
delimiter='\t'))
labels_path = os.path.join(self.test_data_root, 'l1l2_t2_labels.txt')
self.labels = numpy.loadtxt(labels_path,
skiprows=self.skiprows,
usecols=self.labels_usecols)
with open(labels_path, 'rb') as f:
self.samples = [
r['Samples'] for r in csv.DictReader(f, dialect='excel-tab')
]
self.ref_data_shape = (30, 10)
self.ref_labels_shape = (10, )
self.dof = 'DOF'
self.l1l2_ols_cfg1 = {
'error_func': l1l2py.tools.balanced_classification_error,
'return_predictions': True,
'global_degrees_of_freedom': [self.dof],
'job_importable': False,
}
self.ssname1 = ['SS%d' % i for i in self.data_range]
self.pkcid1 = ['SS%d' % i for i in self.data_range]
self.vars1 = ['M%d' % i for i in range(1, 31)]
self.initial_additionalJobData1 = {
'samples': self.samples,
}
self.additionalJobData1 = {
'depfuncs': (),
'modules': ['os', 'l1l2py', 'numpy'],
}
self.runtime1 = {'techID': 'TECH_ID_OLS'}
self.additionalReportData1 = {'em2annotation': Mock_em2annotation()}
self.storageManager1 = StorageManager(root_path=self.test_write_root)
self.ssloc1 = 'ss'
self.reporter_name_part = '_vars_freqs.txt'
self.reports_loc1 = os.path.join(self.test_write_root, self.ssloc1)
self.reports_names1 = [
'%s_%s_%s' % (ss, self.dof, self.reporter_name_part)
for ss in self.ssname1
]
self.reports_paths1 = [
os.path.join(self.reports_loc1, ss, rname)
for ss, rname in zip(self.ssname1, self.reports_names1)
]
def test_init1(self):
userdir=os.path.expanduser('~/.%s/'%(SYSTEM_NAME_LC))
if os.path.exists(userdir):
sm = StorageManager()
self.assertEqual(1, len(sm.locations))
self.assertRegexpMatches(sm.name, "[0-9a-f]{16}")
self.assertRegexpMatches(sm.locations.keys()[0], 'ROOT_[0-9a-f]{16}')
self.assertRegexpMatches(sm.locations.values()[0], os.path.abspath(userdir))
else:
self.skipTest('%s not present'%userdir)
def test_init4(self):
sm = StorageManager(root_path=self.sample_data_root, create_dbm=True)
self.assertEqual(1, len(sm.locations))
self.assertRegexpMatches(sm.name, "[0-9a-f]{16}")
self.assertRegexpMatches(sm.locations.keys()[0], 'ROOT_[0-9a-f]{16}')
# self.assertRegexpMatches(sm.locations.values()[0], os.path.abspath(self.sample_data_root))
self.assertEqual(sm.locations.values()[0], os.path.abspath(self.sample_data_root))
self.assertTrue(os.path.exists(os.path.join(self.sample_data_root, '%s.root.db'%SYSTEM_NAME_LC)))
self.assertEqual(os.path.abspath(self.sample_data_root), sm.dbm.abs_data_root)
ref_keys=set(['memdb', '%s.root.db'%SYSTEM_NAME_LC])
self.assertEqual(ref_keys, set(sm.dbm.db.keys()))
sm.dbm.close()
def test_initialize1(self):
sm = StorageManager(root_path=self.test_write_root)
rep1 = Reporter(self.refparams1, **self.reportparams1)
rep1.initialize(sm, self.resloc1, self.addata1)
self.assertEqual(self.addata1, rep1.getAdditionalData())
class TestFSStorageManager2(unittest.TestCase):
def setUp(self):
self.test_write_root=TEST_INVARIANTS['test_write_root']
self.sample_data_root=self.test_write_root
self.sm0 = StorageManager(root_path=self.sample_data_root)
self.locroot0='rootloc0'
self.locroot0_path=os.path.abspath(os.path.join(self.sample_data_root, self.locroot0))
self.sep0=self.sm0.sublocation_separator
self.loc01='rootloc0%s1'%self.sep0
self.loc01_path=os.path.abspath(os.path.join(self.sample_data_root, self.loc01))
self.loc02='rootloc0%s2%sa%sbb'%(self.sep0, self.sep0, self.sep0)
self.loc02_path=os.path.abspath(os.path.join(self.sample_data_root, self.loc02))
self.loc02_sublocs=('rootloc0%s2'%self.sep0, 'rootloc0%s2%sa'%(self.sep0, self.sep0))
def tearDown(self):
if os.path.exists(self.locroot0_path):
shutil.rmtree(self.locroot0_path)
def test_createLocation1(self):
self.assertIsNone(self.sm0.getLocation(self.locroot0))
self.sm0.createLocation(self.locroot0)
self.assertEqual(self.locroot0_path, self.sm0.getLocation(self.locroot0))
self.assertTrue(os.path.exists(self.locroot0_path))
def test_createLocation2(self):
self.assertIsNone(self.sm0.getLocation(self.locroot0))
self.sm0.createLocation(self.locroot0)
self.assertIsNone(self.sm0.getLocation(self.loc01))
self.sm0.createLocation(self.loc01)
self.assertTrue(os.path.exists(self.locroot0_path))
self.assertTrue(os.path.exists(self.loc01_path))
self.assertEqual(self.locroot0_path, self.sm0.getLocation(self.locroot0))
self.assertEqual(self.loc01_path, self.sm0.getLocation(self.loc01))
def test_createLocation3(self):
self.assertIsNone(self.sm0.getLocation(self.locroot0))
self.sm0.createLocation(self.locroot0)
self.assertIsNone(self.sm0.getLocation(self.loc01))
self.sm0.createLocation(self.loc01)
self.assertIsNone(self.sm0.getLocation(self.loc02))
self.sm0.createLocation(self.loc02)
self.assertTrue(os.path.exists(self.locroot0_path))
self.assertTrue(os.path.exists(self.loc01_path))
self.assertTrue(os.path.exists(self.loc02_path))
self.assertEqual(self.locroot0_path, self.sm0.getLocation(self.locroot0))
self.assertEqual(self.loc01_path, self.sm0.getLocation(self.loc01))
self.assertEqual(self.loc02_path, self.sm0.getLocation(self.loc02))
for sloc in self.loc02_sublocs:
self.assertIn(sloc, self.sm0.locations.keys())
def test_init3(self):
sname="StorageManager1"
sm = StorageManager(name=sname, root_path=self.sample_data_root)
self.assertEqual(sm.name, sname)
class TestFSStorageManager3(unittest.TestCase):
def setUp(self):
self.test_write_root=TEST_INVARIANTS['test_write_root']
self.sample_data_root=self.test_write_root
self.sm0 = StorageManager(root_path=self.sample_data_root)
self.locroot0='rootloc0'
self.locroot0_path=os.path.abspath(os.path.join(self.sample_data_root, self.locroot0))
self.sep0=self.sm0.sublocation_separator
self.loc01='rootloc0%s1'%self.sep0
self.loc01_path=os.path.abspath(os.path.join(self.sample_data_root, self.loc01))
self.loc02='rootloc0%s2%sa%sbb'%(self.sep0, self.sep0, self.sep0)
self.loc02_path=os.path.abspath(os.path.join(self.sample_data_root, self.loc02))
self.loc02_sublocs=('rootloc0%s2'%self.sep0, 'rootloc0%s2%sa'%(self.sep0, self.sep0))
self.loc02_sublocs_paths={
'rootloc0%s2'%self.sep0 : os.path.abspath(os.path.join(self.sample_data_root, 'rootloc0%s2'%self.sep0)),
'rootloc0%s2%sa'%(self.sep0, self.sep0) : os.path.abspath(os.path.join(
self.sample_data_root, 'rootloc0%s2%sa'%(self.sep0, self.sep0))),
}
def tearDown(self):
if os.path.exists(self.locroot0_path):
shutil.rmtree(self.locroot0_path)
def test_removeLocation1(self):
with self.assertRaises(Error):
self.sm0.removeLocation('AAA')
def test_removeLocation2(self):
self.assertIsNone(self.sm0.getLocation(self.locroot0))
self.sm0.createLocation(self.locroot0)
self.assertEqual(self.locroot0_path, self.sm0.getLocation(self.locroot0))
self.assertTrue(os.path.exists(self.locroot0_path))
self.sm0.removeLocation(self.locroot0)
self.assertNotIn(self.locroot0, self.sm0.locations)
self.assertIsNone(self.sm0.getLocation(self.locroot0))
self.assertFalse(os.path.exists(self.locroot0_path))
def test_removeLocation3(self):
self.assertIsNone(self.sm0.getLocation(self.loc02))
self.sm0.createLocation(self.loc02)
self.assertTrue(os.path.exists(self.loc02_path))
self.assertEqual(self.loc02_path, self.sm0.getLocation(self.loc02))
for sloc in self.loc02_sublocs:
self.assertIn(sloc, self.sm0.locations.keys())
self.sm0.removeLocation(self.loc02, leafonly=True)
self.assertNotIn(self.loc02, self.sm0.locations)
self.assertIsNone(self.sm0.getLocation(self.loc02))
self.assertFalse(os.path.exists(self.loc02_path))
for sl, p in self.loc02_sublocs_paths.iteritems():
self.assertIn(sl, self.sm0.locations)
self.assertEqual(p, self.sm0.getLocation(sl))
self.assertTrue(os.path.exists(p))
def test_removeLocation4(self):
self.assertIsNone(self.sm0.getLocation(self.loc02))
self.sm0.createLocation(self.loc02)
self.assertTrue(os.path.exists(self.loc02_path))
self.assertEqual(self.loc02_path, self.sm0.getLocation(self.loc02))
for sloc in self.loc02_sublocs:
self.assertIn(sloc, self.sm0.locations.keys())
self.sm0.removeLocation(self.loc02, leafonly=False)
self.assertNotIn(self.loc02, self.sm0.locations)
self.assertIsNone(self.sm0.getLocation(self.loc02))
self.assertFalse(os.path.exists(self.loc02_path))
for sl, p in self.loc02_sublocs_paths.iteritems():
self.assertNotIn(sl, self.sm0.locations)
self.assertNotEqual(p, self.sm0.getLocation(sl))
self.assertFalse(os.path.exists(p))