def testNewEntry(self): # test the default case d = dumpStorage.DumpStorage(self.testDir) dateLeafSet = set() expectedLeafs = set(['55', '00', '20']) for k,v in self.testData.items(): nd,dd = d.newEntry(k,v[1]) dateLeafSet.add(os.path.split(dd)[1]) assert os.path.isdir(nd) assert os.path.isdir(dd) assert os.path.islink(os.path.join(dd,k)) e = os.path.abspath(nd) g = os.path.abspath(os.path.join(dd,os.readlink(os.path.join(dd,k)))) assert e == g,'Expected %s, got %s'%(e,g) assert expectedLeafs == dateLeafSet, 'Expected %s, got %s'%(expectedLeafs,dateLeafSet) # test the for JsonDumpStorage default d = dumpStorage.DumpStorage(self.testDir,subSlotCount=1) dateLeafSet = set() expectedLeafs = set(['55_0', '00_0', '20_0']) for k,v in self.testData.items(): nd,dd = d.newEntry(k,v[1]) dateLeafSet.add(os.path.split(dd)[1]) assert os.path.isdir(nd) assert os.path.isdir(dd) assert os.path.islink(os.path.join(dd,k)) e = os.path.abspath(nd) g = os.path.abspath(os.path.join(dd,os.readlink(os.path.join(dd,k)))) assert e == g,'Expected %s, got %s'%(e,g) assert expectedLeafs == dateLeafSet, 'Expected %s, got %s'%(expectedLeafs,dateLeafSet) # test the trailing _n case at same level d = dumpStorage.DumpStorage(self.testDir,subSlotCount=3) dateLeafSet = set() expectedLeafs = set(['00_0', '20_0', '55_0']) for k,v in self.testData.items(): nd,dd = d.newEntry(k,v[1]) dateLeafSet.add(os.path.split(dd)[1]) assert os.path.isdir(nd) assert os.path.isdir(dd) assert os.path.islink(os.path.join(dd,k)) e = os.path.abspath(nd) g = os.path.abspath(os.path.join(dd,os.readlink(os.path.join(dd,k)))) assert e == g,'Expected %s, got %s'%(e,g) assert expectedLeafs == dateLeafSet, 'Expected %s, got %s'%(expectedLeafs,dateLeafSet) # test with subdirectory further down d = dumpStorage.DumpStorage(self.testDir,subSlotCount=3) dateLeafSet = set() expectedLeafs = set(['wh_0', 'wh_1', 'wh_2']) for k,v in self.testData.items(): nd,dd = d.newEntry(k,v[1],webheadName='wh') dateLeafSet.add(os.path.split(dd)[1]) assert os.path.isdir(nd) assert os.path.isdir(dd) assert os.path.islink(os.path.join(dd,k)) e = os.path.abspath(nd) g = os.path.abspath(os.path.join(dd,os.readlink(os.path.join(dd,k)))) assert e == g,'Expected %s, got %s'%(e,g) assert expectedLeafs == dateLeafSet, 'Expected %s, got %s'%(expectedLeafs,dateLeafSet)
def testLookupOoidInDatePath(self): d = dumpStorage.DumpStorage(self.testDir) expected = {} count = 0 for ooid,v in createJDS.jsonFileData.items(): dateS = v[0] if 0 == count%2: nd,dd = d.newEntry(ooid,datetime.datetime(*[int(x) for x in dateS.split('-')], tzinfo=UTC)) expected[ooid] = dd elif 0 == count%5: expected[ooid] = None pass else: nd,dd = d.newEntry(ooid) expected[ooid] = dd count += 1 dateS = v[0] count = 0 for ooid in createJDS.jsonFileData.keys(): dateS = v[0] if expected[ooid]: exEnd = datetime.datetime(*[int(x) for x in dateS.split('-')], tzinfo=UTC) passDate = utc_now() if 0 == count%3: passDate = None else: passDate = exEnd got,ignore = d.lookupOoidInDatePath(passDate,ooid) assert expected[ooid] == got, 'For %s: Expected %s, got %s'%(ooid,expected[ooid],got)
def testMakeNameDir(self): d = dumpStorage.DumpStorage(self.testDir) # first test: Make a file of the same name and see it fail as expected testItem = self.testData.items()[0] testOoid = testItem[0] testPath = testItem[1][0] testDate = testItem[1][1] while True: head,tail = os.path.split(testPath) if head == tail: break dirPart = os.sep.join((d.root,d.dailyPart(testOoid,testDate),d.indexName,head)) try: shutil.rmtree(d.root) except: pass filePart = os.path.join(dirPart,tail) os.makedirs(dirPart) f = open(filePart,'w') f.write("nothing\n") f.close() assert_raises(OSError,d.makeNameDir,testOoid) testPath = head try: shutil.rmtree(d.root) except: pass for k,v in self.testData.items(): g,dum = d.makeNameDir(k) e = os.path.join(d.root,d.dailyPart(k,v[1]),d.indexName,v[0]) assert e == g, 'Expected "%s" got "%s"'%(e,g)
def testRelativeNameParts(self): ooid = '12345678-dead-beef-feeb-daed2%d081225' expected = {1:['12'],2:['12','34'],3:['12','34','56'],0:['12','34','56','78']} d = dumpStorage.DumpStorage(self.testDir) for depth in range(4): tooid = ooid%(depth) assert expected[depth] == d.relativeNameParts(tooid)
def testLookupNamePath(self): d = dumpStorage.DumpStorage(self.testDir) count = 0 expected = {} for ooid, v in createJDS.jsonFileData.items(): dateS = v[0] if 0 == count % 2: nd, dd = d.newEntry( ooid, datetime.datetime(*[int(x) for x in dateS.split('-')], tzinfo=UTC)) expected[ooid] = nd elif 0 == count % 5: expected[ooid] = None pass else: nd, dd = d.newEntry(ooid) expected[ooid] = nd count += 1 for ooid, v in createJDS.jsonFileData.items(): dateS = v[0] testDate = datetime.datetime(*[int(x) for x in dateS.split('-')], tzinfo=UTC) got, ignore = d.lookupNamePath(ooid, testDate) assert expected[ ooid] == got, 'For %s, expected path %s, got %s' % ( ooid, expected, got)
def testPathToDate(self): d = dumpStorage.DumpStorage(self.testDir) testCases = [ ([ 'blob', 'f**k', 'nigl', d.root, '20081211', d.dateName, '10', '09_0' ], [2008, 12, 11, 10, 9]), ([ 'blob', 'f**k', 'nigl', d.root, '20081211', d.dateName, '10', '09', 'wh_0' ], [2008, 12, 11, 10, 9]), ([d.root, '20081211', d.dateName, '10', '09', 'wh_3'], [2008, 12, 11, 10, 9]), ([d.root, '200z1211', d.dateName, '10', '09', 'wh_3'], None), ([d.root, '20081g11', d.dateName, '10', '09', 'wh_3'], None), ([d.root, '2008121-', d.dateName, '10', '09', 'wh_3'], None), ([d.root, '20081211', d.dateName, '26', '09', 'wh_3'], None), ([d.root, '20081211', d.dateName, '10', '65', 'wh_3'], None), ([d.root, '20081311', d.dateName, '10', '09', 'wh_3'], None), ([d.root, '20081232', d.dateName, '10', '09', 'wh_3'], None), ] for (pathInfo, dateParts) in testCases: path = os.sep.join(pathInfo) if dateParts: expected = datetime.datetime(*dateParts, tzinfo=UTC) got = d.pathToDate(path) assert expected == got, 'Expected: %s but got %s' % (expected, got) else: assert_raises(ValueError, d.pathToDate, path)
def testMakeDateDir(self): d = dumpStorage.DumpStorage(self.testDir) d3 = dumpStorage.DumpStorage(self.testDir, subSlotCount=3) # first test: Make a file of the same name as a subdir and see it fail as expected testItem = self.testData.items()[0][1] date = testItem[1] datePathPart = testItem[2] while True: head, tail = os.path.split(datePathPart) if head == tail: break dirPart = os.sep.join( (d.root, d.dailyPart('', date), d.dateName, head)) try: shutil.rmtree(d.root) except: pass filePart = os.path.join(dirPart, tail) os.makedirs(dirPart) f = open(filePart, 'w') f.write("nothing\n") f.close() assert_raises(OSError, d.makeDateDir, date) assert_raises(OSError, d3.makeDateDir, date, 'boot') datePathPart = head try: shutil.rmtree(d.root) except: pass for k, v in self.testData.items(): g, dum = d.makeDateDir(v[1]) e = os.sep.join((d.root, d.dailyPart(k, v[1]), d.dateName, v[2])) g0, dum0 = d3.makeDateDir(v[1]) e0 = os.sep.join( (d.root, d.dailyPart(k, v[1]), d.dateName, "%s_%d" % (v[2], 0))) g3, dum3 = d3.makeDateDir(v[1], 'boot') e3 = os.sep.join((d.root, d.dailyPart(k, v[1]), d.dateName, v[2], "%s_%d" % ('boot', 0))) assert e == g, 'Expected "%s", got "%s"' % (e, g) assert os.path.isdir(g), 'But "%s" is not a dir' % g assert e0 == g0, 'Expected "%s", got "%s"' % (e0, g0) assert os.path.isdir(g0), 'But "%s" is not a dir' % g assert e3 == g3, 'Expected "%s", got "%s"' % (e3, g3) assert os.path.isdir(g3), 'But "%s" is not a dir' % g
def testDatePath(self): d = dumpStorage.DumpStorage(self.testDir) for k,v in self.testData.items(): g = d.datePath(v[1])[0] e = os.sep.join((d.root,d.dailyPart(k,v[1]),d.dateName,v[2])) assert e == g, 'Expected "%s", got "%s"'%(e,g) d = dumpStorage.DumpStorage(self.testDir,subSlotCount=3) curcount = 0 for k,v in self.testData.items(): g = d.datePath(v[1])[0] e = os.sep.join((d.root,d.dailyPart(k,v[1]),d.dateName,"%s_%d"%(v[2],curcount))) #curcount = (curcount + 1) % d.subSlotCount assert e == g, 'Expected "%s", got "%s"'%(e,g) curcount = 0 for k,v in self.testData.items(): g = d.datePath(v[1],webheadName='boot')[0] e = os.sep.join((d.root,d.dailyPart(k,v[1]),d.dateName,v[2],"%s_%d"%('boot',curcount))) #curcount = (curcount + 1) % d.subSlotCount assert e == g, 'Expected "%s", got "%s"'%(e,g)
def testConstructor(self): for i in range(len(self.expectedCtor)): if i in (1,3,5): root = self.expectedTestDir else: root = self.testDir d = dumpStorage.DumpStorage(root,**self.ctorData[i]) for k in self.expectedCtor[i]: e = self.expectedCtor[i][k] g = eval("d."+k) if type(1) == type(e): assert e == g,'At loop %d, key %s: Wanted "%0o", got "%0o"'%(i,k,e,g) else: assert e == g,'At loop %d, key %s: Wanted "%s", got "%s"'%(i,k,e,g)
def testDailyPart(self): d = dumpStorage.DumpStorage(self.testDir) testData = [ ('12345678-dead-beef-feeb-daed20081225',datetime.datetime(2008,12,25,1,2,3, tzinfo=UTC),'20081225'), ('12345678-dead-beef-feeb-daed20081225',datetime.datetime(2008,12,26,1,2,3, tzinfo=UTC),'20081226'), ('12345678-dead-beef-feeb-daed20081225',None,'20081225'), ('',datetime.datetime(2008,12,25,1,2,3, tzinfo=UTC),'20081225'), (None,None,None), ('',None,None), ] for ooid,date,expected in testData: if expected: got = d.dailyPart(ooid,date) assert expected == got, 'Expected "%s" but got "%s"'%(expected,got) else: now = utc_now() expected = "%4d%02d%02d"%(now.year,now.month,now.day) assert expected == d.dailyPart(ooid,date), 'From (%s,%s) Expected "%s" but got "%s"'%(ooid,date,expected,got)
def testNamePath(self): d = dumpStorage.DumpStorage(self.testDir) for k,v in self.testData.items(): g = d.namePath(k,v[1])[0] e = os.sep.join((d.root,d.dailyPart(k,v[1]),d.indexName,v[0])) assert e == g, 'Expected "%s", got "%s"'%(e,g)