def testNewEntry(self):
# test the default case
d = dumpStorage.DumpStorage(self.testDir)
dateLeafSet = set()
expectedLeafs = set(['55', '00', '20'])
for k,v in self.testData.items():
nd,dd = d.newEntry(k,v[1])
dateLeafSet.add(os.path.split(dd)[1])
assert os.path.isdir(nd)
assert os.path.isdir(dd)
assert os.path.islink(os.path.join(dd,k))
e = os.path.abspath(nd)
g = os.path.abspath(os.path.join(dd,os.readlink(os.path.join(dd,k))))
assert e == g,'Expected %s, got %s'%(e,g)
assert expectedLeafs == dateLeafSet, 'Expected %s, got %s'%(expectedLeafs,dateLeafSet)
# test the for JsonDumpStorage default
d = dumpStorage.DumpStorage(self.testDir,subSlotCount=1)
dateLeafSet = set()
expectedLeafs = set(['55_0', '00_0', '20_0'])
for k,v in self.testData.items():
nd,dd = d.newEntry(k,v[1])
dateLeafSet.add(os.path.split(dd)[1])
assert os.path.isdir(nd)
assert os.path.isdir(dd)
assert os.path.islink(os.path.join(dd,k))
e = os.path.abspath(nd)
g = os.path.abspath(os.path.join(dd,os.readlink(os.path.join(dd,k))))
assert e == g,'Expected %s, got %s'%(e,g)
assert expectedLeafs == dateLeafSet, 'Expected %s, got %s'%(expectedLeafs,dateLeafSet)
# test the trailing _n case at same level
d = dumpStorage.DumpStorage(self.testDir,subSlotCount=3)
dateLeafSet = set()
expectedLeafs = set(['00_0', '20_0', '55_0'])
for k,v in self.testData.items():
nd,dd = d.newEntry(k,v[1])
dateLeafSet.add(os.path.split(dd)[1])
assert os.path.isdir(nd)
assert os.path.isdir(dd)
assert os.path.islink(os.path.join(dd,k))
e = os.path.abspath(nd)
g = os.path.abspath(os.path.join(dd,os.readlink(os.path.join(dd,k))))
assert e == g,'Expected %s, got %s'%(e,g)
assert expectedLeafs == dateLeafSet, 'Expected %s, got %s'%(expectedLeafs,dateLeafSet)
# test with subdirectory further down
d = dumpStorage.DumpStorage(self.testDir,subSlotCount=3)
dateLeafSet = set()
expectedLeafs = set(['wh_0', 'wh_1', 'wh_2'])
for k,v in self.testData.items():
nd,dd = d.newEntry(k,v[1],webheadName='wh')
dateLeafSet.add(os.path.split(dd)[1])
assert os.path.isdir(nd)
assert os.path.isdir(dd)
assert os.path.islink(os.path.join(dd,k))
e = os.path.abspath(nd)
g = os.path.abspath(os.path.join(dd,os.readlink(os.path.join(dd,k))))
assert e == g,'Expected %s, got %s'%(e,g)
assert expectedLeafs == dateLeafSet, 'Expected %s, got %s'%(expectedLeafs,dateLeafSet)
def testLookupOoidInDatePath(self):
d = dumpStorage.DumpStorage(self.testDir)
expected = {}
count = 0
for ooid,v in createJDS.jsonFileData.items():
dateS = v[0]
if 0 == count%2:
nd,dd = d.newEntry(ooid,datetime.datetime(*[int(x) for x in dateS.split('-')], tzinfo=UTC))
expected[ooid] = dd
elif 0 == count%5:
expected[ooid] = None
pass
else:
nd,dd = d.newEntry(ooid)
expected[ooid] = dd
count += 1
dateS = v[0]
count = 0
for ooid in createJDS.jsonFileData.keys():
dateS = v[0]
if expected[ooid]:
exEnd = datetime.datetime(*[int(x) for x in dateS.split('-')], tzinfo=UTC)
passDate = utc_now()
if 0 == count%3:
passDate = None
else:
passDate = exEnd
got,ignore = d.lookupOoidInDatePath(passDate,ooid)
assert expected[ooid] == got, 'For %s: Expected %s, got %s'%(ooid,expected[ooid],got)
def testMakeNameDir(self):
d = dumpStorage.DumpStorage(self.testDir)
# first test: Make a file of the same name and see it fail as expected
testItem = self.testData.items()[0]
testOoid = testItem[0]
testPath = testItem[1][0]
testDate = testItem[1][1]
while True:
head,tail = os.path.split(testPath)
if head == tail: break
dirPart = os.sep.join((d.root,d.dailyPart(testOoid,testDate),d.indexName,head))
try:
shutil.rmtree(d.root)
except:
pass
filePart = os.path.join(dirPart,tail)
os.makedirs(dirPart)
f = open(filePart,'w')
f.write("nothing\n")
f.close()
assert_raises(OSError,d.makeNameDir,testOoid)
testPath = head
try:
shutil.rmtree(d.root)
except:
pass
for k,v in self.testData.items():
g,dum = d.makeNameDir(k)
e = os.path.join(d.root,d.dailyPart(k,v[1]),d.indexName,v[0])
assert e == g, 'Expected "%s" got "%s"'%(e,g)
def testRelativeNameParts(self):
ooid = '12345678-dead-beef-feeb-daed2%d081225'
expected = {1:['12'],2:['12','34'],3:['12','34','56'],0:['12','34','56','78']}
d = dumpStorage.DumpStorage(self.testDir)
for depth in range(4):
tooid = ooid%(depth)
assert expected[depth] == d.relativeNameParts(tooid)
def testLookupNamePath(self):
d = dumpStorage.DumpStorage(self.testDir)
count = 0
expected = {}
for ooid, v in createJDS.jsonFileData.items():
dateS = v[0]
if 0 == count % 2:
nd, dd = d.newEntry(
ooid,
datetime.datetime(*[int(x) for x in dateS.split('-')],
tzinfo=UTC))
expected[ooid] = nd
elif 0 == count % 5:
expected[ooid] = None
pass
else:
nd, dd = d.newEntry(ooid)
expected[ooid] = nd
count += 1
for ooid, v in createJDS.jsonFileData.items():
dateS = v[0]
testDate = datetime.datetime(*[int(x) for x in dateS.split('-')],
tzinfo=UTC)
got, ignore = d.lookupNamePath(ooid, testDate)
assert expected[
ooid] == got, 'For %s, expected path %s, got %s' % (
ooid, expected, got)
def testPathToDate(self):
d = dumpStorage.DumpStorage(self.testDir)
testCases = [
([
'blob', 'f**k', 'nigl', d.root, '20081211', d.dateName, '10',
'09_0'
], [2008, 12, 11, 10, 9]),
([
'blob', 'f**k', 'nigl', d.root, '20081211', d.dateName, '10',
'09', 'wh_0'
], [2008, 12, 11, 10, 9]),
([d.root, '20081211', d.dateName, '10', '09',
'wh_3'], [2008, 12, 11, 10, 9]),
([d.root, '200z1211', d.dateName, '10', '09', 'wh_3'], None),
([d.root, '20081g11', d.dateName, '10', '09', 'wh_3'], None),
([d.root, '2008121-', d.dateName, '10', '09', 'wh_3'], None),
([d.root, '20081211', d.dateName, '26', '09', 'wh_3'], None),
([d.root, '20081211', d.dateName, '10', '65', 'wh_3'], None),
([d.root, '20081311', d.dateName, '10', '09', 'wh_3'], None),
([d.root, '20081232', d.dateName, '10', '09', 'wh_3'], None),
]
for (pathInfo, dateParts) in testCases:
path = os.sep.join(pathInfo)
if dateParts:
expected = datetime.datetime(*dateParts, tzinfo=UTC)
got = d.pathToDate(path)
assert expected == got, 'Expected: %s but got %s' % (expected,
got)
else:
assert_raises(ValueError, d.pathToDate, path)
def testMakeDateDir(self):
d = dumpStorage.DumpStorage(self.testDir)
d3 = dumpStorage.DumpStorage(self.testDir, subSlotCount=3)
# first test: Make a file of the same name as a subdir and see it fail as expected
testItem = self.testData.items()[0][1]
date = testItem[1]
datePathPart = testItem[2]
while True:
head, tail = os.path.split(datePathPart)
if head == tail: break
dirPart = os.sep.join(
(d.root, d.dailyPart('', date), d.dateName, head))
try:
shutil.rmtree(d.root)
except:
pass
filePart = os.path.join(dirPart, tail)
os.makedirs(dirPart)
f = open(filePart, 'w')
f.write("nothing\n")
f.close()
assert_raises(OSError, d.makeDateDir, date)
assert_raises(OSError, d3.makeDateDir, date, 'boot')
datePathPart = head
try:
shutil.rmtree(d.root)
except:
pass
for k, v in self.testData.items():
g, dum = d.makeDateDir(v[1])
e = os.sep.join((d.root, d.dailyPart(k, v[1]), d.dateName, v[2]))
g0, dum0 = d3.makeDateDir(v[1])
e0 = os.sep.join(
(d.root, d.dailyPart(k,
v[1]), d.dateName, "%s_%d" % (v[2], 0)))
g3, dum3 = d3.makeDateDir(v[1], 'boot')
e3 = os.sep.join((d.root, d.dailyPart(k, v[1]), d.dateName, v[2],
"%s_%d" % ('boot', 0)))
assert e == g, 'Expected "%s", got "%s"' % (e, g)
assert os.path.isdir(g), 'But "%s" is not a dir' % g
assert e0 == g0, 'Expected "%s", got "%s"' % (e0, g0)
assert os.path.isdir(g0), 'But "%s" is not a dir' % g
assert e3 == g3, 'Expected "%s", got "%s"' % (e3, g3)
assert os.path.isdir(g3), 'But "%s" is not a dir' % g
def testDatePath(self):
d = dumpStorage.DumpStorage(self.testDir)
for k,v in self.testData.items():
g = d.datePath(v[1])[0]
e = os.sep.join((d.root,d.dailyPart(k,v[1]),d.dateName,v[2]))
assert e == g, 'Expected "%s", got "%s"'%(e,g)
d = dumpStorage.DumpStorage(self.testDir,subSlotCount=3)
curcount = 0
for k,v in self.testData.items():
g = d.datePath(v[1])[0]
e = os.sep.join((d.root,d.dailyPart(k,v[1]),d.dateName,"%s_%d"%(v[2],curcount)))
#curcount = (curcount + 1) % d.subSlotCount
assert e == g, 'Expected "%s", got "%s"'%(e,g)
curcount = 0
for k,v in self.testData.items():
g = d.datePath(v[1],webheadName='boot')[0]
e = os.sep.join((d.root,d.dailyPart(k,v[1]),d.dateName,v[2],"%s_%d"%('boot',curcount)))
#curcount = (curcount + 1) % d.subSlotCount
assert e == g, 'Expected "%s", got "%s"'%(e,g)
def testConstructor(self):
for i in range(len(self.expectedCtor)):
if i in (1,3,5):
root = self.expectedTestDir
else:
root = self.testDir
d = dumpStorage.DumpStorage(root,**self.ctorData[i])
for k in self.expectedCtor[i]:
e = self.expectedCtor[i][k]
g = eval("d."+k)
if type(1) == type(e):
assert e == g,'At loop %d, key %s: Wanted "%0o", got "%0o"'%(i,k,e,g)
else:
assert e == g,'At loop %d, key %s: Wanted "%s", got "%s"'%(i,k,e,g)
def testDailyPart(self):
d = dumpStorage.DumpStorage(self.testDir)
testData = [
('12345678-dead-beef-feeb-daed20081225',datetime.datetime(2008,12,25,1,2,3, tzinfo=UTC),'20081225'),
('12345678-dead-beef-feeb-daed20081225',datetime.datetime(2008,12,26,1,2,3, tzinfo=UTC),'20081226'),
('12345678-dead-beef-feeb-daed20081225',None,'20081225'),
('',datetime.datetime(2008,12,25,1,2,3, tzinfo=UTC),'20081225'),
(None,None,None),
('',None,None),
]
for ooid,date,expected in testData:
if expected:
got = d.dailyPart(ooid,date)
assert expected == got, 'Expected "%s" but got "%s"'%(expected,got)
else:
now = utc_now()
expected = "%4d%02d%02d"%(now.year,now.month,now.day)
assert expected == d.dailyPart(ooid,date), 'From (%s,%s) Expected "%s" but got "%s"'%(ooid,date,expected,got)
def testNamePath(self):
d = dumpStorage.DumpStorage(self.testDir)
for k,v in self.testData.items():
g = d.namePath(k,v[1])[0]
e = os.sep.join((d.root,d.dailyPart(k,v[1]),d.indexName,v[0]))
assert e == g, 'Expected "%s", got "%s"'%(e,g)