def testMakeNameDir(self):
d = dumpStorage.DumpStorage(self.testDir)
# first test: Make a file of the same name and see it fail as expected
testItem = self.testData.items()[0]
testOoid = testItem[0]
testPath = testItem[1][0]
testDate = testItem[1][1]
while True:
head, tail = os.path.split(testPath)
if head == tail: break
dirPart = os.sep.join(
(d.root, d.dailyPart(testOoid, testDate), d.indexName, head))
try:
shutil.rmtree(d.root)
except:
pass
filePart = os.path.join(dirPart, tail)
os.makedirs(dirPart)
f = open(filePart, 'w')
f.write("nothing\n")
f.close()
assert_raises(OSError, d.makeNameDir, testOoid)
testPath = head
try:
shutil.rmtree(d.root)
except:
pass
for k, v in self.testData.items():
g, dum = d.makeNameDir(k)
e = os.path.join(d.root, d.dailyPart(k, v[1]), d.indexName, v[0])
assert e == g, 'Expected "%s" got "%s"' % (e, g)
def testLookupOoidInDatePath(self):
d = dumpStorage.DumpStorage(self.testDir)
expected = {}
count = 0
for ooid, v in createJDS.jsonFileData.items():
dateS = v[0]
if 0 == count % 2:
nd, dd = d.newEntry(
ooid,
datetime.datetime(*[int(x) for x in dateS.split('-')],
tzinfo=UTC))
expected[ooid] = dd
elif 0 == count % 5:
expected[ooid] = None
pass
else:
nd, dd = d.newEntry(ooid)
expected[ooid] = dd
count += 1
dateS = v[0]
count = 0
for ooid in createJDS.jsonFileData.keys():
dateS = v[0]
if expected[ooid]:
exEnd = datetime.datetime(*[int(x) for x in dateS.split('-')],
tzinfo=UTC)
passDate = utc_now()
if 0 == count % 3:
passDate = None
else:
passDate = exEnd
got, ignore = d.lookupOoidInDatePath(passDate, ooid)
assert expected[ooid] == got, 'For %s: Expected %s, got %s' % (
ooid, expected[ooid], got)
def testLookupNamePath(self):
d = dumpStorage.DumpStorage(self.testDir)
count = 0
expected = {}
for ooid, v in createJDS.jsonFileData.items():
dateS = v[0]
if 0 == count % 2:
nd, dd = d.newEntry(
ooid,
datetime.datetime(*[int(x) for x in dateS.split('-')],
tzinfo=UTC))
expected[ooid] = nd
elif 0 == count % 5:
expected[ooid] = None
pass
else:
nd, dd = d.newEntry(ooid)
expected[ooid] = nd
count += 1
for ooid, v in createJDS.jsonFileData.items():
dateS = v[0]
testDate = datetime.datetime(*[int(x) for x in dateS.split('-')],
tzinfo=UTC)
got, ignore = d.lookupNamePath(ooid, testDate)
assert expected[
ooid] == got, 'For %s, expected path %s, got %s' % (
ooid, expected, got)
def testPathToDate(self):
d = dumpStorage.DumpStorage(self.testDir)
testCases = [
([
'blob', 'f**k', 'nigl', d.root, '20081211', d.dateName, '10',
'09_0'
], [2008, 12, 11, 10, 9]),
([
'blob', 'f**k', 'nigl', d.root, '20081211', d.dateName, '10',
'09', 'wh_0'
], [2008, 12, 11, 10, 9]),
([d.root, '20081211', d.dateName, '10', '09',
'wh_3'], [2008, 12, 11, 10, 9]),
([d.root, '200z1211', d.dateName, '10', '09', 'wh_3'], None),
([d.root, '20081g11', d.dateName, '10', '09', 'wh_3'], None),
([d.root, '2008121-', d.dateName, '10', '09', 'wh_3'], None),
([d.root, '20081211', d.dateName, '26', '09', 'wh_3'], None),
([d.root, '20081211', d.dateName, '10', '65', 'wh_3'], None),
([d.root, '20081311', d.dateName, '10', '09', 'wh_3'], None),
([d.root, '20081232', d.dateName, '10', '09', 'wh_3'], None),
]
for (pathInfo, dateParts) in testCases:
path = os.sep.join(pathInfo)
if dateParts:
expected = datetime.datetime(*dateParts, tzinfo=UTC)
got = d.pathToDate(path)
assert expected == got, 'Expected: %s but got %s' % (expected,
got)
else:
assert_raises(ValueError, d.pathToDate, path)
def testMakeDateDir(self):
d = dumpStorage.DumpStorage(self.testDir)
d3 = dumpStorage.DumpStorage(self.testDir, subSlotCount=3)
# first test: Make a file of the same name as a subdir and see it fail as expected
testItem = self.testData.items()[0][1]
date = testItem[1]
datePathPart = testItem[2]
while True:
head, tail = os.path.split(datePathPart)
if head == tail: break
dirPart = os.sep.join(
(d.root, d.dailyPart('', date), d.dateName, head))
try:
shutil.rmtree(d.root)
except:
pass
filePart = os.path.join(dirPart, tail)
os.makedirs(dirPart)
f = open(filePart, 'w')
f.write("nothing\n")
f.close()
assert_raises(OSError, d.makeDateDir, date)
assert_raises(OSError, d3.makeDateDir, date, 'boot')
datePathPart = head
try:
shutil.rmtree(d.root)
except:
pass
for k, v in self.testData.items():
g, dum = d.makeDateDir(v[1])
e = os.sep.join((d.root, d.dailyPart(k, v[1]), d.dateName, v[2]))
g0, dum0 = d3.makeDateDir(v[1])
e0 = os.sep.join(
(d.root, d.dailyPart(k,
v[1]), d.dateName, "%s_%d" % (v[2], 0)))
g3, dum3 = d3.makeDateDir(v[1], 'boot')
e3 = os.sep.join((d.root, d.dailyPart(k, v[1]), d.dateName, v[2],
"%s_%d" % ('boot', 0)))
assert e == g, 'Expected "%s", got "%s"' % (e, g)
assert os.path.isdir(g), 'But "%s" is not a dir' % g
assert e0 == g0, 'Expected "%s", got "%s"' % (e0, g0)
assert os.path.isdir(g0), 'But "%s" is not a dir' % g
assert e3 == g3, 'Expected "%s", got "%s"' % (e3, g3)
assert os.path.isdir(g3), 'But "%s" is not a dir' % g
def testDatePath(self):
d = dumpStorage.DumpStorage(self.testDir)
for k, v in self.testData.items():
g = d.datePath(v[1])[0]
e = os.sep.join((d.root, d.dailyPart(k, v[1]), d.dateName, v[2]))
assert e == g, 'Expected "%s", got "%s"' % (e, g)
d = dumpStorage.DumpStorage(self.testDir, subSlotCount=3)
curcount = 0
for k, v in self.testData.items():
g = d.datePath(v[1])[0]
e = os.sep.join((d.root, d.dailyPart(k, v[1]), d.dateName,
"%s_%d" % (v[2], curcount)))
#curcount = (curcount + 1) % d.subSlotCount
assert e == g, 'Expected "%s", got "%s"' % (e, g)
curcount = 0
for k, v in self.testData.items():
g = d.datePath(v[1], webheadName='boot')[0]
e = os.sep.join((d.root, d.dailyPart(k, v[1]), d.dateName, v[2],
"%s_%d" % ('boot', curcount)))
#curcount = (curcount + 1) % d.subSlotCount
assert e == g, 'Expected "%s", got "%s"' % (e, g)
def testRelativeNameParts(self):
ooid = '12345678-dead-beef-feeb-daed2%d081225'
expected = {
1: ['12'],
2: ['12', '34'],
3: ['12', '34', '56'],
0: ['12', '34', '56', '78']
}
d = dumpStorage.DumpStorage(self.testDir)
for depth in range(4):
tooid = ooid % (depth)
assert expected[depth] == d.relativeNameParts(tooid)
def testConstructor(self):
for i in range(len(self.expectedCtor)):
if i in (1, 3, 5):
root = self.expectedTestDir
else:
root = self.testDir
d = dumpStorage.DumpStorage(root, **self.ctorData[i])
for k in self.expectedCtor[i]:
e = self.expectedCtor[i][k]
g = eval("d." + k)
if type(1) == type(e):
assert e == g, 'At loop %d, key %s: Wanted "%0o", got "%0o"' % (
i, k, e, g)
else:
assert e == g, 'At loop %d, key %s: Wanted "%s", got "%s"' % (
i, k, e, g)
def testDailyPart(self):
d = dumpStorage.DumpStorage(self.testDir)
testData = [
('12345678-dead-beef-feeb-daed20081225',
datetime.datetime(2008, 12, 25, 1, 2, 3, tzinfo=UTC), '20081225'),
('12345678-dead-beef-feeb-daed20081225',
datetime.datetime(2008, 12, 26, 1, 2, 3, tzinfo=UTC), '20081226'),
('12345678-dead-beef-feeb-daed20081225', None, '20081225'),
('', datetime.datetime(2008, 12, 25, 1, 2, 3,
tzinfo=UTC), '20081225'),
(None, None, None),
('', None, None),
]
for ooid, date, expected in testData:
if expected:
got = d.dailyPart(ooid, date)
assert expected == got, 'Expected "%s" but got "%s"' % (
expected, got)
else:
now = utc_now()
expected = "%4d%02d%02d" % (now.year, now.month, now.day)
assert expected == d.dailyPart(
ooid, date), 'From (%s,%s) Expected "%s" but got "%s"' % (
ooid, date, expected, got)
def testNamePath(self):
d = dumpStorage.DumpStorage(self.testDir)
for k, v in self.testData.items():
g = d.namePath(k, v[1])[0]
e = os.sep.join((d.root, d.dailyPart(k, v[1]), d.indexName, v[0]))
assert e == g, 'Expected "%s", got "%s"' % (e, g)
def testNewEntry(self):
# test the default case
d = dumpStorage.DumpStorage(self.testDir)
dateLeafSet = set()
expectedLeafs = set(['55', '00', '20'])
for k, v in self.testData.items():
nd, dd = d.newEntry(k, v[1])
dateLeafSet.add(os.path.split(dd)[1])
assert os.path.isdir(nd)
assert os.path.isdir(dd)
assert os.path.islink(os.path.join(dd, k))
e = os.path.abspath(nd)
g = os.path.abspath(
os.path.join(dd, os.readlink(os.path.join(dd, k))))
assert e == g, 'Expected %s, got %s' % (e, g)
assert expectedLeafs == dateLeafSet, 'Expected %s, got %s' % (
expectedLeafs, dateLeafSet)
# test the for JsonDumpStorage default
d = dumpStorage.DumpStorage(self.testDir, subSlotCount=1)
dateLeafSet = set()
expectedLeafs = set(['55_0', '00_0', '20_0'])
for k, v in self.testData.items():
nd, dd = d.newEntry(k, v[1])
dateLeafSet.add(os.path.split(dd)[1])
assert os.path.isdir(nd)
assert os.path.isdir(dd)
assert os.path.islink(os.path.join(dd, k))
e = os.path.abspath(nd)
g = os.path.abspath(
os.path.join(dd, os.readlink(os.path.join(dd, k))))
assert e == g, 'Expected %s, got %s' % (e, g)
assert expectedLeafs == dateLeafSet, 'Expected %s, got %s' % (
expectedLeafs, dateLeafSet)
# test the trailing _n case at same level
d = dumpStorage.DumpStorage(self.testDir, subSlotCount=3)
dateLeafSet = set()
expectedLeafs = set(['00_0', '20_0', '55_0'])
for k, v in self.testData.items():
nd, dd = d.newEntry(k, v[1])
dateLeafSet.add(os.path.split(dd)[1])
assert os.path.isdir(nd)
assert os.path.isdir(dd)
assert os.path.islink(os.path.join(dd, k))
e = os.path.abspath(nd)
g = os.path.abspath(
os.path.join(dd, os.readlink(os.path.join(dd, k))))
assert e == g, 'Expected %s, got %s' % (e, g)
assert expectedLeafs == dateLeafSet, 'Expected %s, got %s' % (
expectedLeafs, dateLeafSet)
# test with subdirectory further down
d = dumpStorage.DumpStorage(self.testDir, subSlotCount=3)
dateLeafSet = set()
expectedLeafs = set(['wh_0', 'wh_1', 'wh_2'])
for k, v in self.testData.items():
nd, dd = d.newEntry(k, v[1], webheadName='wh')
dateLeafSet.add(os.path.split(dd)[1])
assert os.path.isdir(nd)
assert os.path.isdir(dd)
assert os.path.islink(os.path.join(dd, k))
e = os.path.abspath(nd)
g = os.path.abspath(
os.path.join(dd, os.readlink(os.path.join(dd, k))))
assert e == g, 'Expected %s, got %s' % (e, g)
assert expectedLeafs == dateLeafSet, 'Expected %s, got %s' % (
expectedLeafs, dateLeafSet)