def testTransferOne(self):
createJDS.createTestSet(createJDS.jsonFileData,
self.initKwargs[0],
rootDir=self.testMoveFrom)
storage = JDS.JsonDumpStorage(self.testDir, **self.initKwargs[0])
oldStorage = JDS.JsonDumpStorage(self.testMoveFrom,
**self.initKwargs[0])
itemNumber = 0
xmas = datetime.datetime(2001, 12, 25, 12, 25, tzinfo=UTC)
for id in createJDS.jsonFileData.keys():
createLinks = 0 == itemNumber % 2
removeOld = 0 == itemNumber % 3
itemNumber += 1
storage.transferOne(id,
oldStorage,
createLinks=createLinks,
removeOld=removeOld,
aDate=xmas)
try:
storage.getJson(id)
except Exception, x:
print '(%s): %s' % (type(x), x)
assert False, 'Expected to find a transferred json file for %s' % id
if createLinks:
self.__hasLinkOrFail(storage, id)
self.__hasDatePathOrFail(storage, id, xmas)
if removeOld:
assert_raises(OSError, oldStorage.getJson, id)
def testDestructiveDateWalkNotNow(self):
createJDS.createTestSet(self.currenttimes,self.initKwargs[1],self.testDir)
storage = JDS.JsonDumpStorage(self.testDir,**self.initKwargs[1])
seenids = []
for id in storage.destructiveDateWalk():
seenids.append(id)
assert [] == seenids
def testMove(self):
createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[3],self.testDir)
storage = JDS.JsonDumpStorage(self.testDir,**self.initKwargs[3])
os.mkdir(self.testMoveTo)
for uuid in createJDS.jsonFileData.keys():
storage.move(uuid,os.path.join('.','TEST-MOVETO'))
allfiles = []
alllinks = []
for dir, dirs, files in os.walk(self.testDir):
for file in files:
allfiles.append(file)
if os.path.islink(os.path.join(dir,file)):
alllinks.append(file)
for d in dirs:
if os.path.islink(os.path.join(dir,d)):
alllinks.append(d)
assert [] == allfiles, 'Expect that all moved files are gone, but found %s' % allfiles
assert [] == alllinks, 'Expcet that all links are gone, but found %s' % alllinks
allfiles = []
alllinks = []
expectedFiles = [x+storage.jsonSuffix for x in createJDS.jsonFileData.keys() ]
expectedFiles.extend([x+storage.dumpSuffix for x in createJDS.jsonFileData.keys() ])
for dir, dirs, files in os.walk(os.path.join('.','TEST-MOVETO')):
for file in files:
allfiles.append(file)
assert file in expectedFiles, 'Expect that each moved file will be expected but found %s' % file
if os.path.islink(os.path.join(dir,file)): alllinks.append(file)
for d in dirs:
if os.path.islink(os.path.join(dir,d)): alllinks.append(d)
assert [] == alllinks, 'Expect no links in the move-to directory, but found %s' % alllinks
for file in expectedFiles:
assert file in allfiles, 'Expect that every file will be moved but did not find %s' % file
def testRemove(self):
createJDS.createTestSet(createJDS.jsonFileData, self.initKwargs[2],
self.testDir)
storage = JDS.JsonDumpStorage(self.testDir, **self.initKwargs[2])
counter = 0
for uuid in createJDS.jsonFileData.keys():
if 0 == counter % 3:
# test that we don't throw for missing links
storage.markAsSeen(uuid)
if 1 == counter % 3:
# test that we don't throw for one missing file
if 0 == counter % 2:
os.unlink(storage.getDump(uuid))
else:
os.unlink(storage.getJson(uuid))
if 2 == counter % 3:
# test that we don't throw for both missing files, but with links
os.unlink(storage.getJson(uuid))
os.unlink(storage.getDump(uuid))
storage.remove(uuid)
counter += 1
allfiles = []
alllinks = []
for dir, dirs, files in os.walk(self.testDir):
for file in files:
allfiles.append(file)
if os.path.islink(os.path.join(dir, file)):
alllinks.append(file)
for d in dirs:
if os.path.islink(os.path.join(dir, d)):
alllinks.append(d)
assert [] == allfiles, 'Expect that all removed files are gone, but found %s' % allfiles
assert [] == alllinks, 'Expect that all links are gone, but found %s' % alllinks
assert_raises(JDS.NoSuchUuidFound, storage.remove, "bogusdatax3yymmdd")
def markAsSeen(self):
createJDS.createTestSet(createJDS.jsonFileData, self.initKwargs[3],
self.testDir)
storage = JDS.JsonDumpStorage(self.testDir, **self.initKwargs[3])
for uuid, data in createJDS.jsonFileData.items():
assert os.path.islink(
os.sep.join(
(storage.dateBranch, data[3],
uuid))), 'Expect a link from date to name for %s' % uuid
assert os.path.islink(
os.sep.join(
(storage.nameBranch, data[2],
uuid))), 'Expect link from name to timed for %s' % uuid
assert not os.path.islink(
os.sep.join(
(storage.dateBranch, data[3],
uuid))), 'Expect no link from date to name for %s' % uuid
assert not os.path.islink(
os.sep.join(
(storage.nameBranch, data[2],
uuid))), 'Expect no link from name to date for %s' % uuid
try:
storage.markAsSeen(createJDS.jsonBadUuid)
assert False, 'Expect to throw IOError from attempt to openAndMarkAsSeen(non-existent-uuid)'
except IOError:
assert True, 'Got expected error from attempt to openAndMarkAsSeen(non-existent-uuid)'
except Exception, e:
assert False, 'Got unexpected error %s from attempt to openAndMarkAsSeen(non-existent-uuid' % e
def testTransferMany(self):
createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[0],rootDir=self.testMoveFrom)
oldStorage = JDS.JsonDumpStorage(self.testMoveFrom, **self.initKwargs[0])
itemNumber = 0
xmas = DT.datetime(2001,12,25,12,25)
hasLinks = {}
for id in createJDS.jsonFileData.keys():
hasLinks[id] = True
if 0 == itemNumber %2 :
oldStorage.markAsSeen(id)
self.__hasNoLinkOrFail(oldStorage,id)
hasLinks[id] = False
opts = ((False,True),(True,False),(False,False)) #copyLinks, makeNewLinks
targets = (self.testMoveTo, self.testMoveToAlt,self.testDir)
assert len(opts) == len(targets), "set of opts must be one-to-one with set of targets, or fail"
for i in range(len(opts)):
aDate = None
if opts[i][1]: aDate = xmas
storage = JDS.JsonDumpStorage(targets[i], **self.initKwargs[0])
storage.transferMany(createJDS.jsonFileData.keys(),oldStorage,copyLinksBoolean=opts[i][0],makeNewDateLinksBoolean=opts[i][1],aDate=aDate)
for id in createJDS.jsonFileData.keys():
try:
storage.getJson(id)
except Exception,x:
print '(%s): %s'%(type(x),x)
assert False, 'Expected to find a transferred json file for %s' % id
if opts[i][1] or hasLinks[id]:
self.__hasLinkOrFail(storage,id)
if opts[i][1]:
self.__hasDatePathOrFail(storage,xmas)
if not opts[i][1] and not hasLinks[id]:
self.__hasNoLinkOrFail(storage,id)
def testDestructiveDateWalkNotNow(self):
createJDS.createTestSet(self.currenttimes,self.initKwargs[1],self.testDir)
storage = JDS.JsonDumpStorage(self.testDir,**self.initKwargs[1])
seenids = []
for id in storage.destructiveDateWalk():
seenids.append(id)
assert [] == seenids
def testRemove(self):
createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[2],self.testDir)
storage = JDS.JsonDumpStorage(self.testDir,**self.initKwargs[2])
counter = 0
for uuid in createJDS.jsonFileData.keys():
if 0 == counter % 3:
# test that we don't throw for missing links
storage.markAsSeen(uuid)
if 1 == counter % 3:
# test that we don't throw for one missing file
if 0 == counter % 2:
os.unlink(storage.getDump(uuid))
else:
os.unlink(storage.getJson(uuid))
if 2 == counter % 3:
# test that we don't throw for both missing files, but with links
os.unlink(storage.getJson(uuid))
os.unlink(storage.getDump(uuid))
storage.remove(uuid)
counter += 1
allfiles = []
alllinks = []
for dir, dirs, files in os.walk(self.testDir):
for file in files:
allfiles.append(file)
if os.path.islink(os.path.join(dir,file)):
alllinks.append(file)
for d in dirs:
if os.path.islink(os.path.join(dir,d)):
alllinks.append(d)
assert [] == allfiles, 'Expect that all removed files are gone, but found %s' % allfiles
assert [] == alllinks, 'Expect that all links are gone, but found %s' % alllinks
assert_raises(JDS.NoSuchUuidFound, storage.remove, "bogusdatax3yymmdd")
def testCleanUpCompletedAndFailedJobs_WithSaves(self):
"""
testCleanUpCompletedAndFailedJobs_WithSaves(self):
The default config asks for successful and failed jobs to be saved
"""
global me
cursor = self.connection.cursor()
dbtestutil.fillProcessorTable(cursor,4)
m = monitor.Monitor(me.config)
createJDS.createTestSet(createJDS.jsonFileData,jsonKwargs={'logger':me.logger},rootDir=me.config.storageRoot)
runInOtherProcess(m.standardJobAllocationLoop, stopCondition=(lambda : self.jobsAllocated() == 14))
started = dbtestutil.datetimeNow(cursor)
self.connection.commit()
completed = started + dt.timedelta(microseconds=100)
idTimesAndSuccessSeq = [
[started,completed,True,1],
[started,completed,True,3],
[started,completed,True,5],
[started,completed,True,11],
[started,None,False,2],
[started,None,False,4],
[started,None,False,8],
[started,None,False,12],
]
dbCon,dbCur = m.getDatabaseConnectionPair()
try:
jobdata = self.setJobSuccess(dbCur,idTimesAndSuccessSeq)
m.cleanUpCompletedAndFailedJobs()
finally:
m.databaseConnectionPool.cleanup()
successSave = set()
failSave = set()
expectSuccessSave = set()
expectFailSave = set()
remainBehind = set()
for dir, dirs, files in os.walk(me.config.storageRoot):
remainBehind.update(os.path.splitext(x)[0] for x in files)
for d in idTimesAndSuccessSeq:
if d[2]:
expectSuccessSave.add(d[3])
else:
expectFailSave.add(d[3])
for dir,dirs,files in os.walk(me.config.saveSuccessfulMinidumpsTo):
successSave.update((os.path.splitext(x)[0] for x in files))
for dir,dirs,files in os.walk(me.config.saveFailedMinidumpsTo):
failSave.update((os.path.splitext(x)[0] for x in files))
for x in jobdata:
if None == x[2]:
assert not x[1] in failSave and not x[1] in successSave, "if we didn't set success state for %s, then it wasn't copied"%(x[1])
assert x[1] in remainBehind, "if we didn't set success state for %s, then it should remain behind"%(x[1])
assert not x[0] in expectFailSave and not x[0] in expectSuccessSave, "database should match expectatations for id=%s"%(x[0])
elif True == x[2]:
assert not x[1] in failSave and x[1] in successSave, "if we set success for %s, it is copied to %s"%(x[1],me.config.saveSussessfulMinidumpsTo)
assert not x[0] in expectFailSave and x[0] in expectSuccessSave, "database should match expectatations for id=%s"%(x[0])
assert not x[1] in remainBehind, "if we did set success state for %s, then it should not remain behind"%(x[1])
elif False == x[2]:
assert x[1] in failSave and not x[1] in successSave, "if we set failure for %s, it is copied to %s"%(x[1],me.config.saveFailedMinidumpsTo)
assert x[0] in expectFailSave and not x[0] in expectSuccessSave, "database should match expectatations for id=%s"%(x[0])
assert not x[1] in remainBehind, "if we did set success state for %s, then it should not remain behind"%(x[1])
def testRemoveBadUuidFromJsonDumpStorage(self):
"""
testRemoveBadUuidFromJsonDumpStorage(self):
This just wraps JsonDumpStorage. Assure we aren't futzing up the wrap (fail with non-exist uuid)
"""
global me
createJDS.createTestSet(createJDS.jsonFileData,jsonKwargs={'logger':me.logger},rootDir=me.config.storageRoot)
mon = monitor.Monitor(me.config)
badUuid = '0bad0bad-0bad-6666-9999-0bad20001025'
assert_raises(monitor.UuidNotFoundException,mon.removeUuidFromJsonDumpStorage,badUuid)
def testDestructiveDateWalk(self):
createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[0],self.testDir)
storage = JDS.JsonDumpStorage(self.testDir,**self.initKwargs[0])
uuids = createJDS.jsonFileData.keys()
seenids = []
for id in storage.destructiveDateWalk():
assert id in uuids, 'Expect that %s is among the uuids we stored' % uuid
seenids.append(id)
for id in uuids:
assert id in seenids, 'Expect that we found every uuid we stored (%s) from %s' % (id,seenids)
assert not os.listdir(storage.dateBranch), 'Expect that destructive walk will remove all date links, and their dirs'
def testRemoveAlsoNames(self):
"""testJsonDumpStorage:TestJsonDumpStorage.testRemoveAlsoNames(self)
Try to remove them all, and check that they are indeed all gone.
"""
createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[2],self.testDir)
kwargs = self.initKwargs[2]
kwargs['cleanIndexDirectories'] = 'True'
storage = JDS.JsonDumpStorage(self.testDir,**kwargs)
for uuid,data in createJDS.jsonFileData.items():
storage.remove(uuid)
assert not os.listdir(storage.root), 'Expected them all to go, but %s'%(os.listdir(storage.root))
def testRemoveAlsoNames(self):
"""testJsonDumpStorage:TestJsonDumpStorage.testRemoveAlsoNames(self)
Try to remove them all, and check that they are indeed all gone.
"""
createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[2],self.testDir)
kwargs = self.initKwargs[2]
kwargs['cleanIndexDirectories'] = 'True'
storage = JDS.JsonDumpStorage(self.testDir,**kwargs)
for uuid,data in createJDS.jsonFileData.items():
storage.remove(uuid)
assert not os.listdir(storage.root), 'Expected them all to go, but %s'%(os.listdir(storage.root))
def testGetDump(self):
createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[1],self.testDir)
storage = JDS.JsonDumpStorage(self.testDir,**self.initKwargs[1])
for uuid,data in createJDS.jsonFileData.items():
expected = os.sep.join((storage.nameBranch,data[2],uuid+storage.dumpSuffix))
got = storage.getDump(uuid)
assert expected == got, 'Expected dump file %s, got %s' % (expected,got)
try:
storage.getDump(createJDS.jsonBadUuid)
assert False, 'Should throw IOError from attempt to getDump(non-existent-uuid)'
except OSError,e:
assert True
def testRemoveOlderThan(self):
createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[0],self.testDir)
storage = JDS.JsonDumpStorage(self.testDir,**self.initKwargs[0])
cutoff = DT.datetime(2008,12,26,05,0)
youngkeys = [x for x,d in createJDS.jsonFileData.items() if DT.datetime(*[int(i) for i in d[0].split('-')]) >= cutoff]
oldkeys = [x for x,d in createJDS.jsonFileData.items() if DT.datetime(*[int(i) for i in d[0].split('-')]) < cutoff]
for k in youngkeys:
assert k in createJDS.jsonFileData.keys(),"Expected %s in %s"%(k,createJDS.jsonFileData.keys())
for k in oldkeys:
assert k in createJDS.jsonFileData.keys()
for k in createJDS.jsonFileData.keys():
assert (k in youngkeys or k in oldkeys)
storage.removeOlderThan(cutoff)
seenuuid = {}
seendirs = []
for dir,dirs,files in os.walk(storage.nameBranch):
for f in files:
if os.path.islink(os.path.join(dir,f)):
uuid = os.path.splitext(f)[0]
seenuuid[uuid] = True
assert uuid in youngkeys, 'File: Expect that each remaining link has a young uuid, got %s' % uuid
assert not uuid in oldkeys, 'File Expect no remaining link has old uuid, got %s' % uuid
for d in dirs:
if os.path.islink(os.path.join(dir,d)):
uuid = os.path.splitext(d)[0]
seenuuid[uuid] = True
assert uuid in youngkeys, 'Dir: Expect that each remaining link has a young uuid, got %s' % uuid
assert not uuid in oldkeys, 'Dir: Expect no remaining link has old uuid, got %s' % uuid
for id in oldkeys:
assert not id in seenuuid,'Expect that no old key is found, but %s' % id
for id in youngkeys:
assert id in seenuuid, 'Expect that every new key is found, but %s' % id
seenuuid = {}
seendirs = []
for dir, dirs, files in os.walk(storage.dateBranch):
for f in files:
uuid = os.path.splitext(f)[0]
seenuuid[uuid] = True
assert uuid in youngkeys, 'Expect that each remaining file has a young uuid, got %s' % uuid
assert not uuid in oldkeys, 'Expect no remaining file has old uuid, got %s' % uuid
for d in dirs:
uuid = os.path.splitext(d)[0]
if '-' in uuid:
seenuuid[uuid] = True
assert uuid in youngkeys, 'Expect that each remaining file has a young uuid, got %s' % uuid
assert not uuid in oldkeys, 'Expect no remaining file has old uuid, got %s' % uuid
for id in oldkeys:
assert not id in seenuuid,'Expect that no old key is found but %s' % id
for id in youngkeys:
assert id in seenuuid, 'Expect that every new key is found, but %s' % id
assert os.path.isdir(os.path.join(storage.dateBranch,createJDS.jsonFileData[id][3]))
def testRemoveWithBadlyFormattedDateLink(self):
createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[2],self.testDir)
storage = JDS.JsonDumpStorage(self.testDir,**self.initKwargs[2])
uuid = createJDS.jsonFileData.keys()[0]
head, json_unused = os.path.split(storage.getJson(uuid))
target = os.readlink(os.path.join(head,uuid))
idx = target.index('/date/')
target = "%s%s%s" %(target[:idx+6],target[idx+7:idx+10],target[idx+10:])
os.unlink(os.path.join(head,uuid))
os.symlink(target,os.path.join(head,uuid))
#print "LINK:%s"%(os.readlink(os.path.join(head,uuid)))
# assure that we don't throw for a badly formatted path
storage.remove(uuid)
def testRemoveWithBadlyFormattedDateLink(self):
createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[2],self.testDir)
storage = JDS.JsonDumpStorage(self.testDir,**self.initKwargs[2])
uuid = createJDS.jsonFileData.keys()[0]
head, json_unused = os.path.split(storage.getJson(uuid))
target = os.readlink(os.path.join(head,uuid))
idx = target.index('/date/')
target = "%s%s%s" %(target[:idx+6],target[idx+7:idx+10],target[idx+10:])
os.unlink(os.path.join(head,uuid))
os.symlink(target,os.path.join(head,uuid))
#print "LINK:%s"%(os.readlink(os.path.join(head,uuid)))
# assure that we don't throw for a badly formatted path
storage.remove(uuid)
def testDestructiveDateWalk(self):
createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[0],self.testDir)
storage = JDS.JsonDumpStorage(self.testDir,**self.initKwargs[0])
uuids = createJDS.jsonFileData.keys()
#uuidsSet = set(uuids)
seenids = set()
for id in storage.destructiveDateWalk():
assert id in uuids, 'Expect that %s is among the uuids we stored\n%s' % (id,uuids)
seenids.add(id)
for id in uuids:
assert id in seenids, 'Expect that we found every uuid we stored (%s) from %s' % (id,seenids)
daily = os.listdir(storage.root)
for d in daily:
assert not storage.dateName in os.listdir(os.path.join(storage.root,d)), 'Expected all date subdirs to be gone, but %s'%d
def testGetDump(self):
createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[1],self.testDir)
storage = JDS.JsonDumpStorage(self.testDir,**self.initKwargs[1])
for uuid,data in createJDS.jsonFileData.items():
dateparts = data[0].split('-')
daily = "%4d%02d%02d"%tuple([int(x) for x in dateparts[:3]])
expected = os.sep.join((storage.root,daily,storage.indexName,data[2],uuid+storage.dumpSuffix))
got = storage.getDump(uuid)
assert expected == got, 'Expected dump file %s, got %s' % (expected,got)
try:
storage.getDump(createJDS.jsonBadUuid)
assert False, 'Should throw IOError from attempt to getDumpAsFile(non-existent-uuid)'
except OSError,e:
assert True
def testRemoveGoodUuidFromJsonDumpStorage(self):
"""
testRemoveGoodUuidFromJsonDumpStorage(self):
This really just wraps JsonDumpStorage call. Assure we aren't futzing up the wrap (succeed with existing uuids)
"""
global me
createJDS.createTestSet(createJDS.jsonFileData,jsonKwargs={'logger':me.logger},rootDir=me.config.storageRoot)
createJDS.createTestSet(createJDS.jsonMoreData,jsonKwargs={'logger':me.logger},rootDir=me.config.deferredStorageRoot)
mon = monitor.Monitor(me.config)
goodUuid = '0b781b88-ecbe-4cc4-dead-6bbb20081225';
# this should work the first time...
mon.removeUuidFromJsonDumpStorage(goodUuid)
# ... and then fail the second time
assert_raises(monitor.UuidNotFoundException,mon.removeUuidFromJsonDumpStorage, goodUuid)
def testGetDump(self):
createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[1],self.testDir)
storage = JDS.JsonDumpStorage(self.testDir,**self.initKwargs[1])
for uuid,data in createJDS.jsonFileData.items():
dateparts = data[0].split('-')
daily = "%4d%02d%02d"%tuple([int(x) for x in dateparts[:3]])
expected = os.sep.join((storage.root,daily,storage.indexName,data[2],uuid+storage.dumpSuffix))
got = storage.getDump(uuid)
assert expected == got, 'Expected dump file %s, got %s' % (expected,got)
try:
storage.getDump(createJDS.jsonBadUuid)
assert False, 'Should throw IOError from attempt to getDumpAsFile(non-existent-uuid)'
except OSError,e:
assert True
def testDestructiveDateWalk(self):
createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[0],self.testDir)
storage = JDS.JsonDumpStorage(self.testDir,**self.initKwargs[0])
uuids = createJDS.jsonFileData.keys()
#uuidsSet = set(uuids)
seenids = set()
for id in storage.destructiveDateWalk():
assert id in uuids, 'Expect that %s is among the uuids we stored\n%s' % (id,uuids)
seenids.add(id)
for id in uuids:
assert id in seenids, 'Expect that we found every uuid we stored (%s) from %s' % (id,seenids)
daily = os.listdir(storage.root)
for d in daily:
assert not storage.dateName in os.listdir(os.path.join(storage.root,d)), 'Expected all date subdirs to be gone, but %s'%d
def markAsSeen(self):
createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[3],self.testDir)
storage = JDS.JsonDumpStorage(self.testDir,**self.initKwargs[3])
for uuid,data in createJDS.jsonFileData.items():
assert os.path.islink(os.sep.join((storage.dateBranch,data[3],uuid))), 'Expect a link from date to name for %s' % uuid
assert os.path.islink(os.sep.join((storage.nameBranch,data[2],uuid))), 'Expect link from name to timed for %s' % uuid
assert not os.path.islink(os.sep.join((storage.dateBranch,data[3],uuid))), 'Expect no link from date to name for %s' % uuid
assert not os.path.islink(os.sep.join((storage.nameBranch,data[2],uuid))), 'Expect no link from name to date for %s' % uuid
try:
storage.markAsSeen(createJDS.jsonBadUuid)
assert False, 'Expect to throw IOError from attempt to openAndMarkAsSeen(non-existent-uuid)'
except IOError:
assert True, 'Got expected error from attempt to openAndMarkAsSeen(non-existent-uuid)'
except Exception, e:
assert False, 'Got unexpected error %s from attempt to openAndMarkAsSeen(non-existent-uuid' % e
def testMarkAsSeen(self):
"""testNewJsonDumpStorage:TestJsonDumpStorage.testMarkAsSeen()
somewhat bogus test: Doesn't look for failure modes
"""
createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[0],rootDir=self.testDir)
storage = JDS.JsonDumpStorage(self.testDir,**self.initKwargs[2])
for ooid in createJDS.jsonFileData.keys():
namePath,parts = storage.namePath(ooid)
linkInName = os.path.join(namePath,ooid)
assert os.path.islink(linkInName), 'expected %s as link'%linkInName
dpath = os.path.join(namePath,os.readlink(linkInName))
linkInDate = os.path.join(dpath,ooid)
assert os.path.islink(linkInDate), 'expected %s as link'%linkInDate
storage.markAsSeen(ooid)
assert not os.path.exists(linkInName), 'expected %s gone'%linkInName
assert not os.path.exists(linkInDate), 'expected %s gone'%linkInDate
def testMarkAsSeen(self):
"""testNewJsonDumpStorage:TestJsonDumpStorage.testMarkAsSeen()
somewhat bogus test: Doesn't look for failure modes
"""
createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[0],rootDir=self.testDir)
storage = JDS.JsonDumpStorage(self.testDir,**self.initKwargs[2])
for ooid in createJDS.jsonFileData.keys():
namePath,parts = storage.namePath(ooid)
linkInName = os.path.join(namePath,ooid)
assert os.path.islink(linkInName), 'expected %s as link'%linkInName
dpath = os.path.join(namePath,os.readlink(linkInName))
linkInDate = os.path.join(dpath,ooid)
assert os.path.islink(linkInDate), 'expected %s as link'%linkInDate
storage.markAsSeen(ooid)
assert not os.path.exists(linkInName), 'expected %s gone'%linkInName
assert not os.path.exists(linkInDate), 'expected %s gone'%linkInDate
def testLookForPriorityJobsInJsonDumpStorage(self):
"""
testLookForPriorityJobsInJsonDumpStorage(self):
assure that we can find each uuid in standard and deferred storage
assure that we do not find any bogus uuid
assure that found uuids are added to jobs table with priority 1, and priority_jobs_NNN table for processor id NNN
"""
global me
m = monitor.Monitor(me.config)
createJDS.createTestSet(createJDS.jsonFileData,jsonKwargs={'logger':me.logger},rootDir=me.config.storageRoot)
createJDS.createTestSet(createJDS.jsonMoreData,jsonKwargs={'logger':me.logger},rootDir=me.config.deferredStorageRoot)
normUuids = createJDS.jsonFileData.keys()
defUuids = createJDS.jsonMoreData.keys()
allUuids = []
allUuids.extend(normUuids)
allUuids.extend(defUuids)
badUuid = '0bad0bad-0bad-6666-9999-0bad20001025'
dbCon,dbCur = m.getDatabaseConnectionPair()
try:
numProcessors = 5
dbtestutil.fillProcessorTable(self.connection.cursor(),numProcessors)
self.markLog()
m.lookForPriorityJobsInJsonDumpStorage(dbCur,allUuids)
assert [] == allUuids, 'Expect that all the uuids were found and removed from the looked for "set"'
m.lookForPriorityJobsInJsonDumpStorage(dbCur,(badUuid,))
self.markLog()
seg = self.extractLogSegment()
getIdAndPrioritySql = "SELECT owner,priority from jobs WHERE uuid = %s"
getCountSql = "SELECT count(*) from %s"
idCounts = dict( ( (x,0) for x in range(1,numProcessors+1) ) )
allUuids.extend(normUuids)
allUuids.extend(defUuids)
for uuid in allUuids:
dbCur.execute(getIdAndPrioritySql,(uuid,))
procid,pri = dbCur.fetchone()
assert 1 == pri, 'Expected priority of 1 for %s, but got %s'%(uuid,pri)
idCounts[procid] += 1
dbCur.execute(getIdAndPrioritySql,(badUuid,))
assert not dbCur.fetchone(), "Expect to get None entries in jobs table for badUuid"
for id,expectCount in idCounts.items():
dbCur.execute(getCountSql%('priority_jobs_%s'%id))
seenCount = dbCur.fetchone()[0]
assert expectCount == seenCount, 'Expected %s, got %s as count in priority_jobs_%s'%(expectCount,seenCount,id)
finally:
m.databaseConnectionPool.cleanup()
def testRemoveRemovesOnlyDate(self):
createJDS.createTestSet(createJDS.jsonFileData, self.initKwargs[2],
self.testDir)
storage = JDS.JsonDumpStorage(self.testDir, **self.initKwargs[2])
dailies = set([])
expectedSubs = []
alldirs = []
allfiles = []
alllinks = []
for uuid, data in createJDS.jsonFileData.items():
dailies.add(''.join(data[0].split('-')[:3]))
storage.remove(uuid)
for day in dailies:
for dir, dirs, files in os.walk(
os.sep.join((storage.root, day, storage.dateName))):
for file in files:
allfiles.append(file)
if os.path.islink(os.path.join(dir, file)):
alllinks.append(file)
for d in dirs:
if os.path.islink(os.path.join(dir, d)):
alllinks.append(d)
alldirs.append(os.path.join(dir, d))
assert [] == allfiles, 'Expect that all removed files are gone, but found %s' % allfiles
assert [] == alllinks, 'Expcet that all links are gone, but found %s' % alllinks
assert [] == alldirs, 'Expect that all date dirs are gone, but found %s' % alldirs
for day in dailies:
for dir, dirs, files in os.walk(
os.sep.join((storage.root, day, storage.indexName))):
for file in files:
allfiles.append(file)
if os.path.islink(os.path.join(dir, file)):
alllinks.append(file)
for d in dirs:
if os.path.islink(os.path.join(dir, d)):
alllinks.append(d)
alldirs.append(os.path.join(dir, d))
assert [] == allfiles, 'Expect that all removed files are gone, but found %s' % allfiles
assert [] == alllinks, 'Expect that all links are gone, but found %s' % alllinks
for sub in expectedSubs:
assert sub in alldirs, "Expect each subdirectory is still there, but didn't find %s" % sub
def testTransferOne(self):
createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[0],rootDir=self.testMoveFrom)
storage = JDS.JsonDumpStorage(self.testDir,**self.initKwargs[0])
oldStorage = JDS.JsonDumpStorage(self.testMoveFrom, **self.initKwargs[0])
itemNumber = 0
xmas = DT.datetime(2001,12,25,12,25)
for id in createJDS.jsonFileData.keys():
#case 0: copyLinks = True, makeNewDateLinks = False and there are links
#case 1: copyLinks = True, makeNewDateLinks = False and there are no links
#case 2: makeNewDateLinks = True and there were existing date links
#case 3: makeNewDateLinks = True and there were no existing date links
copyLinks = True
makeNewLinks = False
removeOldLink = False
newDate = None
if 0 == itemNumber % 4:
pass
elif 1 == itemNumber % 4:
removeOldLink = True
elif 2 == itemNumber % 4:
makeNewLinks = True
newDate = xmas
elif 3 == itemNumber % 4:
removeOldLink = True
makeNewLinks = True
newDate = xmas
itemNumber += 1
if(removeOldLink):
oldStorage.markAsSeen(id)
self.__hasNoLinkOrFail(oldStorage,id)
storage.transferOne(id,oldStorage,copyLinksBoolean=copyLinks,makeNewDateLinksBoolean=makeNewLinks,aDate=newDate)
try:
storage.getJson(id)
except Exception,x:
print '(%s): %s'%(type(x),x)
assert False, 'Expected to find a transferred json file for %s' % id
if makeNewLinks or not removeOldLink:
self.__hasLinkOrFail(storage,id)
if makeNewLinks:
self.__hasDatePathOrFail(storage,xmas)
if not makeNewLinks and removeOldLink:
self.__hasNoLinkOrFail(storage,id)
def testGetStorageFor(self):
"""
testGetStorageFor(self):
Test that the wrapper for JsonDumpStorage doesn't twist things incorrectly
"""
global me
self.markLog()
createJDS.createTestSet(createJDS.jsonFileData,jsonKwargs={'logger':me.logger},rootDir=me.config.storageRoot)
createJDS.createTestSet(createJDS.jsonMoreData,jsonKwargs={'logger':me.logger},rootDir=me.config.deferredStorageRoot)
mon = monitor.Monitor(me.config)
assert_raises(monitor.UuidNotFoundException,mon.getStorageFor,'nothing')
expected = me.config.storageRoot.rstrip(os.sep)
got = mon.getStorageFor('0bba929f-8721-460c-dead-a43c20071025').root
assert expected == got, 'Expected [%s] got [%s]'%(expected,got)
expected = me.config.deferredStorageRoot.rstrip(os.sep)
got = mon.getStorageFor('29adfb61-f75b-11dc-b6be-001320081225').root
assert expected == got, 'Expected [%s] got [%s]'%(expected,got)
self.markLog()
assert [] == self.extractLogSegment(), 'expected no logging for this test'
def testUuidInJsonDumpStorage(self):
"""
testUuidInJsonDumpStorage(self):
Test that the wrapper for JsonDumpStorage isn't all twisted up:
assure we find something in normal and deferred storage, and miss something that isn't there
do NOT test that the 'markAsSeen' actually works: That should be testJsonDumpStorage's job
"""
global me
m = monitor.Monitor(me.config)
createJDS.createTestSet(createJDS.jsonFileData,jsonKwargs={'logger':me.logger},rootDir=me.config.storageRoot)
createJDS.createTestSet(createJDS.jsonMoreData,jsonKwargs={'logger':me.logger},rootDir=me.config.deferredStorageRoot)
self.markLog()
badUuid = '0bad0bad-0bad-6666-9999-0bad20001025'
goodUuid = '0bba929f-8721-460c-dead-a43c20071025'
defUuid = '29adfb61-f75b-11dc-b6be-001320081225'
assert m.uuidInJsonDumpStorage(goodUuid), 'Dunno how that happened'
assert m.uuidInJsonDumpStorage(defUuid), 'Dunno how that happened'
assert not m.uuidInJsonDumpStorage(badUuid), 'Dunno how that happened'
self.markLog()
seg = self.extractLogSegment()
assert [] == seg, "Shouldn't log for success or failure"
def testRemoveAlsoNames(self):
createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[2],self.testDir)
kwargs = self.initKwargs[2]
kwargs['cleanIndexDirectories'] = 'True'
storage = JDS.JsonDumpStorage(self.testDir,**kwargs)
expectedSubs = []
fullSubs = []
for uuid in createJDS.jsonFileData.keys():
storage.remove(uuid)
exp = os.path.join(storage.nameBranch,createJDS.jsonFileData[uuid][2][:2])
if not exp in expectedSubs:
expectedSubs.append(exp)
fullSubs.append(os.path.join(storage.nameBranch,createJDS.jsonFileData[uuid][2]))
alldirs = []
for dir, dirs, files in os.walk(storage.nameBranch):
for d in dirs:
alldirs.append(os.path.join(dir,d))
for d in fullSubs:
assert not d in alldirs, 'Expected %s NOT in alldirs, but it was' % d
for d in expectedSubs:
assert d in alldirs, 'Expected %s in all dirs(%s) but it was not' % (d,alldirs)
def testTransferOne(self):
createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[0],rootDir=self.testMoveFrom)
storage = JDS.JsonDumpStorage(self.testDir,**self.initKwargs[0])
oldStorage = JDS.JsonDumpStorage(self.testMoveFrom, **self.initKwargs[0])
itemNumber = 0
xmas = datetime.datetime(2001,12,25,12,25, tzinfo=UTC)
for id in createJDS.jsonFileData.keys():
createLinks = 0 == itemNumber%2
removeOld = 0 == itemNumber%3
itemNumber += 1
storage.transferOne(id,oldStorage,createLinks=createLinks,removeOld=removeOld,aDate=xmas)
try:
storage.getJson(id)
except Exception,x:
print '(%s): %s'%(type(x),x)
assert False, 'Expected to find a transferred json file for %s' % id
if createLinks:
self.__hasLinkOrFail(storage,id)
self.__hasDatePathOrFail(storage,id,xmas)
if removeOld:
assert_raises(OSError,oldStorage.getJson,id)
def testRemoveRemovesOnlyDate(self):
createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[2],self.testDir)
storage = JDS.JsonDumpStorage(self.testDir,**self.initKwargs[2])
dailies = set([])
expectedSubs = []
alldirs = []
allfiles = []
alllinks = []
for uuid,data in createJDS.jsonFileData.items():
dailies.add(''.join(data[0].split('-')[:3]))
storage.remove(uuid)
for day in dailies:
for dir, dirs, files in os.walk(os.sep.join((storage.root,day,storage.dateName))):
for file in files:
allfiles.append(file)
if os.path.islink(os.path.join(dir,file)):
alllinks.append(file)
for d in dirs:
if os.path.islink(os.path.join(dir,d)):
alllinks.append(d)
alldirs.append(os.path.join(dir,d))
assert [] == allfiles, 'Expect that all removed files are gone, but found %s' % allfiles
assert [] == alllinks, 'Expcet that all links are gone, but found %s' % alllinks
assert [] == alldirs, 'Expect that all date dirs are gone, but found %s' % alldirs
for day in dailies:
for dir,dirs,files in os.walk(os.sep.join((storage.root,day,storage.indexName))):
for file in files:
allfiles.append(file)
if os.path.islink(os.path.join(dir,file)):
alllinks.append(file)
for d in dirs:
if os.path.islink(os.path.join(dir,d)):
alllinks.append(d)
alldirs.append(os.path.join(dir,d))
assert [] == allfiles, 'Expect that all removed files are gone, but found %s' % allfiles
assert [] == alllinks, 'Expect that all links are gone, but found %s' % alllinks
for sub in expectedSubs:
assert sub in alldirs, "Expect each subdirectory is still there, but didn't find %s" % sub
def testNewEntryDirectoryOverflow(self):
''' tests that we write new date links in appropriate overflow dir when we get too many in the regular dir'''
createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[3],self.testDir)
storage = JDS.JsonDumpStorage(self.testDir,maxDirectoryEntries=3,**self.initKwargs[3])
for uuid,data in createJDS.jsonTooMany.items():
abspathpart = data[3]
datetimedata = [int(x) for x in data[0].split('-')]
storage.newEntry(uuid,webheadHostName=data[1],timestamp = DT.datetime(*datetimedata))
datePathUpOne = os.path.join(storage.dateBranch,abspathpart)
webheads = os.listdir(datePathUpOne)
assert 3 == len(webheads)
for datePath in [os.path.join(datePathUpOne,x) for x in webheads]:
assert 3 >= len(os.listdir(datePath))
storage2 = JDS.JsonDumpStorage(self.testDir,maxDirectoryEntries=3, **self.initKwargs[3])
for uuid,data in createJDS.jsonMoreData.items():
abspathpart = data[3]
datetimedata = [int(x) for x in data[0].split('-')]
storage2.newEntry(uuid,webheadHostName=data[1],timestamp = DT.datetime(*datetimedata))
webheads = os.listdir(datePathUpOne)
assert 4 == len(webheads)
for datePath in [os.path.join(datePathUpOne,x) for x in webheads]:
assert 3 >= len(os.listdir(datePath))
def testRemoveRemovesOnlyDate(self):
createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[2],self.testDir)
storage = JDS.JsonDumpStorage(self.testDir,**self.initKwargs[2])
expectedSubs = []
for uuid in createJDS.jsonFileData.keys():
storage.remove(uuid)
expectedSubs.append(os.path.join(storage.nameBranch,createJDS.jsonFileData[uuid][2]))
allfiles = []
alllinks = []
alldirs = []
for dir, dirs, files in os.walk(storage.dateBranch):
for file in files:
allfiles.append(file)
if os.path.islink(os.path.join(dir,file)):
alllinks.append(file)
for d in dirs:
if os.path.islink(os.path.join(dir,d)):
alllinks.append(d)
alldirs.append(os.path.join(dir,d))
assert [] == allfiles, 'Expect that all removed files are gone, but found %s' % allfiles
assert [] == alllinks, 'Expcet that all links are gone, but found %s' % alllinks
assert [] == alldirs, 'Expect that all date dirs are gone, but found %s' % alldirs
for dir,dirs,files in os.walk(storage.nameBranch):
for file in files:
allfiles.append(file)
if os.path.islink(os.path.join(dir,file)):
alllinks.append(file)
for d in dirs:
if os.path.islink(os.path.join(dir,d)):
alllinks.append(d)
alldirs.append(os.path.join(dir,d))
assert [] == allfiles, 'Expect that all removed files are gone, but found %s' % allfiles
assert [] == alllinks, 'Expect that all links are gone, but found %s' % alllinks
for sub in expectedSubs:
assert sub in alldirs, "Expect each subdirectory is still there, but didn't find %s" % sub
def testCleanUpCompletedAndFailedJobs_WithoutSaves(self):
"""
testCleanUpCompletedAndFailedJobs_WithoutSaves(self):
First, dynamically set config to not save successful or failed jobs. They are still removed from the file system
"""
global me
cc = copy.copy(me.config)
cursor = self.connection.cursor()
dbtestutil.fillProcessorTable(cursor,4)
for conf in ['saveSuccessfulMinidumpsTo','saveFailedMinidumpsTo']:
cc[conf] = ''
m = monitor.Monitor(cc)
createJDS.createTestSet(createJDS.jsonFileData,jsonKwargs={'logger':me.logger},rootDir=me.config.storageRoot)
runInOtherProcess(m.standardJobAllocationLoop, stopCondition=(lambda : self.jobsAllocated() == 14))
started = dbtestutil.datetimeNow(cursor)
self.connection.commit()
completed = started + dt.timedelta(microseconds=100)
idTimesAndSuccessSeq = [
[started,completed,True,1],
[started,completed,True,3],
[started,completed,True,5],
[started,completed,True,11],
[started,None,False,2],
[started,None,False,4],
[started,None,False,8],
[started,None,False,12],
]
dbCon,dbCur = m.getDatabaseConnectionPair()
try:
jobdata = self.setJobSuccess(dbCur,idTimesAndSuccessSeq)
m.cleanUpCompletedAndFailedJobs()
finally:
m.databaseConnectionPool.cleanup()
successSave = set()
failSave = set()
expectSuccessSave = set()
expectFailSave = set()
for d in idTimesAndSuccessSeq:
if d[2]:
expectSuccessSave.add(d[3])
else:
expectFailSave.add(d[3])
for dir,dirs,files in os.walk(me.config.saveSuccessfulMinidumpsTo):
successSave.update((os.path.splitext(x)[0] for x in files))
for dir,dirs,files in os.walk(me.config.saveFailedMinidumpsTo):
failSave.update((os.path.splitext(x)[0] for x in files))
remainBehind = set()
for dir, dirs, files in os.walk(me.config.storageRoot):
remainBehind.update(os.path.splitext(x)[0] for x in files)
assert len(successSave) == 0, "We expect not to save any successful jobs with this setting"
assert len(failSave) == 0, "We expect not to save any failed jobs with this setting"
for x in jobdata:
if None == x[2]:
assert not x[0] in expectFailSave and not x[0] in expectSuccessSave, "database should match expectatations for id=%s"%(x[0])
assert x[1] in remainBehind, "if we didn't set success state for %s, then it should remain behind"%(x[1])
elif True == x[2]:
assert not x[1] in remainBehind, "if we did set success state for %s, then it should not remain behind"%(x[1])
assert not x[0] in expectFailSave and x[0] in expectSuccessSave, "database should match expectatations for id=%s"%(x[0])
elif False == x[2]:
assert not x[1] in remainBehind, "if we did set success state for %s, then it should not remain behind"%(x[1])
assert x[0] in expectFailSave and not x[0] in expectSuccessSave, "database should match expectatations for id=%s"%(x[0])
def testCreateTestSet():
testDir = "./TEST_CREATE_DIR"
try:
shutil.rmtree(testDir)
except:
pass
assert not os.path.exists(testDir)
try:
createJDS.createTestSet({},{},testDir)
assert os.path.isdir(testDir)
finally:
try:
shutil.rmtree(testDir)
except:
pass
expected = {
'%s/20071025/date/05'%testDir:(set(['04']), set([])),
'%s/20071025/date'%testDir:(set(['05']), set([])),
'%s/20071025/name/0b/ba/61/c5'%testDir:(set(['0bba61c5-dfc3-43e7-effe-8afd20071025']), set(['0bba61c5-dfc3-43e7-effe-8afd20071025.dump', '0bba61c5-dfc3-43e7-effe-8afd20071025.json'])),
'%s/20071025/name/0b'%testDir:(set(['ba']), set([])),
'%s/20071025/date/05/04'%testDir:(set(['webhead02_0']), set([])),
'%s/20071025/name/0b/ba/61'%testDir:(set(['c5']), set([])),
'%s/20071025'%testDir:(set(['date', 'name']), set([])),
'%s/20071025/date/05/04/webhead02_0'%testDir:(set(['0bba61c5-dfc3-43e7-effe-8afd20071025']), set([])),
'%s/20071025/name'%testDir:(set(['0b']), set([])),
'%s'%testDir:(set(['20071025']), set([])),
'%s/20071025/name/0b/ba'%testDir:(set(['61']), set([])),
}
minSet = {'0bba61c5-dfc3-43e7-effe-8afd20071025': ('2007-10-25-05-04','webhead02','0b/ba/61/c5','2007/10/25/05/00/webhead02_0')}
try:
createJDS.createTestSet(minSet,{},testDir)
got = {}
for dirpath, files, dirs in os.walk(testDir):
got[dirpath] = (set(files),set(dirs))
if expected != got:
print
for k, v in expected.items():
print ' X %s: %s'%(k,v)
if k in got:
if got[k] == expected[k]:
print ' G %s: %s'%(k,got[k])
else:
print 'xx G %s: %s'%(k,got[k])
else:
print 'xx G %s: (IS MISSING)'%(k)
for k,v in got.items():
if not k in expected:
print '++ G %s: %s'%(k,v)
assert expected == got
f = open(os.path.join(testDir,'20071025/name/0b/ba/61/c5/0bba61c5-dfc3-43e7-effe-8afd20071025.dump'))
data = f.readlines()
assert 1 == len(data)
assert 'dump test of 0bba61c5-dfc3-43e7-effe-8afd20071025' == data[0].strip()
f.close()
f = open(os.path.join(testDir,'20071025/name/0b/ba/61/c5/0bba61c5-dfc3-43e7-effe-8afd20071025.json'))
data = f.readlines()
assert 1 == len(data)
assert 'json test of 0bba61c5-dfc3-43e7-effe-8afd20071025' == data[0].strip()
f.close()
finally:
try:
shutil.rmtree(testDir)
except:
pass
try:
createJDS.createTestSet(minSet,{'jsonIsEmpty':True},testDir)
f = open(os.path.join(testDir,'20071025/name/0b/ba/61/c5/0bba61c5-dfc3-43e7-effe-8afd20071025.dump'))
data = f.readlines()
assert 1 == len(data)
assert 'dump test of 0bba61c5-dfc3-43e7-effe-8afd20071025' == data[0].strip()
f.close()
f = open(os.path.join(testDir,'20071025/name/0b/ba/61/c5/0bba61c5-dfc3-43e7-effe-8afd20071025.json'))
data = f.readlines()
assert 0 == len(data)
f.close()
finally:
try:
shutil.rmtree(testDir)
except:
pass
try:
createJDS.createTestSet(minSet,{'jsonIsBogus':False, 'jsonFileGenerator':'default'},testDir)
f = open(os.path.join(testDir,'20071025/name/0b/ba/61/c5/0bba61c5-dfc3-43e7-effe-8afd20071025.dump'))
data = f.readlines()
assert 1 == len(data)
assert 'dump test of 0bba61c5-dfc3-43e7-effe-8afd20071025' == data[0].strip()
f.close()
f = open(os.path.join(testDir,'20071025/name/0b/ba/61/c5//0bba61c5-dfc3-43e7-effe-8afd20071025.json'))
data = f.readlines()
assert 1 == len(data)
expect='{"BuildID": "bogusBuildID-00", "Version": "bogusVersion-00", "ProductName": "bogusName-00"}'
assert expect == data[0].strip()
f.close()
finally:
try:
shutil.rmtree(testDir)
except:
pass
try:
createJDS.createTestSet(minSet,{'jsonIsBogus':False},testDir)
f = open(os.path.join(testDir,'20071025/name/0b/ba/61/c5/0bba61c5-dfc3-43e7-effe-8afd20071025.dump'))
data = f.readlines()
assert 1 == len(data)
assert 'dump test of 0bba61c5-dfc3-43e7-effe-8afd20071025' == data[0].strip()
f.close()
f = open(os.path.join(testDir,'20071025/name/0b/ba/61/c5/0bba61c5-dfc3-43e7-effe-8afd20071025.json'))
data = f.readlines()
assert 1 == len(data)
expect='{"what": "legal json, bad contents", "uuid": "0bba61c5-dfc3-43e7-effe-8afd20071025"}'
assert expect == data[0].strip()
f.close()
finally:
try:
shutil.rmtree(testDir)
except:
pass
def testCreateTestSet():
testDir = "./TEST_CREATE_DIR"
try:
shutil.rmtree(testDir)
except:
pass
assert not os.path.exists(testDir)
try:
createJDS.createTestSet({},{},testDir)
assert os.path.isdir(testDir)
finally:
try:
shutil.rmtree(testDir)
except:
pass
expected = {
'%s/20071025/date/05'%testDir:(set(['04']), set([])),
'%s/20071025/date'%testDir:(set(['05']), set([])),
'%s/20071025/name/0b/ba/61/c5'%testDir:(set(['0bba61c5-dfc3-43e7-effe-8afd20071025']), set(['0bba61c5-dfc3-43e7-effe-8afd20071025.dump', '0bba61c5-dfc3-43e7-effe-8afd20071025.json'])),
'%s/20071025/name/0b'%testDir:(set(['ba']), set([])),
'%s/20071025/date/05/04'%testDir:(set(['webhead02_0']), set([])),
'%s/20071025/name/0b/ba/61'%testDir:(set(['c5']), set([])),
'%s/20071025'%testDir:(set(['date', 'name']), set([])),
'%s/20071025/date/05/04/webhead02_0'%testDir:(set(['0bba61c5-dfc3-43e7-effe-8afd20071025']), set([])),
'%s/20071025/name'%testDir:(set(['0b']), set([])),
'%s'%testDir:(set(['20071025']), set([])),
'%s/20071025/name/0b/ba'%testDir:(set(['61']), set([])),
}
minSet = {'0bba61c5-dfc3-43e7-effe-8afd20071025': ('2007-10-25-05-04','webhead02','0b/ba/61/c5','2007/10/25/05/00/webhead02_0')}
try:
createJDS.createTestSet(minSet,{},testDir)
got = {}
for dirpath, files, dirs in os.walk(testDir):
got[dirpath] = (set(files),set(dirs))
if expected != got:
print
for k, v in expected.items():
print ' X %s: %s'%(k,v)
if k in got:
if got[k] == expected[k]:
print ' G %s: %s'%(k,got[k])
else:
print 'xx G %s: %s'%(k,got[k])
else:
print 'xx G %s: (IS MISSING)'%(k)
for k,v in got.items():
if not k in expected:
print '++ G %s: %s'%(k,v)
assert expected == got
f = open(os.path.join(testDir,'20071025/name/0b/ba/61/c5/0bba61c5-dfc3-43e7-effe-8afd20071025.dump'))
data = f.readlines()
assert 1 == len(data)
assert 'dump test of 0bba61c5-dfc3-43e7-effe-8afd20071025' == data[0].strip()
f.close()
f = open(os.path.join(testDir,'20071025/name/0b/ba/61/c5/0bba61c5-dfc3-43e7-effe-8afd20071025.json'))
data = f.readlines()
assert 1 == len(data)
assert 'json test of 0bba61c5-dfc3-43e7-effe-8afd20071025' == data[0].strip()
f.close()
finally:
try:
shutil.rmtree(testDir)
except:
pass
try:
createJDS.createTestSet(minSet,{'jsonIsEmpty':True},testDir)
f = open(os.path.join(testDir,'20071025/name/0b/ba/61/c5/0bba61c5-dfc3-43e7-effe-8afd20071025.dump'))
data = f.readlines()
assert 1 == len(data)
assert 'dump test of 0bba61c5-dfc3-43e7-effe-8afd20071025' == data[0].strip()
f.close()
f = open(os.path.join(testDir,'20071025/name/0b/ba/61/c5/0bba61c5-dfc3-43e7-effe-8afd20071025.json'))
data = f.readlines()
assert 0 == len(data)
f.close()
finally:
try:
shutil.rmtree(testDir)
except:
pass
try:
createJDS.createTestSet(minSet,{'jsonIsBogus':False, 'jsonFileGenerator':'default'},testDir)
f = open(os.path.join(testDir,'20071025/name/0b/ba/61/c5/0bba61c5-dfc3-43e7-effe-8afd20071025.dump'))
data = f.readlines()
assert 1 == len(data)
assert 'dump test of 0bba61c5-dfc3-43e7-effe-8afd20071025' == data[0].strip()
f.close()
f = open(os.path.join(testDir,'20071025/name/0b/ba/61/c5//0bba61c5-dfc3-43e7-effe-8afd20071025.json'))
data = f.readlines()
assert 1 == len(data)
expect='{"BuildID": "bogusBuildID-00", "Version": "bogusVersion-00", "ProductName": "bogusName-00"}'
assert expect == data[0].strip()
f.close()
finally:
try:
shutil.rmtree(testDir)
except:
pass
try:
createJDS.createTestSet(minSet,{'jsonIsBogus':False},testDir)
f = open(os.path.join(testDir,'20071025/name/0b/ba/61/c5/0bba61c5-dfc3-43e7-effe-8afd20071025.dump'))
data = f.readlines()
assert 1 == len(data)
assert 'dump test of 0bba61c5-dfc3-43e7-effe-8afd20071025' == data[0].strip()
f.close()
f = open(os.path.join(testDir,'20071025/name/0b/ba/61/c5/0bba61c5-dfc3-43e7-effe-8afd20071025.json'))
data = f.readlines()
assert 1 == len(data)
expect='{"what": "legal json, bad contents", "uuid": "0bba61c5-dfc3-43e7-effe-8afd20071025"}'
assert expect == data[0].strip()
f.close()
finally:
try:
shutil.rmtree(testDir)
except:
pass
