def testNewEntryPermissions(self): dirPermissions=0707 dumpPermissions=0500 sfl = SilentFakeLogger() j = JDS.JsonDumpStorage(root=self.testDir,dirPermissions=dirPermissions,dumpPermissions=dumpPermissions,logger=sfl) u = str(socorro_uuid.uuid1()) f1, f2 = j.newEntry(u) f1.close() f2.close() jpath = j.getJson(u) gotPermissions = stat.S_IMODE(os.stat(jpath)[0]) assert stat.S_IMODE(os.stat(jpath)[0]) == dumpPermissions, "%s: Expected %o, got %o" % (jpath, dumpPermissions, gotPermissions) dpath = j.getDump(u) gotPermissions = stat.S_IMODE(os.stat(dpath)[0]) assert stat.S_IMODE(os.stat(dpath)[0]) == dumpPermissions, "%s: Expected %o, got %o" % (dpath, dumpPermissions, gotPermissions) udir = os.path.split(dpath)[0] datePath = os.path.abspath(os.path.join(udir,os.readlink(os.path.splitext(dpath)[0]))) namePath = os.path.abspath(os.path.splitext(dpath)[0]) topPath = os.path.abspath(self.testDir) dailies = os.listdir(topPath) def assertPermVisitor(p): gotPerm = stat.S_IMODE(os.stat(p)[0]) assert dirPermissions == gotPerm, "%s: Expected %0o, got %0o"%(p,dirPermissions,gotPerm) for d in dailies: # visitPath quietly ignores a file as the leaf socorro_fs.visitPath(os.path.join(topPath,d),datePath,assertPermVisitor) socorro_fs.visitPath(os.path.join(topPath,d),namePath,assertPermVisitor)
def constructorAlt(self,*args,**kwargs): storage = JDS.JsonDumpStorage(self.testDir,**kwargs) assert storage.dateName == kwargs.get('dateName','date'),'From kwargs=%s'%kwargs assert storage.indexName == kwargs.get('indexName','name'),'From kwargs=%s'%kwargs assert storage.jsonSuffix == '.'+kwargs.get('jsonSuffix','json'),'We will always pass non-dot json suffix. From kwargs=%s'%kwargs assert storage.dumpSuffix == kwargs.get('dumpSuffix','.dump'),'We will always pass dot dump suffix. From kwargs=%s'%kwargs assert self.testDir.rstrip(os.sep) == storage.root,'From kwargs=%s'%kwargs
def markAsSeen(self): createJDS.createTestSet(createJDS.jsonFileData, self.initKwargs[3], self.testDir) storage = JDS.JsonDumpStorage(self.testDir, **self.initKwargs[3]) for uuid, data in createJDS.jsonFileData.items(): assert os.path.islink( os.sep.join( (storage.dateBranch, data[3], uuid))), 'Expect a link from date to name for %s' % uuid assert os.path.islink( os.sep.join( (storage.nameBranch, data[2], uuid))), 'Expect link from name to timed for %s' % uuid assert not os.path.islink( os.sep.join( (storage.dateBranch, data[3], uuid))), 'Expect no link from date to name for %s' % uuid assert not os.path.islink( os.sep.join( (storage.nameBranch, data[2], uuid))), 'Expect no link from name to date for %s' % uuid try: storage.markAsSeen(createJDS.jsonBadUuid) assert False, 'Expect to throw IOError from attempt to openAndMarkAsSeen(non-existent-uuid)' except IOError: assert True, 'Got expected error from attempt to openAndMarkAsSeen(non-existent-uuid)' except Exception, e: assert False, 'Got unexpected error %s from attempt to openAndMarkAsSeen(non-existent-uuid' % e
def testRemove(self): createJDS.createTestSet(createJDS.jsonFileData, self.initKwargs[2], self.testDir) storage = JDS.JsonDumpStorage(self.testDir, **self.initKwargs[2]) counter = 0 for uuid in createJDS.jsonFileData.keys(): if 0 == counter % 3: # test that we don't throw for missing links storage.markAsSeen(uuid) if 1 == counter % 3: # test that we don't throw for one missing file if 0 == counter % 2: os.unlink(storage.getDump(uuid)) else: os.unlink(storage.getJson(uuid)) if 2 == counter % 3: # test that we don't throw for both missing files, but with links os.unlink(storage.getJson(uuid)) os.unlink(storage.getDump(uuid)) storage.remove(uuid) counter += 1 allfiles = [] alllinks = [] for dir, dirs, files in os.walk(self.testDir): for file in files: allfiles.append(file) if os.path.islink(os.path.join(dir, file)): alllinks.append(file) for d in dirs: if os.path.islink(os.path.join(dir, d)): alllinks.append(d) assert [] == allfiles, 'Expect that all removed files are gone, but found %s' % allfiles assert [] == alllinks, 'Expect that all links are gone, but found %s' % alllinks assert_raises(JDS.NoSuchUuidFound, storage.remove, "bogusdatax3yymmdd")
def test_getDump_with_names(self): storage = JDS.JsonDumpStorage(self.testDir, **self.initKwargs[2]) input_crashes = self._create_multidump_data() for crash_id, raw_crash, dumps_dict in input_crashes: storage.new_entry(crash_id, raw_crash, dumps_dict) crash_id = '0bba61c5-dfc3-43e7-dead-8afd20081225' base_path, parts_unused = storage.lookupNamePath(crash_id) expected_dump_path = self._expected_dump_path(base_path, crash_id, None) actual_dump_path = storage.getDump(crash_id) eq_(expected_dump_path, actual_dump_path) expected_dump_path = self._expected_dump_path(base_path, crash_id, None) actual_dump_path = storage.getDump(crash_id, storage.dump_field) eq_(expected_dump_path, actual_dump_path) expected_dump_path = self._expected_dump_path(base_path, crash_id, 'aux_dump_0') actual_dump_path = storage.getDump(crash_id, 'aux_dump_0') eq_(expected_dump_path, actual_dump_path) expected_dump_path = self._expected_dump_path(base_path, crash_id, 'aux_dump_1') actual_dump_path = storage.getDump(crash_id, 'aux_dump_1') eq_(expected_dump_path, actual_dump_path) assert_raises(OSError, storage.getDump, crash_id, 'does_not_exist')
def testDestructiveDateWalkNotNow(self): createJDS.createTestSet(self.currenttimes,self.initKwargs[1],self.testDir) storage = JDS.JsonDumpStorage(self.testDir,**self.initKwargs[1]) seenids = [] for id in storage.destructiveDateWalk(): seenids.append(id) assert [] == seenids
def testRemoveAlsoNames(self): """testJsonDumpStorage:TestJsonDumpStorage.testRemoveAlsoNames(self) Try to remove them all, and check that they are indeed all gone. """ createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[2],self.testDir) kwargs = self.initKwargs[2] kwargs['cleanIndexDirectories'] = 'True' storage = JDS.JsonDumpStorage(self.testDir,**kwargs) for uuid,data in createJDS.jsonFileData.items(): storage.remove(uuid) assert not os.listdir(storage.root), 'Expected them all to go, but %s'%(os.listdir(storage.root))
def test_quickDelete(self): storage = JDS.JsonDumpStorage(self.testDir, **self.initKwargs[2]) input_crashes = self._create_multidump_data() for crash_id, raw_crash, dumps_dict in input_crashes: storage.new_entry(crash_id, raw_crash, dumps_dict) crash_id = '0bba61c5-dfc3-3333-dead-8afd20081225' storage.quickDelete(crash_id) assert_raises(OSError, storage.getDump, crash_id) assert_raises(OSError, storage.get_dumps, crash_id)
def testRemoveWithBadlyFormattedDateLink(self): createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[2],self.testDir) storage = JDS.JsonDumpStorage(self.testDir,**self.initKwargs[2]) uuid = createJDS.jsonFileData.keys()[0] head, json_unused = os.path.split(storage.getJson(uuid)) target = os.readlink(os.path.join(head,uuid)) idx = target.index('/date/') target = "%s%s%s" %(target[:idx+6],target[idx+7:idx+10],target[idx+10:]) os.unlink(os.path.join(head,uuid)) os.symlink(target,os.path.join(head,uuid)) #print "LINK:%s"%(os.readlink(os.path.join(head,uuid))) # assure that we don't throw for a badly formatted path storage.remove(uuid)
def testGetDump(self): createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[1],self.testDir) storage = JDS.JsonDumpStorage(self.testDir,**self.initKwargs[1]) for uuid,data in createJDS.jsonFileData.items(): dateparts = data[0].split('-') daily = "%4d%02d%02d"%tuple([int(x) for x in dateparts[:3]]) expected = os.sep.join((storage.root,daily,storage.indexName,data[2],uuid+storage.dumpSuffix)) got = storage.getDump(uuid) assert expected == got, 'Expected dump file %s, got %s' % (expected,got) try: storage.getDump(createJDS.jsonBadUuid) assert False, 'Should throw IOError from attempt to getDumpAsFile(non-existent-uuid)' except OSError,e: assert True
def testDestructiveDateWalk(self): createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[0],self.testDir) storage = JDS.JsonDumpStorage(self.testDir,**self.initKwargs[0]) uuids = createJDS.jsonFileData.keys() #uuidsSet = set(uuids) seenids = set() for id in storage.destructiveDateWalk(): assert id in uuids, 'Expect that %s is among the uuids we stored\n%s' % (id,uuids) seenids.add(id) for id in uuids: assert id in seenids, 'Expect that we found every uuid we stored (%s) from %s' % (id,seenids) daily = os.listdir(storage.root) for d in daily: assert not storage.dateName in os.listdir(os.path.join(storage.root,d)), 'Expected all date subdirs to be gone, but %s'%d
def testNewEntry(self): storage = JDS.JsonDumpStorage(self.testDir,**self.initKwargs[2]) for uuid,data in createJDS.jsonFileData.items(): datetimedata = [int(x) for x in data[0].split('-')] uuid = ''.join((uuid[:-7],'2',uuid[-6:])) stamp = datetime.datetime(*datetimedata, tzinfo=UTC) try: fj,fd = storage.newEntry(uuid,webheadHostName=data[1],timestamp = stamp) except IOError: assert False, 'Expect to succeed with newEntry(%s,...)' % uuid assert fj, 'Expect a non-null json file handle from newEntry(%s,...)' % uuid loc2 = data[2][0:5] # We are not honoring ooid depth expectFileBase = os.sep.join((storage.root, storage.dailyPart('',stamp), storage.indexName,loc2)) expectJson = os.path.join(expectFileBase,uuid+storage.jsonSuffix) assert expectJson == fj.name, 'For %s, expect %s, got %s' % (uuid,expectJson,fj.name) assert fd, 'Expect a non-null dump file handle from newEntry(%s,...)' % uuid expectDump = os.path.join(expectFileBase,uuid+storage.dumpSuffix) assert expectDump == fd.name, 'For %s, expect %s, got %s' % (uuid,expectDump,fj.name) loc3parts = self.__relativeDateParts(data[0],storage.minutesPerSlot) loc3parts.append(data[3][-len('webhead0x_x'):]) loc3 = os.sep.join(loc3parts) lbase = os.sep.join((storage.root, storage.dailyPart('',stamp), storage.dateName, loc3)) lpath = os.path.join(lbase,uuid) assert os.path.islink(lpath), 'Expect a link from timed to storage for %s' % uuid relNamePath = os.path.join(lbase,os.readlink(lpath)) assert os.path.isdir(relNamePath), 'Expected %s to be a Name directory'%(relNamePath) lpath = os.path.join(expectFileBase,uuid) assert os.path.islink(lpath), 'Expect link from name storage to timed for %s' % uuid relDatePath = os.path.join(expectFileBase,os.readlink(lpath)) assert os.path.isdir(relDatePath), 'Expected %s to be a Date directory'%(relDatePath) try: try: fj.write("testing\n") assert True, 'must be able to write to the json file for uuid %s' % uuid except: assert False, 'must not fail to write to the json file for uuid %s' % uuid finally: if fj: fj.close() try: try: fd.write("testing\n") assert True, 'must be able to write to the dump file for uuid %s' % uuid except: assert False, 'must not fail to write to the dump file for uuid %s' % uuid finally: if fd: fd.close()
def testMarkAsSeen(self): """testNewJsonDumpStorage:TestJsonDumpStorage.testMarkAsSeen() somewhat bogus test: Doesn't look for failure modes """ createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[0],rootDir=self.testDir) storage = JDS.JsonDumpStorage(self.testDir,**self.initKwargs[2]) for ooid in createJDS.jsonFileData.keys(): namePath,parts = storage.namePath(ooid) linkInName = os.path.join(namePath,ooid) assert os.path.islink(linkInName), 'expected %s as link'%linkInName dpath = os.path.join(namePath,os.readlink(linkInName)) linkInDate = os.path.join(dpath,ooid) assert os.path.islink(linkInDate), 'expected %s as link'%linkInDate storage.markAsSeen(ooid) assert not os.path.exists(linkInName), 'expected %s gone'%linkInName assert not os.path.exists(linkInDate), 'expected %s gone'%linkInDate
def testRemoveRemovesOnlyDate(self): createJDS.createTestSet(createJDS.jsonFileData, self.initKwargs[2], self.testDir) storage = JDS.JsonDumpStorage(self.testDir, **self.initKwargs[2]) dailies = set([]) expectedSubs = [] alldirs = [] allfiles = [] alllinks = [] for uuid, data in createJDS.jsonFileData.items(): dailies.add(''.join(data[0].split('-')[:3])) storage.remove(uuid) for day in dailies: for dir, dirs, files in os.walk( os.sep.join((storage.root, day, storage.dateName))): for file in files: allfiles.append(file) if os.path.islink(os.path.join(dir, file)): alllinks.append(file) for d in dirs: if os.path.islink(os.path.join(dir, d)): alllinks.append(d) alldirs.append(os.path.join(dir, d)) assert [] == allfiles, 'Expect that all removed files are gone, but found %s' % allfiles assert [] == alllinks, 'Expcet that all links are gone, but found %s' % alllinks assert [] == alldirs, 'Expect that all date dirs are gone, but found %s' % alldirs for day in dailies: for dir, dirs, files in os.walk( os.sep.join((storage.root, day, storage.indexName))): for file in files: allfiles.append(file) if os.path.islink(os.path.join(dir, file)): alllinks.append(file) for d in dirs: if os.path.islink(os.path.join(dir, d)): alllinks.append(d) alldirs.append(os.path.join(dir, d)) assert [] == allfiles, 'Expect that all removed files are gone, but found %s' % allfiles assert [] == alllinks, 'Expect that all links are gone, but found %s' % alllinks for sub in expectedSubs: assert sub in alldirs, "Expect each subdirectory is still there, but didn't find %s" % sub
def test_store_and_retrieve_multidumps(self): storage = JDS.JsonDumpStorage(self.testDir, **self.initKwargs[2]) input_crashes = self._create_multidump_data() for crash_id, raw_crash, dumps_dict in input_crashes: storage.new_entry(crash_id, raw_crash, dumps_dict) for crash_id, raw_crash, dumps_dict in input_crashes: base_path, parts_unused = storage.lookupNamePath(crash_id) pathname = "%s/%s%s" % (base_path, crash_id, storage.jsonSuffix) eq_(storage.getJson(crash_id), pathname) with open(pathname) as f: d = json.load(f) eq_(d, raw_crash) dump_paths = storage.get_dumps(crash_id) for a_dump_name, a_dump_contents in dumps_dict.iteritems(): dump_pathname = self._expected_dump_path( base_path, crash_id, a_dump_name) ok_(dump_pathname in dump_paths.values()) with open(dump_pathname) as f: read_contents = f.read() eq_(read_contents, a_dump_contents)
def setUp(self): self.testDir = os.path.join('.','TEST-JSONDUMP')+'/' self.testMoveTo = os.path.join('.','TEST-MOVETO') self.testMoveFrom = os.path.join('.','TEST-MOVEFROM') self.testMoveToAlt = os.path.join('.','TEST-MOVETO-ALT') fakeLogger = socorro.lib.util.SilentFakeLogger() self.initKwargs = { 0:{'logger': fakeLogger}, 1:{'logger': fakeLogger,'dateName':'by_date','indexName':'by_name','jsonSuffix':'JS','dumpSuffix':'.DS',}, 2:{'logger': fakeLogger,'jsonSuffix':'JS','dumpSuffix':'.DS',}, 3:{'logger': fakeLogger,'dateName':'by_date','indexName':'index',}, } self.currenttimes = { '0bba61c5-dfc3-43e7-dead-8afd20081225': ['+0','webhead02', '0b/ba/61/c5','webhead02_0'], '0bba929f-8721-460c-dead-a43c20081225': ['+0','webhead02', '0b/ba/92/9f','webhead02_0'], '0b9ff107-8672-4aac-dead-b2bd20081225': ['+0','webhead01', '0b/9f/f1/07','webhead01_0'], '22adfb61-f75b-11dc-dead-001320081225': ['+0','webhead01', '22/ad/fb/61','webhead01_0'], 'b965de73-ae90-a935-dead-03ae20081225': ['+0','webhead01', 'b9/65/de/73','webhead01_0'], '0b781b88-ecbe-4cc4-dead-6bbb20081225': ['+5','webhead01', '0b/78/1b/88','webhead01_0'], '0b8344d6-9021-4db9-dead-a15320081225': ['+5','webhead01', '0b/83/44/d6','webhead01_0'], '0b94199b-b90b-4683-dead-411420081225': ['+5','webhead01', '0b/94/19/9b','webhead01_0'], '0b9eedc3-9a79-4ce2-dead-155920081225': ['+5','webhead01', '0b/9e/ed/c3','webhead01_0'], '0b9fd6da-27e4-46aa-dead-3deb20081225': ['+10','webhead02','0b/9f/d6/da','webhead02_0'], '0ba32a30-2476-4724-dead-de1720081225': ['+10','webhead02','0b/a3/2a/30','webhead02_0'], '0bad640f-5825-4d42-dead-21b820081225': ['+10','webhead02','0b/ad/64/0f','webhead02_0'], '0bae7049-bbff-49f2-dead-7e9f20081225': ['+10','webhead02','0b/ae/70/49','webhead02_0'], '0baf1b4d-dad3-4d35-dead-b9dc20081225': ['+10','webhead02','0b/af/1b/4d','webhead02_0'], } try: shutil.rmtree(self.testDir) except OSError: pass # ok if there is no such test directory os.mkdir(self.testDir) storage = JDS.JsonDumpStorage(self.testDir,**self.initKwargs[2]) self.dump_field = storage.dump_field self.dumpSuffix = storage.dumpSuffix
} cookie = 0 while True: retMap = {} for k, v in dataMap.items(): retMap[k] = v % cookie yield json.dumps(retMap) cookie += 1 def createTestSet(testData, jsonKwargs, rootDir): try: os.makedirs(rootDir) except OSError, x: if errno.EEXIST != x.errno: raise storage = JDS.JsonDumpStorage(rootDir, **jsonKwargs) jsonIsEmpty = jsonKwargs.get('jsonIsEmpty', False) jsonIsBogus = jsonKwargs.get('jsonIsBogus', True) jsonFileGenerator = jsonKwargs.get('jsonFileGenerator', None) if 'default' == jsonFileGenerator: jsonFileGenerator = minimalJsonFileContents() thedt = utc_now() for uuid, data in testData.items(): if data[0].startswith('+'): if thedt.second >= 58: print "\nSleeping for %d seconds" % (61 - thedt.second) time.sleep(61 - thedt.second) thedt = utc_now() slot = { '+0': getSlot(storage.minutesPerSlot, thedt.minute), '+5': getSlot(storage.minutesPerSlot, thedt.minute + 5),