def destructiveDateWalk (self): """ This function is a generator that yields all ooids found by walking the date branch of the file system. Just before yielding a value, it deletes both the links (from date to name and from name to date) After visiting all the ooids in a given date branch, recursively travels up, deleting any empty subdirectories Since the file system may be manipulated in a different thread, if no .json or .dump file is found, the links are left, and we do not yield that ooid """ def handleLink(dir,name): nameDir = self.namePath(name)[0] if not self.osModule.path.isfile(os.path.join(nameDir,name+self.jsonSuffix)): #print ' handleLink 1' return None if not self.osModule.path.isfile(os.path.join(nameDir,name+self.dumpSuffix)): #print ' handleLink 2' return None if self.osModule.path.islink(os.path.join(nameDir,name)): self.osModule.unlink(os.path.join(nameDir,name)) self.osModule.unlink(os.path.join(dir,name)) #print ' handleLink 3' return name #print ' handleLink off end' dailyParts = [] try: dailyParts = self.osModule.listdir(self.root) except OSError: # If root doesn't exist, quietly do nothing, eh? return for daily in dailyParts: #print 'daily: %s' % daily for dir,dirs,files in self.osModule.walk(os.sep.join((self.root,daily,self.dateName))): #print dir,dirs,files if os.path.split(dir)[0] == os.path.split(self.datePath(datetime.datetime.now())[0]): #print 'skipping dir %s' % dir #print 'because: %s == %s' % (os.path.split(dir)[0],os.path.split(self.datePath(datetime.datetime.now())[0])) continue # the links are all to (relative) directories, so we need not look at files for d in dirs: #print 'dir ', d if self.osModule.path.islink(os.path.join(dir,d)): r = handleLink(dir,d) #print ' r ', r if r: yield r # after finishing a given directory... socorro_fs.cleanEmptySubdirectories(os.path.join(self.root,daily),dir,self.osModule)
def destructiveDateWalk (self): """ This function is a generator that yields all ooids found by walking the date branch of the file system. Just before yielding a value, it deletes both the links (from date to name and from name to date) After visiting all the ooids in a given date branch, recursively travels up, deleting any empty subdirectories Since the file system may be manipulated in a different thread, if no .json or .dump file is found, the links are left, and we do not yield that ooid """ def handleLink(dir,name): nameDir = self.namePath(name)[0] if not self.osModule.path.isfile(os.path.join(nameDir,name+self.jsonSuffix)): #print ' handleLink 1' return None if not self.osModule.path.isfile(os.path.join(nameDir,name+self.dumpSuffix)): #print ' handleLink 2' return None if self.osModule.path.islink(os.path.join(nameDir,name)): self.osModule.unlink(os.path.join(nameDir,name)) self.osModule.unlink(os.path.join(dir,name)) #print ' handleLink 3' return name #print ' handleLink off end' dailyParts = [] try: dailyParts = self.osModule.listdir(self.root) except OSError: # If root doesn't exist, quietly do nothing, eh? return for daily in dailyParts: #print 'daily: %s' % daily for dir,dirs,files in self.osModule.walk(os.sep.join((self.root,daily,self.dateName))): #print dir,dirs,files if os.path.split(dir)[0] == os.path.split(self.datePath(utc_now())[0]): #print 'skipping dir %s' % dir #print 'because: %s == %s' % (os.path.split(dir)[0],os.path.split(self.datePath(utc_now())[0])) continue # the links are all to (relative) directories, so we need not look at files for d in dirs: #print 'dir ', d if self.osModule.path.islink(os.path.join(dir,d)): r = handleLink(dir,d) #print ' r ', r if r: yield r # after finishing a given directory... socorro_fs.cleanEmptySubdirectories(os.path.join(self.root,daily),dir,self.osModule)
def _remove(self, ooid, namePath, nameParts, datePath, dateParts): seenCount = 0 dailyPart = None if nameParts: dailyPart = nameParts[1] elif namePath: dailyPart = namePath.split(os.sep, 2)[1] elif dateParts: dailyPart = dateParts[1] elif datePath: dailyPart = datePath.split(os.sep, 2)[1] if not dailyPart: return stopper = os.path.join(self.root, dailyPart) # unlink on the name side first, thereby erasing any hope of removing relative paths from here... if namePath: try: self.osModule.unlink(os.path.join(namePath, ooid)) seenCount += 1 except: pass try: self.osModule.unlink(os.path.join(namePath, ooid + self.jsonSuffix)) seenCount += 1 except: pass try: self.osModule.unlink(os.path.join(namePath, ooid + self.dumpSuffix)) seenCount += 1 except: pass if self.cleanIndexDirectories: try: socorro_fs.cleanEmptySubdirectories( stopper, namePath, self.osModule ) # clean out name side if possible except OSError, x: pass
def _remove(self,ooid,namePath,nameParts,datePath,dateParts): seenCount = 0 dailyPart = None if nameParts: dailyPart = nameParts[1] elif namePath: dailyPart = namePath.split(os.sep,2)[1] elif dateParts: dailyPart = dateParts[1] elif datePath: dailyPart = datePath.split(os.sep,2)[1] if not dailyPart: return stopper = os.path.join(self.root,dailyPart) # unlink on the name side first, thereby erasing any hope of removing relative paths from here... if namePath: try: self.osModule.unlink(os.path.join(namePath,ooid)) seenCount += 1 except: pass try: self.osModule.unlink(os.path.join(namePath,ooid+self.jsonSuffix)) seenCount += 1 except: pass try: self.osModule.unlink(os.path.join(namePath,ooid+self.dumpSuffix)) seenCount += 1 except: pass if self.cleanIndexDirectories: try: socorro_fs.cleanEmptySubdirectories(stopper,namePath,self.osModule) #clean out name side if possible except OSError, x: pass
def testCleanEmptySubdirectories(self): f.makedirs('TestDir/A/B/C/D') f.makedirs('TestDir/AA/BB/C') f.makedirs('TestDir/AA/BB/CC/DD') fi = open('TestDir/A/a','w') fi.write('file a\n') fi.close() # Test short-circuit path, full stopper assert os.path.isdir('TestDir/A/B/C/D') f.cleanEmptySubdirectories('TestDir/A/B/C/D','TestDir/A/B/C/D') assert os.path.isdir('TestDir/A/B/C/D') # Test short-circuit path, name stopper f.cleanEmptySubdirectories('D','TestDir/A/B/C/D') assert os.path.isdir('TestDir/A/B/C/D') # Test some empties, name stopper f.cleanEmptySubdirectories('C','TestDir/A/B/C/D') assert not os.path.exists('TestDir/A/B/C/D') assert os.path.isdir('TestDir/A/B/C') # Test some empties, path stopper f.cleanEmptySubdirectories('TestDir/A/B','TestDir/A/B/C') assert not os.path.exists('TestDir/A/B/C') assert os.path.isdir('TestDir/A/B') #Test stopping on a file in a subdir f.cleanEmptySubdirectories('TestDir','TestDir/A/B') assert not os.path.exists('TestDir/A/B') assert os.path.isdir('TestDir/A') #Test stopping on another subdir f.cleanEmptySubdirectories('TestDir/AA','TestDir/AA/BB/CC/DD') assert not os.path.exists('TestDir/AA/BB/CC') assert os.path.isdir('TestDir/AA/BB') #Test for stopper not in path assert_raises(OSError,f.cleanEmptySubdirectories,'Woo','TestDir/AA/BB') #Test for non-existent leaf assert_raises(OSError,f.cleanEmptySubdirectories,'TestDir','TestDir/AA/BB/CC/DD')
def testCleanEmptySubdirectories(self): f.makedirs('TestDir/A/B/C/D') f.makedirs('TestDir/AA/BB/C') f.makedirs('TestDir/AA/BB/CC/DD') fi = open('TestDir/A/a', 'w') fi.write('file a\n') fi.close() # Test short-circuit path, full stopper assert os.path.isdir('TestDir/A/B/C/D') f.cleanEmptySubdirectories('TestDir/A/B/C/D', 'TestDir/A/B/C/D') assert os.path.isdir('TestDir/A/B/C/D') # Test short-circuit path, name stopper f.cleanEmptySubdirectories('D', 'TestDir/A/B/C/D') assert os.path.isdir('TestDir/A/B/C/D') # Test some empties, name stopper f.cleanEmptySubdirectories('C', 'TestDir/A/B/C/D') assert not os.path.exists('TestDir/A/B/C/D') assert os.path.isdir('TestDir/A/B/C') # Test some empties, path stopper f.cleanEmptySubdirectories('TestDir/A/B', 'TestDir/A/B/C') assert not os.path.exists('TestDir/A/B/C') assert os.path.isdir('TestDir/A/B') #Test stopping on a file in a subdir f.cleanEmptySubdirectories('TestDir', 'TestDir/A/B') assert not os.path.exists('TestDir/A/B') assert os.path.isdir('TestDir/A') #Test stopping on another subdir f.cleanEmptySubdirectories('TestDir/AA', 'TestDir/AA/BB/CC/DD') assert not os.path.exists('TestDir/AA/BB/CC') assert os.path.isdir('TestDir/AA/BB') #Test for stopper not in path assert_raises(OSError, f.cleanEmptySubdirectories, 'Woo', 'TestDir/AA/BB') #Test for non-existent leaf assert_raises(OSError, f.cleanEmptySubdirectories, 'TestDir', 'TestDir/AA/BB/CC/DD')
except: pass if self.cleanIndexDirectories: try: socorro_fs.cleanEmptySubdirectories(stopper,namePath,self.osModule) #clean out name side if possible except OSError, x: pass # and the date directory if datePath: try: self.osModule.unlink(os.path.join(datePath,ooid)) seenCount += 1 except: pass try: socorro_fs.cleanEmptySubdirectories(self.root,datePath,self.osModule) except: pass if not seenCount: self.logger.warning("%s - %s was totally unknown" % (threading.currentThread().getName(), ooid)) raise NoSuchUuidFound, "no trace of %s was found" % ooid #----------------------------------------------------------------------------------------------------------------- def quickDelete (self, ooid): """ deletes just the json and dump files without testing for the links. This is only to be used after destructiveDateWalk that will have already removed the symbolic links. """ namePath, nameParts = self.lookupNamePath(ooid) try: self.osModule.unlink(os.path.join(namePath,ooid+self.jsonSuffix))