def _remove(self, ooid, namePath, nameParts, datePath, dateParts): seenCount = 0 dailyPart = None if nameParts: dailyPart = nameParts[1] elif namePath: dailyPart = namePath.split(os.sep, 2)[1] elif dateParts: dailyPart = dateParts[1] elif datePath: dailyPart = datePath.split(os.sep, 2)[1] if not dailyPart: return stopper = os.path.join(self.root, dailyPart) # unlink on the name side first, thereby erasing any hope of removing # relative paths from here... if namePath: try: self.osModule.unlink(os.path.join(namePath, ooid)) seenCount += 1 except: pass try: self.osModule.unlink( os.path.join(namePath, ooid + self.jsonSuffix)) seenCount += 1 except: pass try: self.osModule.unlink( os.path.join(namePath, ooid + self.dumpSuffix)) seenCount += 1 except: pass if self.cleanIndexDirectories: try: socorro_fs.cleanEmptySubdirectories( stopper, namePath, self.osModule) # clean out name side if possible except OSError: pass # and the date directory if datePath: try: self.osModule.unlink(os.path.join(datePath, ooid)) seenCount += 1 except: pass try: socorro_fs.cleanEmptySubdirectories(self.root, datePath, self.osModule) except: pass if not seenCount: self.logger.warning("%s was totally unknown", ooid) raise NoSuchUuidFound("no trace of %s was found" % ooid)
def destructiveDateWalk(self): """ This function is a generator that yields all ooids found by walking the date branch of the file system. Just before yielding a value, it deletes both the links (from date to name and from name to date) After visiting all the ooids in a given date branch, recursively travels up, deleting any empty subdirectories. Since the file system may be manipulated in a different thread, if no .json or .dump file is found, the links are left, and we do not yield that crash_id """ def handleLink(dir, name): nameDir = self.namePath(name)[0] if not self.osModule.path.isfile( os.path.join(nameDir, name + self.jsonSuffix)): #print ' handleLink 1' return None if not self.osModule.path.isfile( os.path.join(nameDir, name + self.dumpSuffix)): #print ' handleLink 2' return None if self.osModule.path.islink(os.path.join(nameDir, name)): self.osModule.unlink(os.path.join(nameDir, name)) self.osModule.unlink(os.path.join(dir, name)) #print ' handleLink 3' return name #print ' handleLink off end' dailyParts = [] try: dailyParts = self.osModule.listdir(self.root) except OSError: # If root doesn't exist, quietly do nothing, eh? return for daily in dailyParts: #print 'daily: %s' % daily for dir, dirs, files in self.osModule.walk( os.sep.join((self.root, daily, self.dateName))): #print dir,dirs,files if (os.path.split(dir)[0] == os.path.split( self.datePath(utc_now())[0])): #print 'skipping dir %s' % dir #print 'because: %s == %s' % (os.path.split(dir)[0], # os.path.split(self.datePath(utc_now())[0])) continue # the links are all to (relative) directories, so we need not # look at files for d in dirs: #print 'dir ', d if self.osModule.path.islink(os.path.join(dir, d)): r = handleLink(dir, d) #print ' r ', r if r: yield r # after finishing a given directory... socorro_fs.cleanEmptySubdirectories( os.path.join(self.root, daily), dir, self.osModule)
def _remove(self, crash_id, namePath, nameParts, datePath, dateParts): seenCount = 0 dailyPart = None if nameParts: dailyPart = nameParts[1] elif namePath: dailyPart = namePath.split(os.sep, 2)[1] elif dateParts: dailyPart = dateParts[1] elif datePath: dailyPart = datePath.split(os.sep, 2)[1] if not dailyPart: return stopper = os.path.join(self.root, dailyPart) # unlink on the name side first, thereby erasing any hope of removing # relative paths from here... if namePath: #print "*****", namePath #raw_crash_path = self.getJson(crash_id) #with open(raw_crash_path) as crash_file: #raw_crash = json.load(crash_file) #dump_names = raw_crash.get('dump_names', [self.dump_field]) #try: #self.osModule.unlink(os.path.join(namePath, crash_id)) #seenCount += 1 #except: #pass files_list = [ x for x in os.listdir(namePath) if x.startswith(crash_id) ] for a_file_name in files_list: try: self.osModule.unlink(os.path.join(namePath, a_file_name)) seenCount += 1 except OSError: self.logger.warning("%s wasn't found", a_file_name) try: self.osModule.unlink( os.path.join(namePath, crash_id + self.jsonSuffix)) seenCount += 1 except OSError: pass if self.cleanIndexDirectories: try: socorro_fs.cleanEmptySubdirectories( stopper, namePath, self.osModule) # clean out name side if possible except OSError: pass # and the date directory if datePath: try: self.osModule.unlink(os.path.join(datePath, crash_id)) seenCount += 1 except Exception: pass try: socorro_fs.cleanEmptySubdirectories(self.root, datePath, self.osModule) except Exception: pass if not seenCount: self.logger.warning("%s was totally unknown", crash_id) raise NoSuchUuidFound("no trace of %s was found" % crash_id)
def _remove(self, crash_id, namePath, nameParts, datePath, dateParts): seenCount = 0 dailyPart = None if nameParts: dailyPart = nameParts[1] elif namePath: dailyPart = namePath.split(os.sep, 2)[1] elif dateParts: dailyPart = dateParts[1] elif datePath: dailyPart = datePath.split(os.sep, 2)[1] if not dailyPart: return stopper = os.path.join(self.root, dailyPart) # unlink on the name side first, thereby erasing any hope of removing # relative paths from here... if namePath: #print "*****", namePath #raw_crash_path = self.getJson(crash_id) #with open(raw_crash_path) as crash_file: #raw_crash = json.load(crash_file) #dump_names = raw_crash.get('dump_names', [self.dump_field]) #try: #self.osModule.unlink(os.path.join(namePath, crash_id)) #seenCount += 1 #except: #pass files_list = [x for x in os.listdir(namePath) if x.startswith(crash_id)] for a_file_name in files_list: try: self.osModule.unlink(os.path.join(namePath, a_file_name)) seenCount += 1 except OSError: self.logger.warning("%s wasn't found", a_file_name) try: self.osModule.unlink( os.path.join(namePath, crash_id + self.jsonSuffix) ) seenCount += 1 except OSError: pass if self.cleanIndexDirectories: try: socorro_fs.cleanEmptySubdirectories( stopper, namePath, self.osModule ) # clean out name side if possible except OSError: pass # and the date directory if datePath: try: self.osModule.unlink(os.path.join(datePath, crash_id)) seenCount += 1 except Exception: pass try: socorro_fs.cleanEmptySubdirectories( self.root, datePath, self.osModule ) except Exception: pass if not seenCount: self.logger.warning("%s was totally unknown", crash_id) raise NoSuchUuidFound("no trace of %s was found" % crash_id)
def destructiveDateWalk(self): """ This function is a generator that yields all ooids found by walking the date branch of the file system. Just before yielding a value, it deletes both the links (from date to name and from name to date) After visiting all the ooids in a given date branch, recursively travels up, deleting any empty subdirectories. Since the file system may be manipulated in a different thread, if no .json or .dump file is found, the links are left, and we do not yield that crash_id """ def handleLink(dir, name): nameDir = self.namePath(name)[0] if not self.osModule.path.isfile( os.path.join(nameDir, name + self.jsonSuffix) ): #print ' handleLink 1' return None if not self.osModule.path.isfile( os.path.join(nameDir, name + self.dumpSuffix) ): #print ' handleLink 2' return None if self.osModule.path.islink(os.path.join(nameDir, name)): self.osModule.unlink(os.path.join(nameDir, name)) self.osModule.unlink(os.path.join(dir, name)) #print ' handleLink 3' return name #print ' handleLink off end' dailyParts = [] try: dailyParts = self.osModule.listdir(self.root) except OSError: # If root doesn't exist, quietly do nothing, eh? return for daily in dailyParts: #print 'daily: %s' % daily for dir, dirs, files in self.osModule.walk( os.sep.join((self.root, daily, self.dateName)) ): #print dir,dirs,files if (os.path.split(dir)[0] == os.path.split(self.datePath(utc_now())[0])): #print 'skipping dir %s' % dir #print 'because: %s == %s' % (os.path.split(dir)[0], # os.path.split(self.datePath(utc_now())[0])) continue # the links are all to (relative) directories, so we need not # look at files for d in dirs: #print 'dir ', d if self.osModule.path.islink(os.path.join(dir, d)): r = handleLink(dir, d) #print ' r ', r if r: yield r # after finishing a given directory... socorro_fs.cleanEmptySubdirectories( os.path.join(self.root, daily), dir, self.osModule )
def testCleanEmptySubdirectories(self): f.makedirs('TestDir/A/B/C/D') f.makedirs('TestDir/AA/BB/C') f.makedirs('TestDir/AA/BB/CC/DD') fi = open('TestDir/A/a','w') fi.write('file a\n') fi.close() # Test short-circuit path, full stopper assert os.path.isdir('TestDir/A/B/C/D') f.cleanEmptySubdirectories('TestDir/A/B/C/D','TestDir/A/B/C/D') assert os.path.isdir('TestDir/A/B/C/D') # Test short-circuit path, name stopper f.cleanEmptySubdirectories('D','TestDir/A/B/C/D') assert os.path.isdir('TestDir/A/B/C/D') # Test some empties, name stopper f.cleanEmptySubdirectories('C','TestDir/A/B/C/D') assert not os.path.exists('TestDir/A/B/C/D') assert os.path.isdir('TestDir/A/B/C') # Test some empties, path stopper f.cleanEmptySubdirectories('TestDir/A/B','TestDir/A/B/C') assert not os.path.exists('TestDir/A/B/C') assert os.path.isdir('TestDir/A/B') #Test stopping on a file in a subdir f.cleanEmptySubdirectories('TestDir','TestDir/A/B') assert not os.path.exists('TestDir/A/B') assert os.path.isdir('TestDir/A') #Test stopping on another subdir f.cleanEmptySubdirectories('TestDir/AA','TestDir/AA/BB/CC/DD') assert not os.path.exists('TestDir/AA/BB/CC') assert os.path.isdir('TestDir/AA/BB') #Test for stopper not in path assert_raises(OSError,f.cleanEmptySubdirectories,'Woo','TestDir/AA/BB') #Test for non-existent leaf assert_raises(OSError,f.cleanEmptySubdirectories,'TestDir','TestDir/AA/BB/CC/DD')
def _remove(self, ooid, namePath, nameParts, datePath, dateParts): seenCount = 0 dailyPart = None if nameParts: dailyPart = nameParts[1] elif namePath: dailyPart = namePath.split(os.sep, 2)[1] elif dateParts: dailyPart = dateParts[1] elif datePath: dailyPart = datePath.split(os.sep, 2)[1] if not dailyPart: return stopper = os.path.join(self.root, dailyPart) # unlink on the name side first, thereby erasing any hope of removing # relative paths from here... if namePath: try: self.osModule.unlink(os.path.join(namePath, ooid)) seenCount += 1 except: pass try: self.osModule.unlink( os.path.join(namePath, ooid + self.jsonSuffix) ) seenCount += 1 except: pass try: self.osModule.unlink( os.path.join(namePath, ooid + self.dumpSuffix) ) seenCount += 1 except: pass if self.cleanIndexDirectories: try: socorro_fs.cleanEmptySubdirectories( stopper, namePath, self.osModule ) # clean out name side if possible except OSError: pass # and the date directory if datePath: try: self.osModule.unlink(os.path.join(datePath, ooid)) seenCount += 1 except: pass try: socorro_fs.cleanEmptySubdirectories( self.root, datePath, self.osModule ) except: pass if not seenCount: self.logger.warning("%s was totally unknown", ooid) raise NoSuchUuidFound("no trace of %s was found" % ooid)