Пример #1
0
 def _remove(self, ooid, namePath, nameParts, datePath, dateParts):
     seenCount = 0
     dailyPart = None
     if nameParts:
         dailyPart = nameParts[1]
     elif namePath:
         dailyPart = namePath.split(os.sep, 2)[1]
     elif dateParts:
         dailyPart = dateParts[1]
     elif datePath:
         dailyPart = datePath.split(os.sep, 2)[1]
     if not dailyPart:
         return
     stopper = os.path.join(self.root, dailyPart)
     # unlink on the name side first, thereby erasing any hope of removing
     # relative paths from here...
     if namePath:
         try:
             self.osModule.unlink(os.path.join(namePath, ooid))
             seenCount += 1
         except:
             pass
         try:
             self.osModule.unlink(
                 os.path.join(namePath, ooid + self.jsonSuffix))
             seenCount += 1
         except:
             pass
         try:
             self.osModule.unlink(
                 os.path.join(namePath, ooid + self.dumpSuffix))
             seenCount += 1
         except:
             pass
         if self.cleanIndexDirectories:
             try:
                 socorro_fs.cleanEmptySubdirectories(
                     stopper, namePath,
                     self.osModule)  # clean out name side if possible
             except OSError:
                 pass
     # and the date directory
     if datePath:
         try:
             self.osModule.unlink(os.path.join(datePath, ooid))
             seenCount += 1
         except:
             pass
         try:
             socorro_fs.cleanEmptySubdirectories(self.root, datePath,
                                                 self.osModule)
         except:
             pass
     if not seenCount:
         self.logger.warning("%s was totally unknown", ooid)
         raise NoSuchUuidFound("no trace of %s was found" % ooid)
Пример #2
0
    def destructiveDateWalk(self):
        """
        This function is a generator that yields all ooids found by walking the
        date branch of the file system.  Just before yielding a value, it
        deletes both the links (from date to name and from name to date)
        After visiting all the ooids in a given date branch, recursively
        travels up, deleting any empty subdirectories. Since the file system
        may be manipulated in a different thread, if no .json or .dump file
        is found, the links are left, and we do not yield that crash_id
        """
        def handleLink(dir, name):
            nameDir = self.namePath(name)[0]
            if not self.osModule.path.isfile(
                    os.path.join(nameDir, name + self.jsonSuffix)):
                #print '        handleLink 1'
                return None
            if not self.osModule.path.isfile(
                    os.path.join(nameDir, name + self.dumpSuffix)):
                #print '        handleLink 2'
                return None
            if self.osModule.path.islink(os.path.join(nameDir, name)):
                self.osModule.unlink(os.path.join(nameDir, name))
                self.osModule.unlink(os.path.join(dir, name))
                #print '        handleLink 3'
                return name
            #print '        handleLink off end'

        dailyParts = []
        try:
            dailyParts = self.osModule.listdir(self.root)
        except OSError:
            # If root doesn't exist, quietly do nothing, eh?
            return
        for daily in dailyParts:
            #print 'daily: %s' % daily
            for dir, dirs, files in self.osModule.walk(
                    os.sep.join((self.root, daily, self.dateName))):
                #print dir,dirs,files
                if (os.path.split(dir)[0] == os.path.split(
                        self.datePath(utc_now())[0])):
                    #print 'skipping dir %s' % dir
                    #print 'because: %s == %s' % (os.path.split(dir)[0],
                    #       os.path.split(self.datePath(utc_now())[0]))
                    continue
                # the links are all to (relative) directories, so we need not
                # look at files
                for d in dirs:
                    #print 'dir  ', d
                    if self.osModule.path.islink(os.path.join(dir, d)):
                        r = handleLink(dir, d)
                        #print '       r ', r
                        if r:
                            yield r
                # after finishing a given directory...
                socorro_fs.cleanEmptySubdirectories(
                    os.path.join(self.root, daily), dir, self.osModule)
Пример #3
0
 def _remove(self, crash_id, namePath, nameParts, datePath, dateParts):
     seenCount = 0
     dailyPart = None
     if nameParts:
         dailyPart = nameParts[1]
     elif namePath:
         dailyPart = namePath.split(os.sep, 2)[1]
     elif dateParts:
         dailyPart = dateParts[1]
     elif datePath:
         dailyPart = datePath.split(os.sep, 2)[1]
     if not dailyPart:
         return
     stopper = os.path.join(self.root, dailyPart)
     # unlink on the name side first, thereby erasing any hope of removing
     # relative paths from here...
     if namePath:
         #print "*****", namePath
         #raw_crash_path = self.getJson(crash_id)
         #with open(raw_crash_path) as crash_file:
         #raw_crash = json.load(crash_file)
         #dump_names = raw_crash.get('dump_names', [self.dump_field])
         #try:
         #self.osModule.unlink(os.path.join(namePath, crash_id))
         #seenCount += 1
         #except:
         #pass
         files_list = [
             x for x in os.listdir(namePath) if x.startswith(crash_id)
         ]
         for a_file_name in files_list:
             try:
                 self.osModule.unlink(os.path.join(namePath, a_file_name))
                 seenCount += 1
             except OSError:
                 self.logger.warning("%s wasn't found", a_file_name)
         try:
             self.osModule.unlink(
                 os.path.join(namePath, crash_id + self.jsonSuffix))
             seenCount += 1
         except OSError:
             pass
         if self.cleanIndexDirectories:
             try:
                 socorro_fs.cleanEmptySubdirectories(
                     stopper, namePath,
                     self.osModule)  # clean out name side if possible
             except OSError:
                 pass
     # and the date directory
     if datePath:
         try:
             self.osModule.unlink(os.path.join(datePath, crash_id))
             seenCount += 1
         except Exception:
             pass
         try:
             socorro_fs.cleanEmptySubdirectories(self.root, datePath,
                                                 self.osModule)
         except Exception:
             pass
     if not seenCount:
         self.logger.warning("%s was totally unknown", crash_id)
         raise NoSuchUuidFound("no trace of %s was found" % crash_id)
Пример #4
0
 def _remove(self, crash_id, namePath, nameParts, datePath, dateParts):
     seenCount = 0
     dailyPart = None
     if nameParts:
         dailyPart = nameParts[1]
     elif namePath:
         dailyPart = namePath.split(os.sep, 2)[1]
     elif dateParts:
         dailyPart = dateParts[1]
     elif datePath:
         dailyPart = datePath.split(os.sep, 2)[1]
     if not dailyPart:
         return
     stopper = os.path.join(self.root, dailyPart)
     # unlink on the name side first, thereby erasing any hope of removing
     # relative paths from here...
     if namePath:
         #print "*****", namePath
         #raw_crash_path = self.getJson(crash_id)
         #with open(raw_crash_path) as crash_file:
             #raw_crash = json.load(crash_file)
         #dump_names = raw_crash.get('dump_names', [self.dump_field])
         #try:
             #self.osModule.unlink(os.path.join(namePath, crash_id))
             #seenCount += 1
         #except:
             #pass
         files_list = [x for x in os.listdir(namePath)
                                  if x.startswith(crash_id)]
         for a_file_name in files_list:
             try:
                 self.osModule.unlink(os.path.join(namePath, a_file_name))
                 seenCount += 1
             except OSError:
                 self.logger.warning("%s wasn't found", a_file_name)
         try:
             self.osModule.unlink(
               os.path.join(namePath, crash_id + self.jsonSuffix)
             )
             seenCount += 1
         except OSError:
             pass
         if self.cleanIndexDirectories:
             try:
                 socorro_fs.cleanEmptySubdirectories(
                   stopper,
                   namePath,
                   self.osModule
                 )  # clean out name side if possible
             except OSError:
                 pass
     # and the date directory
     if datePath:
         try:
             self.osModule.unlink(os.path.join(datePath, crash_id))
             seenCount += 1
         except Exception:
             pass
         try:
             socorro_fs.cleanEmptySubdirectories(
               self.root,
               datePath,
               self.osModule
             )
         except Exception:
             pass
     if not seenCount:
         self.logger.warning("%s was totally unknown", crash_id)
         raise NoSuchUuidFound("no trace of %s was found" % crash_id)
Пример #5
0
    def destructiveDateWalk(self):
        """
        This function is a generator that yields all ooids found by walking the
        date branch of the file system.  Just before yielding a value, it
        deletes both the links (from date to name and from name to date)
        After visiting all the ooids in a given date branch, recursively
        travels up, deleting any empty subdirectories. Since the file system
        may be manipulated in a different thread, if no .json or .dump file
        is found, the links are left, and we do not yield that crash_id
        """

        def handleLink(dir, name):
            nameDir = self.namePath(name)[0]
            if not self.osModule.path.isfile(
              os.path.join(nameDir, name + self.jsonSuffix)
            ):
                #print '        handleLink 1'
                return None
            if not self.osModule.path.isfile(
              os.path.join(nameDir, name + self.dumpSuffix)
            ):
                #print '        handleLink 2'
                return None
            if self.osModule.path.islink(os.path.join(nameDir, name)):
                self.osModule.unlink(os.path.join(nameDir, name))
                self.osModule.unlink(os.path.join(dir, name))
                #print '        handleLink 3'
                return name
            #print '        handleLink off end'
        dailyParts = []
        try:
            dailyParts = self.osModule.listdir(self.root)
        except OSError:
            # If root doesn't exist, quietly do nothing, eh?
            return
        for daily in dailyParts:
            #print 'daily: %s' % daily
            for dir, dirs, files in self.osModule.walk(
              os.sep.join((self.root, daily, self.dateName))
            ):
                #print dir,dirs,files
                if (os.path.split(dir)[0] ==
                    os.path.split(self.datePath(utc_now())[0])):
                    #print 'skipping dir %s' % dir
                    #print 'because: %s == %s' % (os.path.split(dir)[0],
                    #       os.path.split(self.datePath(utc_now())[0]))
                    continue
                # the links are all to (relative) directories, so we need not
                # look at files
                for d in dirs:
                    #print 'dir  ', d
                    if self.osModule.path.islink(os.path.join(dir, d)):
                        r = handleLink(dir, d)
                        #print '       r ', r
                        if r:
                            yield r
                # after finishing a given directory...
                socorro_fs.cleanEmptySubdirectories(
                  os.path.join(self.root, daily),
                  dir,
                  self.osModule
                )
Пример #6
0
  def testCleanEmptySubdirectories(self):
    f.makedirs('TestDir/A/B/C/D')
    f.makedirs('TestDir/AA/BB/C')
    f.makedirs('TestDir/AA/BB/CC/DD')
    fi = open('TestDir/A/a','w')
    fi.write('file a\n')
    fi.close()
    # Test short-circuit path, full stopper
    assert os.path.isdir('TestDir/A/B/C/D')
    f.cleanEmptySubdirectories('TestDir/A/B/C/D','TestDir/A/B/C/D')
    assert os.path.isdir('TestDir/A/B/C/D')
    # Test short-circuit path, name stopper
    f.cleanEmptySubdirectories('D','TestDir/A/B/C/D')
    assert os.path.isdir('TestDir/A/B/C/D')

    # Test some empties, name stopper
    f.cleanEmptySubdirectories('C','TestDir/A/B/C/D')
    assert not os.path.exists('TestDir/A/B/C/D')
    assert os.path.isdir('TestDir/A/B/C')
    # Test some empties, path stopper
    f.cleanEmptySubdirectories('TestDir/A/B','TestDir/A/B/C')
    assert not os.path.exists('TestDir/A/B/C')
    assert os.path.isdir('TestDir/A/B')

    #Test stopping on a file in a subdir
    f.cleanEmptySubdirectories('TestDir','TestDir/A/B')
    assert not os.path.exists('TestDir/A/B')
    assert os.path.isdir('TestDir/A')

    #Test stopping on another subdir
    f.cleanEmptySubdirectories('TestDir/AA','TestDir/AA/BB/CC/DD')
    assert not os.path.exists('TestDir/AA/BB/CC')
    assert os.path.isdir('TestDir/AA/BB')

    #Test for stopper not in path
    assert_raises(OSError,f.cleanEmptySubdirectories,'Woo','TestDir/AA/BB')

    #Test for non-existent leaf
    assert_raises(OSError,f.cleanEmptySubdirectories,'TestDir','TestDir/AA/BB/CC/DD')
Пример #7
0
  def testCleanEmptySubdirectories(self):
    f.makedirs('TestDir/A/B/C/D')
    f.makedirs('TestDir/AA/BB/C')
    f.makedirs('TestDir/AA/BB/CC/DD')
    fi = open('TestDir/A/a','w')
    fi.write('file a\n')
    fi.close()
    # Test short-circuit path, full stopper
    assert os.path.isdir('TestDir/A/B/C/D')
    f.cleanEmptySubdirectories('TestDir/A/B/C/D','TestDir/A/B/C/D')
    assert os.path.isdir('TestDir/A/B/C/D')
    # Test short-circuit path, name stopper
    f.cleanEmptySubdirectories('D','TestDir/A/B/C/D')
    assert os.path.isdir('TestDir/A/B/C/D')

    # Test some empties, name stopper
    f.cleanEmptySubdirectories('C','TestDir/A/B/C/D')
    assert not os.path.exists('TestDir/A/B/C/D')
    assert os.path.isdir('TestDir/A/B/C')
    # Test some empties, path stopper
    f.cleanEmptySubdirectories('TestDir/A/B','TestDir/A/B/C')
    assert not os.path.exists('TestDir/A/B/C')
    assert os.path.isdir('TestDir/A/B')

    #Test stopping on a file in a subdir
    f.cleanEmptySubdirectories('TestDir','TestDir/A/B')
    assert not os.path.exists('TestDir/A/B')
    assert os.path.isdir('TestDir/A')

    #Test stopping on another subdir
    f.cleanEmptySubdirectories('TestDir/AA','TestDir/AA/BB/CC/DD')
    assert not os.path.exists('TestDir/AA/BB/CC')
    assert os.path.isdir('TestDir/AA/BB')

    #Test for stopper not in path
    assert_raises(OSError,f.cleanEmptySubdirectories,'Woo','TestDir/AA/BB')

    #Test for non-existent leaf
    assert_raises(OSError,f.cleanEmptySubdirectories,'TestDir','TestDir/AA/BB/CC/DD')
Пример #8
0
 def _remove(self, ooid, namePath, nameParts, datePath, dateParts):
     seenCount = 0
     dailyPart = None
     if nameParts:
         dailyPart = nameParts[1]
     elif namePath:
         dailyPart = namePath.split(os.sep, 2)[1]
     elif dateParts:
         dailyPart = dateParts[1]
     elif datePath:
         dailyPart = datePath.split(os.sep, 2)[1]
     if not dailyPart:
         return
     stopper = os.path.join(self.root, dailyPart)
     # unlink on the name side first, thereby erasing any hope of removing
     # relative paths from here...
     if namePath:
         try:
             self.osModule.unlink(os.path.join(namePath, ooid))
             seenCount += 1
         except:
             pass
         try:
             self.osModule.unlink(
               os.path.join(namePath, ooid + self.jsonSuffix)
             )
             seenCount += 1
         except:
             pass
         try:
             self.osModule.unlink(
               os.path.join(namePath, ooid + self.dumpSuffix)
             )
             seenCount += 1
         except:
             pass
         if self.cleanIndexDirectories:
             try:
                 socorro_fs.cleanEmptySubdirectories(
                   stopper,
                   namePath,
                   self.osModule
                 )  # clean out name side if possible
             except OSError:
                 pass
     # and the date directory
     if datePath:
         try:
             self.osModule.unlink(os.path.join(datePath, ooid))
             seenCount += 1
         except:
             pass
         try:
             socorro_fs.cleanEmptySubdirectories(
               self.root,
               datePath,
               self.osModule
             )
         except:
             pass
     if not seenCount:
         self.logger.warning("%s was totally unknown", ooid)
         raise NoSuchUuidFound("no trace of %s was found" % ooid)