Beispiel #1
0
 def destructiveDateWalk (self):
   """
   This function is a generator that yields all ooids found by walking the date branch of the file system.
   Just before yielding a value, it deletes both the links (from date to name and from name to date)
   After visiting all the ooids in a given date branch, recursively travels up, deleting any empty subdirectories
   Since the file system may be manipulated in a different thread, if no .json or .dump file is found, the
   links are left, and we do not yield that ooid
   """
   def handleLink(dir,name):
     nameDir = self.namePath(name)[0]
     if not self.osModule.path.isfile(os.path.join(nameDir,name+self.jsonSuffix)):
       #print '        handleLink 1'
       return None
     if not self.osModule.path.isfile(os.path.join(nameDir,name+self.dumpSuffix)):
       #print '        handleLink 2'
       return None
     if self.osModule.path.islink(os.path.join(nameDir,name)):
       self.osModule.unlink(os.path.join(nameDir,name))
       self.osModule.unlink(os.path.join(dir,name))
       #print '        handleLink 3'
       return name
     #print '        handleLink off end'
   dailyParts = []
   try:
     dailyParts = self.osModule.listdir(self.root)
   except OSError:
     # If root doesn't exist, quietly do nothing, eh?
     return
   for daily in dailyParts:
     #print 'daily: %s' % daily
     for dir,dirs,files in self.osModule.walk(os.sep.join((self.root,daily,self.dateName))):
       #print dir,dirs,files
       if os.path.split(dir)[0] == os.path.split(self.datePath(datetime.datetime.now())[0]):
         #print 'skipping dir %s' % dir
         #print 'because: %s == %s' % (os.path.split(dir)[0],os.path.split(self.datePath(datetime.datetime.now())[0]))
         continue
       # the links are all to (relative) directories, so we need not look at files
       for d in dirs:
         #print 'dir  ', d
         if self.osModule.path.islink(os.path.join(dir,d)):
           r = handleLink(dir,d)
           #print '       r ', r
           if r:
             yield r
       # after finishing a given directory...
       socorro_fs.cleanEmptySubdirectories(os.path.join(self.root,daily),dir,self.osModule)
Beispiel #2
0
 def destructiveDateWalk (self):
   """
   This function is a generator that yields all ooids found by walking the date branch of the file system.
   Just before yielding a value, it deletes both the links (from date to name and from name to date)
   After visiting all the ooids in a given date branch, recursively travels up, deleting any empty subdirectories
   Since the file system may be manipulated in a different thread, if no .json or .dump file is found, the
   links are left, and we do not yield that ooid
   """
   def handleLink(dir,name):
     nameDir = self.namePath(name)[0]
     if not self.osModule.path.isfile(os.path.join(nameDir,name+self.jsonSuffix)):
       #print '        handleLink 1'
       return None
     if not self.osModule.path.isfile(os.path.join(nameDir,name+self.dumpSuffix)):
       #print '        handleLink 2'
       return None
     if self.osModule.path.islink(os.path.join(nameDir,name)):
       self.osModule.unlink(os.path.join(nameDir,name))
       self.osModule.unlink(os.path.join(dir,name))
       #print '        handleLink 3'
       return name
     #print '        handleLink off end'
   dailyParts = []
   try:
     dailyParts = self.osModule.listdir(self.root)
   except OSError:
     # If root doesn't exist, quietly do nothing, eh?
     return
   for daily in dailyParts:
     #print 'daily: %s' % daily
     for dir,dirs,files in self.osModule.walk(os.sep.join((self.root,daily,self.dateName))):
       #print dir,dirs,files
       if os.path.split(dir)[0] == os.path.split(self.datePath(utc_now())[0]):
         #print 'skipping dir %s' % dir
         #print 'because: %s == %s' % (os.path.split(dir)[0],os.path.split(self.datePath(utc_now())[0]))
         continue
       # the links are all to (relative) directories, so we need not look at files
       for d in dirs:
         #print 'dir  ', d
         if self.osModule.path.islink(os.path.join(dir,d)):
           r = handleLink(dir,d)
           #print '       r ', r
           if r:
             yield r
       # after finishing a given directory...
       socorro_fs.cleanEmptySubdirectories(os.path.join(self.root,daily),dir,self.osModule)
Beispiel #3
0
 def _remove(self, ooid, namePath, nameParts, datePath, dateParts):
     seenCount = 0
     dailyPart = None
     if nameParts:
         dailyPart = nameParts[1]
     elif namePath:
         dailyPart = namePath.split(os.sep, 2)[1]
     elif dateParts:
         dailyPart = dateParts[1]
     elif datePath:
         dailyPart = datePath.split(os.sep, 2)[1]
     if not dailyPart:
         return
     stopper = os.path.join(self.root, dailyPart)
     # unlink on the name side first, thereby erasing any hope of removing relative paths from here...
     if namePath:
         try:
             self.osModule.unlink(os.path.join(namePath, ooid))
             seenCount += 1
         except:
             pass
         try:
             self.osModule.unlink(os.path.join(namePath, ooid + self.jsonSuffix))
             seenCount += 1
         except:
             pass
         try:
             self.osModule.unlink(os.path.join(namePath, ooid + self.dumpSuffix))
             seenCount += 1
         except:
             pass
         if self.cleanIndexDirectories:
             try:
                 socorro_fs.cleanEmptySubdirectories(
                     stopper, namePath, self.osModule
                 )  # clean out name side if possible
             except OSError, x:
                 pass
Beispiel #4
0
 def _remove(self,ooid,namePath,nameParts,datePath,dateParts):
   seenCount = 0
   dailyPart = None
   if nameParts:
     dailyPart = nameParts[1]
   elif namePath:
     dailyPart = namePath.split(os.sep,2)[1]
   elif dateParts:
     dailyPart = dateParts[1]
   elif datePath:
     dailyPart = datePath.split(os.sep,2)[1]
   if not dailyPart:
     return
   stopper = os.path.join(self.root,dailyPart)
   # unlink on the name side first, thereby erasing any hope of removing relative paths from here...
   if namePath:
     try:
       self.osModule.unlink(os.path.join(namePath,ooid))
       seenCount += 1
     except:
       pass
     try:
       self.osModule.unlink(os.path.join(namePath,ooid+self.jsonSuffix))
       seenCount += 1
     except:
       pass
     try:
       self.osModule.unlink(os.path.join(namePath,ooid+self.dumpSuffix))
       seenCount += 1
     except:
       pass
     if self.cleanIndexDirectories:
       try:
         socorro_fs.cleanEmptySubdirectories(stopper,namePath,self.osModule) #clean out name side if possible
       except OSError, x:
         pass
Beispiel #5
0
  def testCleanEmptySubdirectories(self):
    f.makedirs('TestDir/A/B/C/D')
    f.makedirs('TestDir/AA/BB/C')
    f.makedirs('TestDir/AA/BB/CC/DD')
    fi = open('TestDir/A/a','w')
    fi.write('file a\n')
    fi.close()
    # Test short-circuit path, full stopper
    assert os.path.isdir('TestDir/A/B/C/D')
    f.cleanEmptySubdirectories('TestDir/A/B/C/D','TestDir/A/B/C/D')
    assert os.path.isdir('TestDir/A/B/C/D')
    # Test short-circuit path, name stopper
    f.cleanEmptySubdirectories('D','TestDir/A/B/C/D')
    assert os.path.isdir('TestDir/A/B/C/D')

    # Test some empties, name stopper
    f.cleanEmptySubdirectories('C','TestDir/A/B/C/D')
    assert not os.path.exists('TestDir/A/B/C/D')
    assert os.path.isdir('TestDir/A/B/C')
    # Test some empties, path stopper
    f.cleanEmptySubdirectories('TestDir/A/B','TestDir/A/B/C')
    assert not os.path.exists('TestDir/A/B/C')
    assert os.path.isdir('TestDir/A/B')

    #Test stopping on a file in a subdir
    f.cleanEmptySubdirectories('TestDir','TestDir/A/B')
    assert not os.path.exists('TestDir/A/B')
    assert os.path.isdir('TestDir/A')

    #Test stopping on another subdir
    f.cleanEmptySubdirectories('TestDir/AA','TestDir/AA/BB/CC/DD')
    assert not os.path.exists('TestDir/AA/BB/CC')
    assert os.path.isdir('TestDir/AA/BB')

    #Test for stopper not in path
    assert_raises(OSError,f.cleanEmptySubdirectories,'Woo','TestDir/AA/BB')

    #Test for non-existent leaf
    assert_raises(OSError,f.cleanEmptySubdirectories,'TestDir','TestDir/AA/BB/CC/DD')
Beispiel #6
0
    def testCleanEmptySubdirectories(self):
        f.makedirs('TestDir/A/B/C/D')
        f.makedirs('TestDir/AA/BB/C')
        f.makedirs('TestDir/AA/BB/CC/DD')
        fi = open('TestDir/A/a', 'w')
        fi.write('file a\n')
        fi.close()
        # Test short-circuit path, full stopper
        assert os.path.isdir('TestDir/A/B/C/D')
        f.cleanEmptySubdirectories('TestDir/A/B/C/D', 'TestDir/A/B/C/D')
        assert os.path.isdir('TestDir/A/B/C/D')
        # Test short-circuit path, name stopper
        f.cleanEmptySubdirectories('D', 'TestDir/A/B/C/D')
        assert os.path.isdir('TestDir/A/B/C/D')

        # Test some empties, name stopper
        f.cleanEmptySubdirectories('C', 'TestDir/A/B/C/D')
        assert not os.path.exists('TestDir/A/B/C/D')
        assert os.path.isdir('TestDir/A/B/C')
        # Test some empties, path stopper
        f.cleanEmptySubdirectories('TestDir/A/B', 'TestDir/A/B/C')
        assert not os.path.exists('TestDir/A/B/C')
        assert os.path.isdir('TestDir/A/B')

        #Test stopping on a file in a subdir
        f.cleanEmptySubdirectories('TestDir', 'TestDir/A/B')
        assert not os.path.exists('TestDir/A/B')
        assert os.path.isdir('TestDir/A')

        #Test stopping on another subdir
        f.cleanEmptySubdirectories('TestDir/AA', 'TestDir/AA/BB/CC/DD')
        assert not os.path.exists('TestDir/AA/BB/CC')
        assert os.path.isdir('TestDir/AA/BB')

        #Test for stopper not in path
        assert_raises(OSError, f.cleanEmptySubdirectories, 'Woo',
                      'TestDir/AA/BB')

        #Test for non-existent leaf
        assert_raises(OSError, f.cleanEmptySubdirectories, 'TestDir',
                      'TestDir/AA/BB/CC/DD')
Beispiel #7
0
      except:
        pass
      if self.cleanIndexDirectories:
        try:
          socorro_fs.cleanEmptySubdirectories(stopper,namePath,self.osModule) #clean out name side if possible
        except OSError, x:
          pass
    # and the date directory
    if datePath:
      try:
        self.osModule.unlink(os.path.join(datePath,ooid))
        seenCount += 1
      except:
        pass
      try:
        socorro_fs.cleanEmptySubdirectories(self.root,datePath,self.osModule)
      except:
        pass
    if not seenCount:
      self.logger.warning("%s - %s was totally unknown" % (threading.currentThread().getName(), ooid))
      raise NoSuchUuidFound, "no trace of %s was found" % ooid

  #-----------------------------------------------------------------------------------------------------------------
  def quickDelete (self, ooid):
    """
    deletes just the json and dump files without testing for the links.  This is
    only to be used after destructiveDateWalk that will have already removed the
    symbolic links. """
    namePath, nameParts = self.lookupNamePath(ooid)
    try:
      self.osModule.unlink(os.path.join(namePath,ooid+self.jsonSuffix))
Beispiel #8
0
      except:
        pass
      if self.cleanIndexDirectories:
        try:
          socorro_fs.cleanEmptySubdirectories(stopper,namePath,self.osModule) #clean out name side if possible
        except OSError, x:
          pass
    # and the date directory
    if datePath:
      try:
        self.osModule.unlink(os.path.join(datePath,ooid))
        seenCount += 1
      except:
        pass
      try:
        socorro_fs.cleanEmptySubdirectories(self.root,datePath,self.osModule)
      except:
        pass
    if not seenCount:
      self.logger.warning("%s - %s was totally unknown" % (threading.currentThread().getName(), ooid))
      raise NoSuchUuidFound, "no trace of %s was found" % ooid

  #-----------------------------------------------------------------------------------------------------------------
  def quickDelete (self, ooid):
    """
    deletes just the json and dump files without testing for the links.  This is
    only to be used after destructiveDateWalk that will have already removed the
    symbolic links. """
    namePath, nameParts = self.lookupNamePath(ooid)
    try:
      self.osModule.unlink(os.path.join(namePath,ooid+self.jsonSuffix))