def destructiveDateWalk (self):
"""
This function is a generator that yields all ooids found by walking the date branch of the file system.
Just before yielding a value, it deletes both the links (from date to name and from name to date)
After visiting all the ooids in a given date branch, recursively travels up, deleting any empty subdirectories
Since the file system may be manipulated in a different thread, if no .json or .dump file is found, the
links are left, and we do not yield that ooid
"""
def handleLink(dir,name):
nameDir = self.namePath(name)[0]
if not self.osModule.path.isfile(os.path.join(nameDir,name+self.jsonSuffix)):
#print ' handleLink 1'
return None
if not self.osModule.path.isfile(os.path.join(nameDir,name+self.dumpSuffix)):
#print ' handleLink 2'
return None
if self.osModule.path.islink(os.path.join(nameDir,name)):
self.osModule.unlink(os.path.join(nameDir,name))
self.osModule.unlink(os.path.join(dir,name))
#print ' handleLink 3'
return name
#print ' handleLink off end'
dailyParts = []
try:
dailyParts = self.osModule.listdir(self.root)
except OSError:
# If root doesn't exist, quietly do nothing, eh?
return
for daily in dailyParts:
#print 'daily: %s' % daily
for dir,dirs,files in self.osModule.walk(os.sep.join((self.root,daily,self.dateName))):
#print dir,dirs,files
if os.path.split(dir)[0] == os.path.split(self.datePath(datetime.datetime.now())[0]):
#print 'skipping dir %s' % dir
#print 'because: %s == %s' % (os.path.split(dir)[0],os.path.split(self.datePath(datetime.datetime.now())[0]))
continue
# the links are all to (relative) directories, so we need not look at files
for d in dirs:
#print 'dir ', d
if self.osModule.path.islink(os.path.join(dir,d)):
r = handleLink(dir,d)
#print ' r ', r
if r:
yield r
# after finishing a given directory...
socorro_fs.cleanEmptySubdirectories(os.path.join(self.root,daily),dir,self.osModule)
def destructiveDateWalk (self):
"""
This function is a generator that yields all ooids found by walking the date branch of the file system.
Just before yielding a value, it deletes both the links (from date to name and from name to date)
After visiting all the ooids in a given date branch, recursively travels up, deleting any empty subdirectories
Since the file system may be manipulated in a different thread, if no .json or .dump file is found, the
links are left, and we do not yield that ooid
"""
def handleLink(dir,name):
nameDir = self.namePath(name)[0]
if not self.osModule.path.isfile(os.path.join(nameDir,name+self.jsonSuffix)):
#print ' handleLink 1'
return None
if not self.osModule.path.isfile(os.path.join(nameDir,name+self.dumpSuffix)):
#print ' handleLink 2'
return None
if self.osModule.path.islink(os.path.join(nameDir,name)):
self.osModule.unlink(os.path.join(nameDir,name))
self.osModule.unlink(os.path.join(dir,name))
#print ' handleLink 3'
return name
#print ' handleLink off end'
dailyParts = []
try:
dailyParts = self.osModule.listdir(self.root)
except OSError:
# If root doesn't exist, quietly do nothing, eh?
return
for daily in dailyParts:
#print 'daily: %s' % daily
for dir,dirs,files in self.osModule.walk(os.sep.join((self.root,daily,self.dateName))):
#print dir,dirs,files
if os.path.split(dir)[0] == os.path.split(self.datePath(utc_now())[0]):
#print 'skipping dir %s' % dir
#print 'because: %s == %s' % (os.path.split(dir)[0],os.path.split(self.datePath(utc_now())[0]))
continue
# the links are all to (relative) directories, so we need not look at files
for d in dirs:
#print 'dir ', d
if self.osModule.path.islink(os.path.join(dir,d)):
r = handleLink(dir,d)
#print ' r ', r
if r:
yield r
# after finishing a given directory...
socorro_fs.cleanEmptySubdirectories(os.path.join(self.root,daily),dir,self.osModule)
def _remove(self, ooid, namePath, nameParts, datePath, dateParts):
seenCount = 0
dailyPart = None
if nameParts:
dailyPart = nameParts[1]
elif namePath:
dailyPart = namePath.split(os.sep, 2)[1]
elif dateParts:
dailyPart = dateParts[1]
elif datePath:
dailyPart = datePath.split(os.sep, 2)[1]
if not dailyPart:
return
stopper = os.path.join(self.root, dailyPart)
# unlink on the name side first, thereby erasing any hope of removing relative paths from here...
if namePath:
try:
self.osModule.unlink(os.path.join(namePath, ooid))
seenCount += 1
except:
pass
try:
self.osModule.unlink(os.path.join(namePath, ooid + self.jsonSuffix))
seenCount += 1
except:
pass
try:
self.osModule.unlink(os.path.join(namePath, ooid + self.dumpSuffix))
seenCount += 1
except:
pass
if self.cleanIndexDirectories:
try:
socorro_fs.cleanEmptySubdirectories(
stopper, namePath, self.osModule
) # clean out name side if possible
except OSError, x:
pass
def _remove(self,ooid,namePath,nameParts,datePath,dateParts):
seenCount = 0
dailyPart = None
if nameParts:
dailyPart = nameParts[1]
elif namePath:
dailyPart = namePath.split(os.sep,2)[1]
elif dateParts:
dailyPart = dateParts[1]
elif datePath:
dailyPart = datePath.split(os.sep,2)[1]
if not dailyPart:
return
stopper = os.path.join(self.root,dailyPart)
# unlink on the name side first, thereby erasing any hope of removing relative paths from here...
if namePath:
try:
self.osModule.unlink(os.path.join(namePath,ooid))
seenCount += 1
except:
pass
try:
self.osModule.unlink(os.path.join(namePath,ooid+self.jsonSuffix))
seenCount += 1
except:
pass
try:
self.osModule.unlink(os.path.join(namePath,ooid+self.dumpSuffix))
seenCount += 1
except:
pass
if self.cleanIndexDirectories:
try:
socorro_fs.cleanEmptySubdirectories(stopper,namePath,self.osModule) #clean out name side if possible
except OSError, x:
pass
def testCleanEmptySubdirectories(self):
f.makedirs('TestDir/A/B/C/D')
f.makedirs('TestDir/AA/BB/C')
f.makedirs('TestDir/AA/BB/CC/DD')
fi = open('TestDir/A/a','w')
fi.write('file a\n')
fi.close()
# Test short-circuit path, full stopper
assert os.path.isdir('TestDir/A/B/C/D')
f.cleanEmptySubdirectories('TestDir/A/B/C/D','TestDir/A/B/C/D')
assert os.path.isdir('TestDir/A/B/C/D')
# Test short-circuit path, name stopper
f.cleanEmptySubdirectories('D','TestDir/A/B/C/D')
assert os.path.isdir('TestDir/A/B/C/D')
# Test some empties, name stopper
f.cleanEmptySubdirectories('C','TestDir/A/B/C/D')
assert not os.path.exists('TestDir/A/B/C/D')
assert os.path.isdir('TestDir/A/B/C')
# Test some empties, path stopper
f.cleanEmptySubdirectories('TestDir/A/B','TestDir/A/B/C')
assert not os.path.exists('TestDir/A/B/C')
assert os.path.isdir('TestDir/A/B')
#Test stopping on a file in a subdir
f.cleanEmptySubdirectories('TestDir','TestDir/A/B')
assert not os.path.exists('TestDir/A/B')
assert os.path.isdir('TestDir/A')
#Test stopping on another subdir
f.cleanEmptySubdirectories('TestDir/AA','TestDir/AA/BB/CC/DD')
assert not os.path.exists('TestDir/AA/BB/CC')
assert os.path.isdir('TestDir/AA/BB')
#Test for stopper not in path
assert_raises(OSError,f.cleanEmptySubdirectories,'Woo','TestDir/AA/BB')
#Test for non-existent leaf
assert_raises(OSError,f.cleanEmptySubdirectories,'TestDir','TestDir/AA/BB/CC/DD')
def testCleanEmptySubdirectories(self):
f.makedirs('TestDir/A/B/C/D')
f.makedirs('TestDir/AA/BB/C')
f.makedirs('TestDir/AA/BB/CC/DD')
fi = open('TestDir/A/a', 'w')
fi.write('file a\n')
fi.close()
# Test short-circuit path, full stopper
assert os.path.isdir('TestDir/A/B/C/D')
f.cleanEmptySubdirectories('TestDir/A/B/C/D', 'TestDir/A/B/C/D')
assert os.path.isdir('TestDir/A/B/C/D')
# Test short-circuit path, name stopper
f.cleanEmptySubdirectories('D', 'TestDir/A/B/C/D')
assert os.path.isdir('TestDir/A/B/C/D')
# Test some empties, name stopper
f.cleanEmptySubdirectories('C', 'TestDir/A/B/C/D')
assert not os.path.exists('TestDir/A/B/C/D')
assert os.path.isdir('TestDir/A/B/C')
# Test some empties, path stopper
f.cleanEmptySubdirectories('TestDir/A/B', 'TestDir/A/B/C')
assert not os.path.exists('TestDir/A/B/C')
assert os.path.isdir('TestDir/A/B')
#Test stopping on a file in a subdir
f.cleanEmptySubdirectories('TestDir', 'TestDir/A/B')
assert not os.path.exists('TestDir/A/B')
assert os.path.isdir('TestDir/A')
#Test stopping on another subdir
f.cleanEmptySubdirectories('TestDir/AA', 'TestDir/AA/BB/CC/DD')
assert not os.path.exists('TestDir/AA/BB/CC')
assert os.path.isdir('TestDir/AA/BB')
#Test for stopper not in path
assert_raises(OSError, f.cleanEmptySubdirectories, 'Woo',
'TestDir/AA/BB')
#Test for non-existent leaf
assert_raises(OSError, f.cleanEmptySubdirectories, 'TestDir',
'TestDir/AA/BB/CC/DD')
except:
pass
if self.cleanIndexDirectories:
try:
socorro_fs.cleanEmptySubdirectories(stopper,namePath,self.osModule) #clean out name side if possible
except OSError, x:
pass
# and the date directory
if datePath:
try:
self.osModule.unlink(os.path.join(datePath,ooid))
seenCount += 1
except:
pass
try:
socorro_fs.cleanEmptySubdirectories(self.root,datePath,self.osModule)
except:
pass
if not seenCount:
self.logger.warning("%s - %s was totally unknown" % (threading.currentThread().getName(), ooid))
raise NoSuchUuidFound, "no trace of %s was found" % ooid
#-----------------------------------------------------------------------------------------------------------------
def quickDelete (self, ooid):
"""
deletes just the json and dump files without testing for the links. This is
only to be used after destructiveDateWalk that will have already removed the
symbolic links. """
namePath, nameParts = self.lookupNamePath(ooid)
try:
self.osModule.unlink(os.path.join(namePath,ooid+self.jsonSuffix))
except:
pass
if self.cleanIndexDirectories:
try:
socorro_fs.cleanEmptySubdirectories(stopper,namePath,self.osModule) #clean out name side if possible
except OSError, x:
pass
# and the date directory
if datePath:
try:
self.osModule.unlink(os.path.join(datePath,ooid))
seenCount += 1
except:
pass
try:
socorro_fs.cleanEmptySubdirectories(self.root,datePath,self.osModule)
except:
pass
if not seenCount:
self.logger.warning("%s - %s was totally unknown" % (threading.currentThread().getName(), ooid))
raise NoSuchUuidFound, "no trace of %s was found" % ooid
#-----------------------------------------------------------------------------------------------------------------
def quickDelete (self, ooid):
"""
deletes just the json and dump files without testing for the links. This is
only to be used after destructiveDateWalk that will have already removed the
symbolic links. """
namePath, nameParts = self.lookupNamePath(ooid)
try:
self.osModule.unlink(os.path.join(namePath,ooid+self.jsonSuffix))
