def _remove(self, ooid, namePath, nameParts, datePath, dateParts):
seenCount = 0
dailyPart = None
if nameParts:
dailyPart = nameParts[1]
elif namePath:
dailyPart = namePath.split(os.sep, 2)[1]
elif dateParts:
dailyPart = dateParts[1]
elif datePath:
dailyPart = datePath.split(os.sep, 2)[1]
if not dailyPart:
return
stopper = os.path.join(self.root, dailyPart)
# unlink on the name side first, thereby erasing any hope of removing
# relative paths from here...
if namePath:
try:
self.osModule.unlink(os.path.join(namePath, ooid))
seenCount += 1
except:
pass
try:
self.osModule.unlink(
os.path.join(namePath, ooid + self.jsonSuffix))
seenCount += 1
except:
pass
try:
self.osModule.unlink(
os.path.join(namePath, ooid + self.dumpSuffix))
seenCount += 1
except:
pass
if self.cleanIndexDirectories:
try:
socorro_fs.cleanEmptySubdirectories(
stopper, namePath,
self.osModule) # clean out name side if possible
except OSError:
pass
# and the date directory
if datePath:
try:
self.osModule.unlink(os.path.join(datePath, ooid))
seenCount += 1
except:
pass
try:
socorro_fs.cleanEmptySubdirectories(self.root, datePath,
self.osModule)
except:
pass
if not seenCount:
self.logger.warning("%s was totally unknown", ooid)
raise NoSuchUuidFound("no trace of %s was found" % ooid)
def destructiveDateWalk(self):
"""
This function is a generator that yields all ooids found by walking the
date branch of the file system. Just before yielding a value, it
deletes both the links (from date to name and from name to date)
After visiting all the ooids in a given date branch, recursively
travels up, deleting any empty subdirectories. Since the file system
may be manipulated in a different thread, if no .json or .dump file
is found, the links are left, and we do not yield that crash_id
"""
def handleLink(dir, name):
nameDir = self.namePath(name)[0]
if not self.osModule.path.isfile(
os.path.join(nameDir, name + self.jsonSuffix)):
#print ' handleLink 1'
return None
if not self.osModule.path.isfile(
os.path.join(nameDir, name + self.dumpSuffix)):
#print ' handleLink 2'
return None
if self.osModule.path.islink(os.path.join(nameDir, name)):
self.osModule.unlink(os.path.join(nameDir, name))
self.osModule.unlink(os.path.join(dir, name))
#print ' handleLink 3'
return name
#print ' handleLink off end'
dailyParts = []
try:
dailyParts = self.osModule.listdir(self.root)
except OSError:
# If root doesn't exist, quietly do nothing, eh?
return
for daily in dailyParts:
#print 'daily: %s' % daily
for dir, dirs, files in self.osModule.walk(
os.sep.join((self.root, daily, self.dateName))):
#print dir,dirs,files
if (os.path.split(dir)[0] == os.path.split(
self.datePath(utc_now())[0])):
#print 'skipping dir %s' % dir
#print 'because: %s == %s' % (os.path.split(dir)[0],
# os.path.split(self.datePath(utc_now())[0]))
continue
# the links are all to (relative) directories, so we need not
# look at files
for d in dirs:
#print 'dir ', d
if self.osModule.path.islink(os.path.join(dir, d)):
r = handleLink(dir, d)
#print ' r ', r
if r:
yield r
# after finishing a given directory...
socorro_fs.cleanEmptySubdirectories(
os.path.join(self.root, daily), dir, self.osModule)
def _remove(self, crash_id, namePath, nameParts, datePath, dateParts):
seenCount = 0
dailyPart = None
if nameParts:
dailyPart = nameParts[1]
elif namePath:
dailyPart = namePath.split(os.sep, 2)[1]
elif dateParts:
dailyPart = dateParts[1]
elif datePath:
dailyPart = datePath.split(os.sep, 2)[1]
if not dailyPart:
return
stopper = os.path.join(self.root, dailyPart)
# unlink on the name side first, thereby erasing any hope of removing
# relative paths from here...
if namePath:
#print "*****", namePath
#raw_crash_path = self.getJson(crash_id)
#with open(raw_crash_path) as crash_file:
#raw_crash = json.load(crash_file)
#dump_names = raw_crash.get('dump_names', [self.dump_field])
#try:
#self.osModule.unlink(os.path.join(namePath, crash_id))
#seenCount += 1
#except:
#pass
files_list = [
x for x in os.listdir(namePath) if x.startswith(crash_id)
]
for a_file_name in files_list:
try:
self.osModule.unlink(os.path.join(namePath, a_file_name))
seenCount += 1
except OSError:
self.logger.warning("%s wasn't found", a_file_name)
try:
self.osModule.unlink(
os.path.join(namePath, crash_id + self.jsonSuffix))
seenCount += 1
except OSError:
pass
if self.cleanIndexDirectories:
try:
socorro_fs.cleanEmptySubdirectories(
stopper, namePath,
self.osModule) # clean out name side if possible
except OSError:
pass
# and the date directory
if datePath:
try:
self.osModule.unlink(os.path.join(datePath, crash_id))
seenCount += 1
except Exception:
pass
try:
socorro_fs.cleanEmptySubdirectories(self.root, datePath,
self.osModule)
except Exception:
pass
if not seenCount:
self.logger.warning("%s was totally unknown", crash_id)
raise NoSuchUuidFound("no trace of %s was found" % crash_id)
def _remove(self, crash_id, namePath, nameParts, datePath, dateParts):
seenCount = 0
dailyPart = None
if nameParts:
dailyPart = nameParts[1]
elif namePath:
dailyPart = namePath.split(os.sep, 2)[1]
elif dateParts:
dailyPart = dateParts[1]
elif datePath:
dailyPart = datePath.split(os.sep, 2)[1]
if not dailyPart:
return
stopper = os.path.join(self.root, dailyPart)
# unlink on the name side first, thereby erasing any hope of removing
# relative paths from here...
if namePath:
#print "*****", namePath
#raw_crash_path = self.getJson(crash_id)
#with open(raw_crash_path) as crash_file:
#raw_crash = json.load(crash_file)
#dump_names = raw_crash.get('dump_names', [self.dump_field])
#try:
#self.osModule.unlink(os.path.join(namePath, crash_id))
#seenCount += 1
#except:
#pass
files_list = [x for x in os.listdir(namePath)
if x.startswith(crash_id)]
for a_file_name in files_list:
try:
self.osModule.unlink(os.path.join(namePath, a_file_name))
seenCount += 1
except OSError:
self.logger.warning("%s wasn't found", a_file_name)
try:
self.osModule.unlink(
os.path.join(namePath, crash_id + self.jsonSuffix)
)
seenCount += 1
except OSError:
pass
if self.cleanIndexDirectories:
try:
socorro_fs.cleanEmptySubdirectories(
stopper,
namePath,
self.osModule
) # clean out name side if possible
except OSError:
pass
# and the date directory
if datePath:
try:
self.osModule.unlink(os.path.join(datePath, crash_id))
seenCount += 1
except Exception:
pass
try:
socorro_fs.cleanEmptySubdirectories(
self.root,
datePath,
self.osModule
)
except Exception:
pass
if not seenCount:
self.logger.warning("%s was totally unknown", crash_id)
raise NoSuchUuidFound("no trace of %s was found" % crash_id)
def destructiveDateWalk(self):
"""
This function is a generator that yields all ooids found by walking the
date branch of the file system. Just before yielding a value, it
deletes both the links (from date to name and from name to date)
After visiting all the ooids in a given date branch, recursively
travels up, deleting any empty subdirectories. Since the file system
may be manipulated in a different thread, if no .json or .dump file
is found, the links are left, and we do not yield that crash_id
"""
def handleLink(dir, name):
nameDir = self.namePath(name)[0]
if not self.osModule.path.isfile(
os.path.join(nameDir, name + self.jsonSuffix)
):
#print ' handleLink 1'
return None
if not self.osModule.path.isfile(
os.path.join(nameDir, name + self.dumpSuffix)
):
#print ' handleLink 2'
return None
if self.osModule.path.islink(os.path.join(nameDir, name)):
self.osModule.unlink(os.path.join(nameDir, name))
self.osModule.unlink(os.path.join(dir, name))
#print ' handleLink 3'
return name
#print ' handleLink off end'
dailyParts = []
try:
dailyParts = self.osModule.listdir(self.root)
except OSError:
# If root doesn't exist, quietly do nothing, eh?
return
for daily in dailyParts:
#print 'daily: %s' % daily
for dir, dirs, files in self.osModule.walk(
os.sep.join((self.root, daily, self.dateName))
):
#print dir,dirs,files
if (os.path.split(dir)[0] ==
os.path.split(self.datePath(utc_now())[0])):
#print 'skipping dir %s' % dir
#print 'because: %s == %s' % (os.path.split(dir)[0],
# os.path.split(self.datePath(utc_now())[0]))
continue
# the links are all to (relative) directories, so we need not
# look at files
for d in dirs:
#print 'dir ', d
if self.osModule.path.islink(os.path.join(dir, d)):
r = handleLink(dir, d)
#print ' r ', r
if r:
yield r
# after finishing a given directory...
socorro_fs.cleanEmptySubdirectories(
os.path.join(self.root, daily),
dir,
self.osModule
)
def testCleanEmptySubdirectories(self):
f.makedirs('TestDir/A/B/C/D')
f.makedirs('TestDir/AA/BB/C')
f.makedirs('TestDir/AA/BB/CC/DD')
fi = open('TestDir/A/a','w')
fi.write('file a\n')
fi.close()
# Test short-circuit path, full stopper
assert os.path.isdir('TestDir/A/B/C/D')
f.cleanEmptySubdirectories('TestDir/A/B/C/D','TestDir/A/B/C/D')
assert os.path.isdir('TestDir/A/B/C/D')
# Test short-circuit path, name stopper
f.cleanEmptySubdirectories('D','TestDir/A/B/C/D')
assert os.path.isdir('TestDir/A/B/C/D')
# Test some empties, name stopper
f.cleanEmptySubdirectories('C','TestDir/A/B/C/D')
assert not os.path.exists('TestDir/A/B/C/D')
assert os.path.isdir('TestDir/A/B/C')
# Test some empties, path stopper
f.cleanEmptySubdirectories('TestDir/A/B','TestDir/A/B/C')
assert not os.path.exists('TestDir/A/B/C')
assert os.path.isdir('TestDir/A/B')
#Test stopping on a file in a subdir
f.cleanEmptySubdirectories('TestDir','TestDir/A/B')
assert not os.path.exists('TestDir/A/B')
assert os.path.isdir('TestDir/A')
#Test stopping on another subdir
f.cleanEmptySubdirectories('TestDir/AA','TestDir/AA/BB/CC/DD')
assert not os.path.exists('TestDir/AA/BB/CC')
assert os.path.isdir('TestDir/AA/BB')
#Test for stopper not in path
assert_raises(OSError,f.cleanEmptySubdirectories,'Woo','TestDir/AA/BB')
#Test for non-existent leaf
assert_raises(OSError,f.cleanEmptySubdirectories,'TestDir','TestDir/AA/BB/CC/DD')
def testCleanEmptySubdirectories(self):
f.makedirs('TestDir/A/B/C/D')
f.makedirs('TestDir/AA/BB/C')
f.makedirs('TestDir/AA/BB/CC/DD')
fi = open('TestDir/A/a','w')
fi.write('file a\n')
fi.close()
# Test short-circuit path, full stopper
assert os.path.isdir('TestDir/A/B/C/D')
f.cleanEmptySubdirectories('TestDir/A/B/C/D','TestDir/A/B/C/D')
assert os.path.isdir('TestDir/A/B/C/D')
# Test short-circuit path, name stopper
f.cleanEmptySubdirectories('D','TestDir/A/B/C/D')
assert os.path.isdir('TestDir/A/B/C/D')
# Test some empties, name stopper
f.cleanEmptySubdirectories('C','TestDir/A/B/C/D')
assert not os.path.exists('TestDir/A/B/C/D')
assert os.path.isdir('TestDir/A/B/C')
# Test some empties, path stopper
f.cleanEmptySubdirectories('TestDir/A/B','TestDir/A/B/C')
assert not os.path.exists('TestDir/A/B/C')
assert os.path.isdir('TestDir/A/B')
#Test stopping on a file in a subdir
f.cleanEmptySubdirectories('TestDir','TestDir/A/B')
assert not os.path.exists('TestDir/A/B')
assert os.path.isdir('TestDir/A')
#Test stopping on another subdir
f.cleanEmptySubdirectories('TestDir/AA','TestDir/AA/BB/CC/DD')
assert not os.path.exists('TestDir/AA/BB/CC')
assert os.path.isdir('TestDir/AA/BB')
#Test for stopper not in path
assert_raises(OSError,f.cleanEmptySubdirectories,'Woo','TestDir/AA/BB')
#Test for non-existent leaf
assert_raises(OSError,f.cleanEmptySubdirectories,'TestDir','TestDir/AA/BB/CC/DD')
def _remove(self, ooid, namePath, nameParts, datePath, dateParts):
seenCount = 0
dailyPart = None
if nameParts:
dailyPart = nameParts[1]
elif namePath:
dailyPart = namePath.split(os.sep, 2)[1]
elif dateParts:
dailyPart = dateParts[1]
elif datePath:
dailyPart = datePath.split(os.sep, 2)[1]
if not dailyPart:
return
stopper = os.path.join(self.root, dailyPart)
# unlink on the name side first, thereby erasing any hope of removing
# relative paths from here...
if namePath:
try:
self.osModule.unlink(os.path.join(namePath, ooid))
seenCount += 1
except:
pass
try:
self.osModule.unlink(
os.path.join(namePath, ooid + self.jsonSuffix)
)
seenCount += 1
except:
pass
try:
self.osModule.unlink(
os.path.join(namePath, ooid + self.dumpSuffix)
)
seenCount += 1
except:
pass
if self.cleanIndexDirectories:
try:
socorro_fs.cleanEmptySubdirectories(
stopper,
namePath,
self.osModule
) # clean out name side if possible
except OSError:
pass
# and the date directory
if datePath:
try:
self.osModule.unlink(os.path.join(datePath, ooid))
seenCount += 1
except:
pass
try:
socorro_fs.cleanEmptySubdirectories(
self.root,
datePath,
self.osModule
)
except:
pass
if not seenCount:
self.logger.warning("%s was totally unknown", ooid)
raise NoSuchUuidFound("no trace of %s was found" % ooid)
