def _remove(self, filepath):
try:
remove(filepath)
except (IOError, OSError):
self.failed_to_remove.append(filepath)
wstderr('ERROR: Failed to remove file "%s"\n' % filepath)
return 'failed'
else:
self.removed.append(filepath)
return 'removed'
finally:
self._done_with(filepath)
def _rename_inner(self, filepath, new_name):
try:
rename(filepath, new_name)
except (IOError, OSError):
wstderr('Failed to rename file "%s" -> "%s"\n'
% (filepath, new_name))
self.failed_to_rename.append((filepath, new_name))
return 'failed'
else:
self.renamed.append((filepath, new_name))
return 'renamed'
finally:
self._done_with(filepath)
def deep_clean(self, n_procs=None):
"""Run the cleaning process
Parameters
----------
ignore_locks : bool
Whether to ignore (valid) locks. Note that invalid locks are
ignored regardless of the value of `ignore_locks`.
"""
start_time = time.time()
wstdout('> Deep-cleaning the I3 files in the directory...\n')
if n_procs is None:
n_procs = min(len(self._allfilepaths), 8*cpu_count())
# Create a manager for objects synchronized across workers
mgr = Manager()
groups = [mgr.list(g) for g in self.group_by_event().values()]
pool = Pool(processes=n_procs)
ret = pool.map(DeepCleaner(), groups)
removed = []
renamed = []
failed_to_remove = []
failed_to_rename = []
for d in ret:
removed.extend(d['removed'])
renamed.extend(d['renamed'])
failed_to_remove.extend(d['failed_to_remove'])
failed_to_rename.extend(d['failed_to_rename'])
wstderr(' '.join([str(len(g)) for g in groups]) + '\n')
# TODO: remove files that -- while not redundant -- have fewer recos
# (or, stretch goal: merge the non-redundant info together)
self.report()
wstdout('> Time to run deep cleaning: %s\n'
% timediffstamp(time.time() - start_time))
def _rename(self, filepath, recos):
new_name = pathFromRecos(filepath, recos)
wstderr(' Renaming to: "%s"\n' % new_name)
if isfile(new_name):
wstderr(' Conflicting file found, recursing...\n')
action_taken = self._rename_or_remove(new_name)
# If conflicting file is moved or removed...
if action_taken in ['removed', 'renamed']:
return self._rename_inner(filepath, new_name)
# Otherwise, conflicting file was kept, or failed to be
# renamed or removed, so this file must be removed
else:
wstderr(' Removing original file (conflict with an'
' existing file of same name)\n')
return self._remove(filepath)
else:
return self._rename_inner(filepath, new_name)
def _sigint_handler(signal, frame): # pylint: disable=unused-argument, redefined-outer-name
wstderr('=' * 79 + '\n')
wstderr('*** CAUGHT CTL-C (sigint) *** ... attempting to cleanup!\n')
wstderr('=' * 79 + '\n')
raise KeyboardInterrupt
finally:
if lock_f is not None:
try:
remove(lockfilepath)
except:
exc_str = ''.join(format_exception(*sys.exc_info()))
wstdout('ERROR! Could not remove lockfile "%s"\n%s\n'
% (lockfilepath, exc_str))
recos_run_on_all_events = set()
if len(unique_recos_in_file) > 0:
recos_run_on_all_events = reduce(set.intersection,
unique_recos_in_file)
if purportedly_run != recos_run_on_all_events:
wstderr(' Recos found do not match filename. Renaming...\n')
return self._rename(filepath,
recos=sorted(recos_run_on_all_events))
wstderr(' File is in order. Moving on.\n')
self._done_with(filepath)
return 'kept'
def _done_with(self, filepath):
try:
self.file_list.remove(filepath)
except ValueError:
pass
def _rename(self, filepath, recos):
new_name = pathFromRecos(filepath, recos)
def _sigint_handler(signal, frame): # pylint: disable=unused-argument, redefined-outer-name
wstderr('='*79 + '\n')
wstderr('*** CAUGHT CTL-C (sigint) *** ... attempting to cleanup!\n')
wstderr('='*79 + '\n')
raise KeyboardInterrupt
# datetime
#==============================================================================
startTimeSec = time.time()
startTime = time.strftime("%Y-%m-%dT%H%M%S") + \
"{0:+05d}".format(-int(round(time.timezone/3600)*100))
#homeDir = os.path.expanduser("~")
homeDir = os.path.join("home", "justin")
backupBaseDir = os.path.join(homeDir, "wiki_archive")
backupSubDir = os.path.join(dataBaseDir, time.strftime("%Y"),
time.strftime("%m-%b"))
#-- Create directory if necessary (and if possible)
wstderr("\nCreating data directory structure...\n")
if os.path.exists(dataDir):
createdDataDir = False
if not os.path.isdir(dataDir):
raise Exception("Wanted to create directory " + dataDir +
" but this path is a file.")
else:
createdDataDir = True
os.makedirs(dataDir, mode=0770)
#==============================================================================
# 2. Take wiki offline: sudo service apache2 stop
#==============================================================================
subprocess.call("service apache2 stop")