def _remove(self, filepath): try: remove(filepath) except (IOError, OSError): self.failed_to_remove.append(filepath) wstderr('ERROR: Failed to remove file "%s"\n' % filepath) return 'failed' else: self.removed.append(filepath) return 'removed' finally: self._done_with(filepath)
def _rename_inner(self, filepath, new_name): try: rename(filepath, new_name) except (IOError, OSError): wstderr('Failed to rename file "%s" -> "%s"\n' % (filepath, new_name)) self.failed_to_rename.append((filepath, new_name)) return 'failed' else: self.renamed.append((filepath, new_name)) return 'renamed' finally: self._done_with(filepath)
def deep_clean(self, n_procs=None): """Run the cleaning process Parameters ---------- ignore_locks : bool Whether to ignore (valid) locks. Note that invalid locks are ignored regardless of the value of `ignore_locks`. """ start_time = time.time() wstdout('> Deep-cleaning the I3 files in the directory...\n') if n_procs is None: n_procs = min(len(self._allfilepaths), 8*cpu_count()) # Create a manager for objects synchronized across workers mgr = Manager() groups = [mgr.list(g) for g in self.group_by_event().values()] pool = Pool(processes=n_procs) ret = pool.map(DeepCleaner(), groups) removed = [] renamed = [] failed_to_remove = [] failed_to_rename = [] for d in ret: removed.extend(d['removed']) renamed.extend(d['renamed']) failed_to_remove.extend(d['failed_to_remove']) failed_to_rename.extend(d['failed_to_rename']) wstderr(' '.join([str(len(g)) for g in groups]) + '\n') # TODO: remove files that -- while not redundant -- have fewer recos # (or, stretch goal: merge the non-redundant info together) self.report() wstdout('> Time to run deep cleaning: %s\n' % timediffstamp(time.time() - start_time))
def _rename(self, filepath, recos): new_name = pathFromRecos(filepath, recos) wstderr(' Renaming to: "%s"\n' % new_name) if isfile(new_name): wstderr(' Conflicting file found, recursing...\n') action_taken = self._rename_or_remove(new_name) # If conflicting file is moved or removed... if action_taken in ['removed', 'renamed']: return self._rename_inner(filepath, new_name) # Otherwise, conflicting file was kept, or failed to be # renamed or removed, so this file must be removed else: wstderr(' Removing original file (conflict with an' ' existing file of same name)\n') return self._remove(filepath) else: return self._rename_inner(filepath, new_name)
def _sigint_handler(signal, frame): # pylint: disable=unused-argument, redefined-outer-name wstderr('=' * 79 + '\n') wstderr('*** CAUGHT CTL-C (sigint) *** ... attempting to cleanup!\n') wstderr('=' * 79 + '\n') raise KeyboardInterrupt
finally: if lock_f is not None: try: remove(lockfilepath) except: exc_str = ''.join(format_exception(*sys.exc_info())) wstdout('ERROR! Could not remove lockfile "%s"\n%s\n' % (lockfilepath, exc_str)) recos_run_on_all_events = set() if len(unique_recos_in_file) > 0: recos_run_on_all_events = reduce(set.intersection, unique_recos_in_file) if purportedly_run != recos_run_on_all_events: wstderr(' Recos found do not match filename. Renaming...\n') return self._rename(filepath, recos=sorted(recos_run_on_all_events)) wstderr(' File is in order. Moving on.\n') self._done_with(filepath) return 'kept' def _done_with(self, filepath): try: self.file_list.remove(filepath) except ValueError: pass def _rename(self, filepath, recos): new_name = pathFromRecos(filepath, recos)
def _sigint_handler(signal, frame): # pylint: disable=unused-argument, redefined-outer-name wstderr('='*79 + '\n') wstderr('*** CAUGHT CTL-C (sigint) *** ... attempting to cleanup!\n') wstderr('='*79 + '\n') raise KeyboardInterrupt
# datetime #============================================================================== startTimeSec = time.time() startTime = time.strftime("%Y-%m-%dT%H%M%S") + \ "{0:+05d}".format(-int(round(time.timezone/3600)*100)) #homeDir = os.path.expanduser("~") homeDir = os.path.join("home", "justin") backupBaseDir = os.path.join(homeDir, "wiki_archive") backupSubDir = os.path.join(dataBaseDir, time.strftime("%Y"), time.strftime("%m-%b")) #-- Create directory if necessary (and if possible) wstderr("\nCreating data directory structure...\n") if os.path.exists(dataDir): createdDataDir = False if not os.path.isdir(dataDir): raise Exception("Wanted to create directory " + dataDir + " but this path is a file.") else: createdDataDir = True os.makedirs(dataDir, mode=0770) #============================================================================== # 2. Take wiki offline: sudo service apache2 stop #============================================================================== subprocess.call("service apache2 stop")