def run(self): """Execute the given function on files from the file queue. """ while self.running: try: filename = self.queue.get(block=True, timeout=self.refresh) LOG.debug("processing %s"%filename) except Empty: filename = None try: self.fun(filename) except: LOG.exception("Something wrong happened in %s for %s. Skipping." %(str(self.fun), filename)) LOG.info("FileProcessor terminated.")
def save_object(self, obj): """save *obj* to the filelist. """ files_by_ext = self._get_by_ext() for extkey in files_by_ext: path, trash = os.path.split(files_by_ext[extkey][0]) del trash try: ensure_dir(files_by_ext[extkey][0]) handle, tmpfilename = tempfile.mkstemp(extkey, "mpop_tmp", path) os.fsync(handle) obj.save(tmpfilename) os.fsync(handle) os.chmod(tmpfilename, 0644) os.fsync(handle) except Exception: LOG.exception("Something went wrong in saving file... " "Dumping trace.") LOG.warning("Job skipped, going on with the next.") continue for filename in files_by_ext[extkey][1:]: path2, trash = os.path.split(filename) del trash ensure_dir(filename) handle2, tmpfilename2 = tempfile.mkstemp(extkey, "mpop_tmp", path2) os.fsync(handle2) try: shutil.copy(tmpfilename, tmpfilename2) os.fsync(handle2) os.close(handle2) except (IOError, OSError): LOG.exception("Copying file %s to %s failed" %(tmpfilename,tmpfilename2)) LOG.info("Retrying...") try: shutil.copy(tmpfilename, tmpfilename2) os.fsync(handle2) os.close(handle2) LOG.info("Went OK this time...") except (IOError, OSError): LOG.exception("No way...") try: os.rename(tmpfilename2, filename) except (IOError, OSError): LOG.exception("Renaming file %s to %s failed" %(tmpfilename2,filename)) LOG.info("Retrying...") try: os.rename(tmpfilename2, filename) except (IOError, OSError): LOG.exception("No way...") LOG.debug("Done saving "+filename) os.rename(tmpfilename, files_by_ext[extkey][0]) os.fsync(handle) os.close(handle) LOG.debug("Done saving "+files_by_ext[extkey][0])