def run(self):
"""Execute the given function on files from the file queue.
"""
while self.running:
try:
filename = self.queue.get(block=True, timeout=self.refresh)
LOG.debug("processing %s"%filename)
except Empty:
filename = None
try:
self.fun(filename)
except:
LOG.exception("Something wrong happened in %s for %s. Skipping."
%(str(self.fun), filename))
LOG.info("FileProcessor terminated.")
def save_object(self, obj):
"""save *obj* to the filelist.
"""
files_by_ext = self._get_by_ext()
for extkey in files_by_ext:
path, trash = os.path.split(files_by_ext[extkey][0])
del trash
try:
ensure_dir(files_by_ext[extkey][0])
handle, tmpfilename = tempfile.mkstemp(extkey,
"mpop_tmp",
path)
os.fsync(handle)
obj.save(tmpfilename)
os.fsync(handle)
os.chmod(tmpfilename, 0644)
os.fsync(handle)
except Exception:
LOG.exception("Something went wrong in saving file... "
"Dumping trace.")
LOG.warning("Job skipped, going on with the next.")
continue
for filename in files_by_ext[extkey][1:]:
path2, trash = os.path.split(filename)
del trash
ensure_dir(filename)
handle2, tmpfilename2 = tempfile.mkstemp(extkey,
"mpop_tmp",
path2)
os.fsync(handle2)
try:
shutil.copy(tmpfilename, tmpfilename2)
os.fsync(handle2)
os.close(handle2)
except (IOError, OSError):
LOG.exception("Copying file %s to %s failed"
%(tmpfilename,tmpfilename2))
LOG.info("Retrying...")
try:
shutil.copy(tmpfilename, tmpfilename2)
os.fsync(handle2)
os.close(handle2)
LOG.info("Went OK this time...")
except (IOError, OSError):
LOG.exception("No way...")
try:
os.rename(tmpfilename2, filename)
except (IOError, OSError):
LOG.exception("Renaming file %s to %s failed"
%(tmpfilename2,filename))
LOG.info("Retrying...")
try:
os.rename(tmpfilename2, filename)
except (IOError, OSError):
LOG.exception("No way...")
LOG.debug("Done saving "+filename)
os.rename(tmpfilename, files_by_ext[extkey][0])
os.fsync(handle)
os.close(handle)
LOG.debug("Done saving "+files_by_ext[extkey][0])
