def terminate(self):
"""Terminate thread.
"""
self.running = False
self.cond.acquire()
self.cond.notify()
self.cond.release()
LOG.debug("Termination request received in FileWatcher")
def run(self):
"""Run the file watcher.
"""
filelist = set()
sleep_time = 8
while self.running:
self.cond.acquire()
if isinstance(self.template, (list, tuple)):
new_filelist = []
for template in self.template:
new_filelist += glob.glob(template)
new_filelist = set(new_filelist)
else:
new_filelist = set(glob.glob(self.template))
files_to_process = list(new_filelist - filelist)
filelist = new_filelist
files_dict = {}
for fil in files_to_process:
files_dict[fil] = os.path.getmtime(fil)
files_to_process.sort(lambda x, y: cmp(files_dict[x],
files_dict[y]))
if len(files_to_process) != 0 and self.running:
sleep_time = 8
times = []
for i in files_to_process:
LOG.debug("queueing %s..."%i)
self.queue.put(i)
times.append(os.stat(i).st_ctime)
times.sort()
since_creation = datetime.timedelta(seconds=time.time() -
times[-1])
if(self.frequency > since_creation):
to_wait = self.frequency - since_creation
LOG.info("Waiting at least "+str(to_wait)+" for next file")
sleep_time = (to_wait.seconds +
to_wait.microseconds / 1000000.0)
self.wait(sleep_time)
sleep_time = 8
elif self.running:
LOG.info("no new file has come, waiting %s secs"
%str(sleep_time))
self.wait(sleep_time)
if sleep_time < 60:
sleep_time *= 2
self.cond.release()
LOG.info("FileWatcher terminated.")
def run(self):
"""Execute the given function on files from the file queue.
"""
while self.running:
try:
filename = self.queue.get(block=True, timeout=self.refresh)
LOG.debug("processing %s"%filename)
except Empty:
filename = None
try:
self.fun(filename)
except:
LOG.exception("Something wrong happened in %s for %s. Skipping."
%(str(self.fun), filename))
LOG.info("FileProcessor terminated.")
def terminate(self):
"""Terminate thread.
"""
self.running = False
LOG.debug("Termination request received in FileProcessor")
def save_object(self, obj):
"""save *obj* to the filelist.
"""
files_by_ext = self._get_by_ext()
for extkey in files_by_ext:
path, trash = os.path.split(files_by_ext[extkey][0])
del trash
try:
ensure_dir(files_by_ext[extkey][0])
handle, tmpfilename = tempfile.mkstemp(extkey,
"mpop_tmp",
path)
os.fsync(handle)
obj.save(tmpfilename)
os.fsync(handle)
os.chmod(tmpfilename, 0644)
os.fsync(handle)
except Exception:
LOG.exception("Something went wrong in saving file... "
"Dumping trace.")
LOG.warning("Job skipped, going on with the next.")
continue
for filename in files_by_ext[extkey][1:]:
path2, trash = os.path.split(filename)
del trash
ensure_dir(filename)
handle2, tmpfilename2 = tempfile.mkstemp(extkey,
"mpop_tmp",
path2)
os.fsync(handle2)
try:
shutil.copy(tmpfilename, tmpfilename2)
os.fsync(handle2)
os.close(handle2)
except (IOError, OSError):
LOG.exception("Copying file %s to %s failed"
%(tmpfilename,tmpfilename2))
LOG.info("Retrying...")
try:
shutil.copy(tmpfilename, tmpfilename2)
os.fsync(handle2)
os.close(handle2)
LOG.info("Went OK this time...")
except (IOError, OSError):
LOG.exception("No way...")
try:
os.rename(tmpfilename2, filename)
except (IOError, OSError):
LOG.exception("Renaming file %s to %s failed"
%(tmpfilename2,filename))
LOG.info("Retrying...")
try:
os.rename(tmpfilename2, filename)
except (IOError, OSError):
LOG.exception("No way...")
LOG.debug("Done saving "+filename)
os.rename(tmpfilename, files_by_ext[extkey][0])
os.fsync(handle)
os.close(handle)
LOG.debug("Done saving "+files_by_ext[extkey][0])
