Ejemplo n.º 1
0
 def terminate(self):
     """Terminate thread.
     """
     self.running = False
     self.cond.acquire()
     self.cond.notify()
     self.cond.release()
     LOG.debug("Termination request received in FileWatcher")
Ejemplo n.º 2
0
    def run(self):
        """Run the file watcher.
        """

        filelist = set()
        sleep_time = 8
        
        while self.running:
            self.cond.acquire()
            if isinstance(self.template, (list, tuple)):
                new_filelist = []
                for template in self.template:
                    new_filelist += glob.glob(template)
                new_filelist = set(new_filelist)
            else:
                new_filelist = set(glob.glob(self.template))
            files_to_process = list(new_filelist - filelist)
            filelist = new_filelist

            files_dict = {}
            for fil in files_to_process:
                files_dict[fil] = os.path.getmtime(fil)

            files_to_process.sort(lambda x, y: cmp(files_dict[x],
                                                  files_dict[y]))

            if len(files_to_process) != 0 and self.running:
                sleep_time = 8
                times = []
                for i in files_to_process:
                    LOG.debug("queueing %s..."%i)
                    self.queue.put(i)
                    times.append(os.stat(i).st_ctime)
                times.sort()

                since_creation = datetime.timedelta(seconds=time.time() -
                                                    times[-1])
                if(self.frequency > since_creation):
                    to_wait = self.frequency - since_creation

                    LOG.info("Waiting at least "+str(to_wait)+" for next file")
                    sleep_time = (to_wait.seconds +
                                  to_wait.microseconds / 1000000.0)
                    self.wait(sleep_time)
                    sleep_time = 8
            elif self.running:
                LOG.info("no new file has come, waiting %s secs"
                         %str(sleep_time))
                self.wait(sleep_time)
                if sleep_time < 60:
                    sleep_time *= 2

            self.cond.release()
        LOG.info("FileWatcher terminated.")
Ejemplo n.º 3
0
    def run(self):
        """Execute the given function on files from the file queue.
        """

        while self.running:
            try:
                filename = self.queue.get(block=True, timeout=self.refresh)
                LOG.debug("processing %s"%filename)
            except Empty:
                filename = None
            try:
                self.fun(filename)
            except:
                LOG.exception("Something wrong happened in %s for %s. Skipping."
                              %(str(self.fun), filename))
        LOG.info("FileProcessor terminated.")
Ejemplo n.º 4
0
 def terminate(self):
     """Terminate thread.
     """
     self.running = False
     LOG.debug("Termination request received in FileProcessor")
Ejemplo n.º 5
0
    def save_object(self, obj):
        """save *obj* to the filelist.
        """
        files_by_ext = self._get_by_ext()
        for extkey in files_by_ext:
            path, trash = os.path.split(files_by_ext[extkey][0])
            del trash
            try:
                ensure_dir(files_by_ext[extkey][0])
                handle, tmpfilename = tempfile.mkstemp(extkey,
                                                       "mpop_tmp",
                                                       path)
                os.fsync(handle)
                obj.save(tmpfilename)
                os.fsync(handle)
                os.chmod(tmpfilename, 0644)
                os.fsync(handle)
            except Exception:
                LOG.exception("Something went wrong in saving file... "
                              "Dumping trace.")
                LOG.warning("Job skipped, going on with the next.")
                continue
            for filename in files_by_ext[extkey][1:]:
                path2, trash = os.path.split(filename)
                del trash

                ensure_dir(filename)
                handle2, tmpfilename2 = tempfile.mkstemp(extkey,
                                                         "mpop_tmp",
                                                         path2)
                os.fsync(handle2)
                try:
                    shutil.copy(tmpfilename, tmpfilename2)
                    os.fsync(handle2)
                    os.close(handle2)
                except (IOError, OSError):
                    LOG.exception("Copying file %s to %s failed"
                                  %(tmpfilename,tmpfilename2))
                    LOG.info("Retrying...")
                    try:
                        shutil.copy(tmpfilename, tmpfilename2)
                        os.fsync(handle2)
                        os.close(handle2)
                        LOG.info("Went OK this time...")
                    except (IOError, OSError):
                        LOG.exception("No way...")
                try:
                    os.rename(tmpfilename2, filename)
                except (IOError, OSError):
                    LOG.exception("Renaming file %s to %s failed"
                                %(tmpfilename2,filename))
                    LOG.info("Retrying...")
                    try:
                        os.rename(tmpfilename2, filename)
                    except (IOError, OSError):
                        LOG.exception("No way...")
                LOG.debug("Done saving "+filename)
                
            os.rename(tmpfilename, files_by_ext[extkey][0])
            os.fsync(handle)
            os.close(handle)
            LOG.debug("Done saving "+files_by_ext[extkey][0])