def __init__(self): self.inotify = Inotify() self.watches = {} self.queue = Queue() self.lock = RLock() for t in (self._push, self._pull): t = Thread(target=t) t.setDaemon(True) t.start()
class Watcher(object): def __init__(self): self.inotify = Inotify() self.watches = {} self.queue = Queue() self.lock = RLock() for t in (self._push, self._pull): t = Thread(target=t) t.setDaemon(True) t.start() def _push(self): while True: for wd, mask, cookie, name in self.inotify.read(): self.queue.put(Event(wd, self, mask, cookie, name)) def _pull(self): while True: event = self.queue.get() mask = event.mask self.lock.acquire() watch = self.watches.get(event.wd) self.lock.release() watch.callback(event) if mask & ISDIR and mask & CREATE and watch.auto: self.add(event.path, watch.mask, watch.callback, auto=True, _parent=watch) if mask & IGNORED: self.rem(watch) def add(self, path, mask=ALL, callback=default, auto=True, _parent=None): self.lock.acquire() wd = self.inotify.add_watch(path, mask) watch = Watch(wd, self, mask, path, callback, auto, _parent) self.watches[wd] = watch self.lock.release() return watch def rem(self, watch): self.lock.acquire() for w in list(self.watches.values()): if w._parent == watch: self.rem(w) wd = watch.wd self.watches.pop(wd) self.lock.release() def close(self): self.inotify.close()
def monitor_file(self): signal.signal(signal.SIGTERM, self.exit) while not self.exit_flag: try: self.handlers = self.get_handlers() self._load_pickle() self.inotify = Inotify() self.inotify.add_watch(self.pool_path, IN_CREATE | IN_MOVED_FROM | IN_MOVED_TO | IN_MODIFY | IN_DELETE | IN_DELETE_SELF) _epoll = select.epoll() _epoll.register(self.inotify.fileno(), select.EPOLLIN) self.pre_process() self._break_flag = True while not self.exit_flag and self._break_flag: try: for fd, epoll_event in _epoll.poll(-1): if fd == self.inotify.fileno(): for inotify_event in self.inotify.read_events(): self.process_event(inotify_event) except IOError as e: if e.errno != EINTR: raise e except Exception: self.log.error(traceback.format_exc()) finally: self.clean_handlers() _epoll.close()
dirList = [] remoteList = [] commandList = [] wdDirDict = {} dirs = False remotes = False commands = False user = False # checks environment variables for username, system portable # might be set to a different user later if argument is there username = getpass.getuser() inotify = Inotify() # we don't need to consider name of script as an arg sys.argv.pop(0) if len(sys.argv) == 0: print __doc__ quit() # sort of a switching system for handling arguments # it may not be terribly efficient, but it makes adding arguments easy as pie for arg in sys.argv: if arg == "-h": print __doc__ quit() elif arg == "-c":
def start_multiple(products, user, password, refresh_type="forced", run_once=False, mtimer_options={}): """ similar to start() but using multiple configurations """ import ZPublisher.Client from Globals import DevelopmentMode global ServerError ServerError = ZPublisher.Client.ServerError # objects=[ZPublisher.Client.Object(url, username=user, password=password)] products_by_path = {} for product in products: key = product["productpath"] url = product["CTurl"] dependencies = product.get("dependencies", None) if isinstance(dependencies, basestring): dependencies = [dependencies] elif isinstance(dependencies, tuple): dependencies = list(dependencies) if dependencies: # # this dependency list is expected to be a list of urls # that start like the 'url' variable does. If that's not the case, # fix them. baseurl = "/".join(url.split("/")[:-1]) for i in range(len(dependencies)): if not dependencies[i].startswith(baseurl): dependencies[i] = baseurl + "/" + dependencies[i] if url.endswith("/"): url = url[:-1] products_by_path[key] = product objects = [ZPublisher.Client.Object(url, username=user, password=password)] products_by_path[key]["pre_run_command"] = product.get("pre_run_command") products_by_path[key]["post_run_command"] = product.get("post_run_command") if dependencies: for dep_url in dependencies: objects.append(ZPublisher.Client.Object(dep_url, username=user, password=password)) # products_by_path[key]['dependencies'] = dependencies # cleaned up products_by_path[key]["objects"] = objects if _has_inotify: extensions_to_scan = ("sql", "py", "zpt", "dtml", "pt", "js") if refresh_type == "clever" and DevelopmentMode: # cool, then we don't have refresh when there's a change to # templates like .dtml and .zpt if _has_inotify: extensions_to_scan = ("sql", "py", "js", "css") else: mtimer.EXTENSIONS_TO_SCAN = ("sql", "py", "css", "js") mtimer.FOLDERS_TO_SKIP = ("tests", ".autogenerated", "zpt", "dtml") + mtimer.FOLDERS_TO_SKIP else: mtimer.FOLDERS_TO_SKIP = ("tests", ".autogenerated") + mtimer.FOLDERS_TO_SKIP c = 0 if _has_inotify and not run_once: notifier = Inotify() mask = Inotify.IN_MODIFY for product_path, config in products_by_path.items(): assert os.path.isdir(product_path), "product path does not exist %s" % product_path _recursiveAddWatch(notifier, product_path, mask) while 1: events = notifier.read_events() for event in events: event_string = notifier.event_string(event[Inotify.MASK]) path_wid = notifier.file(event[Inotify.WID]) file_wid = event[Inotify.FILE] file_wid_extension = file_wid.split(".")[-1] if file_wid_extension in extensions_to_scan: # the path might be 'Products/Foo/zpt/widgets' when the product # path is just 'Products/Foo' for product_path, config in products_by_path.items(): if path_wid.startswith(product_path): _file_path = os.path.join(path_wid, file_wid).replace("Products/", "") print _file_path, t0 = time.time() r = refresh_product( config["objects"], config["pre_run_command"], config["post_run_command"] ) t1 = time.time() log("%s:%s:%s" % (_file_path, t1 - t0, time.time())) if r: print r else: print "done" # print event_string break else: def _getSleeptime(): try: return float(open("sleeptime.conf").read().split()[0]) except: return 4.0 # seconds def _saveSleeptime(sleeptime): open("sleeptime.conf", "w").write(str(sleeptime)) delta = 0.0 product_info = {} while 1: try: sleeptime = _getSleeptime() for product_path, config in products_by_path.items(): # t0=time.time() difference, info = mtimer.run( product_path, prev_info=product_info.get(product_path, None), **mtimer_options ) product_info[product_path] = info if difference == "Different": t0 = time.time() refresh_product(config["objects"], config["pre_run_command"], config["post_run_command"]), t1 = time.time() print "(%s)" % (seconds2time(c)) log("%s:%s:%s" % (product_path, t1 - t0, time.time())) delta = 0.0 # print time.time()-t0, "in", product_path if run_once: break # print "Sleep(%s)" % sleeptime time.sleep(sleeptime + delta) # ; print "sleep", sleeptime + delta # every time there's. For every loop, increase the sleeptime with # a tiny delta that can't be bigger than the sleeptime. # this min() makes sure the extra time never exceeds the original # sleeptime. This means that if you leave this running for a very # long time the (sleeptime+delta) === sleeptime*2 delta = min(delta + sleeptime / 200.0, sleeptime) c += sleeptime except KeyboardInterrupt: break
class InotifyPoll(FilePollBase): def __init__(self, config, section): super(InotifyPoll, self).__init__(config, section) self.inotify = None self._break_flag = True def on_file_modify(self, event): return event & IN_MODIFY def on_file_remove(self, event): return event & IN_DELETE | event & IN_MOVED_FROM def on_file_new(self, event): return event & IN_CREATE | event & IN_MOVED_TO def on_directory_delete(self, event): return event & IN_DELETE_SELF def monitor_file(self): signal.signal(signal.SIGTERM, self.exit) while not self.exit_flag: try: self.handlers = self.get_handlers() self._load_pickle() self.inotify = Inotify() self.inotify.add_watch(self.pool_path, IN_CREATE | IN_MOVED_FROM | IN_MOVED_TO | IN_MODIFY | IN_DELETE | IN_DELETE_SELF) _epoll = select.epoll() _epoll.register(self.inotify.fileno(), select.EPOLLIN) self.pre_process() self._break_flag = True while not self.exit_flag and self._break_flag: try: for fd, epoll_event in _epoll.poll(-1): if fd == self.inotify.fileno(): for inotify_event in self.inotify.read_events(): self.process_event(inotify_event) except IOError as e: if e.errno != EINTR: raise e except Exception: self.log.error(traceback.format_exc()) finally: self.clean_handlers() _epoll.close() def process_event(self, inotify_event): (watch_path, mask, cookie, filename) = inotify_event if self.on_file_new(mask): if self.is_need_monitor(filename): full_filename = os.path.join(watch_path, filename) if os.path.isfile(full_filename): _new_file_handler = open(full_filename, 'r') self.monitor_dict[full_filename] = _new_file_handler self.handlers.append(_new_file_handler) self._update_pickle(_new_file_handler) elif self.on_file_modify(mask): _in_process_file_handler = self.monitor_dict.get(os.path.join(watch_path, filename)) if _in_process_file_handler: _data = self._get_log_data_by_handler(_in_process_file_handler) if _data: for _line in _data: msg = self.reformate(filename=_in_process_file_handler.name, data=_line) if msg: self.channel.put(msg) # 数据完整性由collector来保证 self._update_pickle(_in_process_file_handler) elif self.on_file_remove(mask): _delete_file_handler = self.monitor_dict.get(os.path.join(watch_path, filename)) if _delete_file_handler: self.monitor_dict.pop(os.path.join(watch_path, filename)) _delete_file_handler.close() self.handlers.remove(_delete_file_handler) self._reset_pickle([_delete_file_handler.name]) elif self.on_directory_delete(mask): parent_path = os.path.normpath(os.path.join(self.pool_path, os.pardir)) basename = os.path.basename(os.path.normpath(self.pool_path)) self.inotify.remove_watch(self.pool_path) self.inotify.add_watch(parent_path, IN_CREATE | IN_MOVED_TO) self.log.debug('pool_path is deleted, waiting for pool_path to be created') wait_flag = False if os.path.exists(self.pool_path) else True while not self.exit_flag and wait_flag: try: for self.inotify_event in self.inotify.read_events(): (watch_path, mask, cookie, filename) = self.inotify_event if filename == basename: self.inotify.add_watch(self.pool_path, IN_CREATE | IN_MOVED_FROM | IN_MOVED_TO | IN_MODIFY | IN_DELETE | IN_DELETE_SELF) wait_flag = False except IOError as e: if e.errno != EINTR: raise e self._break_flag = False self.inotify.remove_watch(parent_path) self.log.debug('stop waiting for pool_path to be created') def exit(self, *args, **kwargs): self.log.info('Received sigterm, agent[{}] is going down.'.format(self.name)) self.exit_flag = True