def __init__(self):
self.inotify = Inotify()
self.watches = {}
self.queue = Queue()
self.lock = RLock()
for t in (self._push, self._pull):
t = Thread(target=t)
t.setDaemon(True)
t.start()
class Watcher(object):
def __init__(self):
self.inotify = Inotify()
self.watches = {}
self.queue = Queue()
self.lock = RLock()
for t in (self._push, self._pull):
t = Thread(target=t)
t.setDaemon(True)
t.start()
def _push(self):
while True:
for wd, mask, cookie, name in self.inotify.read():
self.queue.put(Event(wd, self, mask, cookie, name))
def _pull(self):
while True:
event = self.queue.get()
mask = event.mask
self.lock.acquire()
watch = self.watches.get(event.wd)
self.lock.release()
watch.callback(event)
if mask & ISDIR and mask & CREATE and watch.auto:
self.add(event.path,
watch.mask,
watch.callback,
auto=True,
_parent=watch)
if mask & IGNORED:
self.rem(watch)
def add(self, path, mask=ALL, callback=default, auto=True, _parent=None):
self.lock.acquire()
wd = self.inotify.add_watch(path, mask)
watch = Watch(wd, self, mask, path, callback, auto, _parent)
self.watches[wd] = watch
self.lock.release()
return watch
def rem(self, watch):
self.lock.acquire()
for w in list(self.watches.values()):
if w._parent == watch: self.rem(w)
wd = watch.wd
self.watches.pop(wd)
self.lock.release()
def close(self):
self.inotify.close()
def monitor_file(self):
signal.signal(signal.SIGTERM, self.exit)
while not self.exit_flag:
try:
self.handlers = self.get_handlers()
self._load_pickle()
self.inotify = Inotify()
self.inotify.add_watch(self.pool_path, IN_CREATE | IN_MOVED_FROM | IN_MOVED_TO |
IN_MODIFY | IN_DELETE | IN_DELETE_SELF)
_epoll = select.epoll()
_epoll.register(self.inotify.fileno(), select.EPOLLIN)
self.pre_process()
self._break_flag = True
while not self.exit_flag and self._break_flag:
try:
for fd, epoll_event in _epoll.poll(-1):
if fd == self.inotify.fileno():
for inotify_event in self.inotify.read_events():
self.process_event(inotify_event)
except IOError as e:
if e.errno != EINTR:
raise e
except Exception:
self.log.error(traceback.format_exc())
finally:
self.clean_handlers()
_epoll.close()
dirList = []
remoteList = []
commandList = []
wdDirDict = {}
dirs = False
remotes = False
commands = False
user = False
# checks environment variables for username, system portable
# might be set to a different user later if argument is there
username = getpass.getuser()
inotify = Inotify()
# we don't need to consider name of script as an arg
sys.argv.pop(0)
if len(sys.argv) == 0:
print __doc__
quit()
# sort of a switching system for handling arguments
# it may not be terribly efficient, but it makes adding arguments easy as pie
for arg in sys.argv:
if arg == "-h":
print __doc__
quit()
elif arg == "-c":
def start_multiple(products, user, password, refresh_type="forced", run_once=False, mtimer_options={}):
""" similar to start() but using multiple configurations """
import ZPublisher.Client
from Globals import DevelopmentMode
global ServerError
ServerError = ZPublisher.Client.ServerError
# objects=[ZPublisher.Client.Object(url, username=user, password=password)]
products_by_path = {}
for product in products:
key = product["productpath"]
url = product["CTurl"]
dependencies = product.get("dependencies", None)
if isinstance(dependencies, basestring):
dependencies = [dependencies]
elif isinstance(dependencies, tuple):
dependencies = list(dependencies)
if dependencies:
#
# this dependency list is expected to be a list of urls
# that start like the 'url' variable does. If that's not the case,
# fix them.
baseurl = "/".join(url.split("/")[:-1])
for i in range(len(dependencies)):
if not dependencies[i].startswith(baseurl):
dependencies[i] = baseurl + "/" + dependencies[i]
if url.endswith("/"):
url = url[:-1]
products_by_path[key] = product
objects = [ZPublisher.Client.Object(url, username=user, password=password)]
products_by_path[key]["pre_run_command"] = product.get("pre_run_command")
products_by_path[key]["post_run_command"] = product.get("post_run_command")
if dependencies:
for dep_url in dependencies:
objects.append(ZPublisher.Client.Object(dep_url, username=user, password=password))
# products_by_path[key]['dependencies'] = dependencies # cleaned up
products_by_path[key]["objects"] = objects
if _has_inotify:
extensions_to_scan = ("sql", "py", "zpt", "dtml", "pt", "js")
if refresh_type == "clever" and DevelopmentMode:
# cool, then we don't have refresh when there's a change to
# templates like .dtml and .zpt
if _has_inotify:
extensions_to_scan = ("sql", "py", "js", "css")
else:
mtimer.EXTENSIONS_TO_SCAN = ("sql", "py", "css", "js")
mtimer.FOLDERS_TO_SKIP = ("tests", ".autogenerated", "zpt", "dtml") + mtimer.FOLDERS_TO_SKIP
else:
mtimer.FOLDERS_TO_SKIP = ("tests", ".autogenerated") + mtimer.FOLDERS_TO_SKIP
c = 0
if _has_inotify and not run_once:
notifier = Inotify()
mask = Inotify.IN_MODIFY
for product_path, config in products_by_path.items():
assert os.path.isdir(product_path), "product path does not exist %s" % product_path
_recursiveAddWatch(notifier, product_path, mask)
while 1:
events = notifier.read_events()
for event in events:
event_string = notifier.event_string(event[Inotify.MASK])
path_wid = notifier.file(event[Inotify.WID])
file_wid = event[Inotify.FILE]
file_wid_extension = file_wid.split(".")[-1]
if file_wid_extension in extensions_to_scan:
# the path might be 'Products/Foo/zpt/widgets' when the product
# path is just 'Products/Foo'
for product_path, config in products_by_path.items():
if path_wid.startswith(product_path):
_file_path = os.path.join(path_wid, file_wid).replace("Products/", "")
print _file_path,
t0 = time.time()
r = refresh_product(
config["objects"], config["pre_run_command"], config["post_run_command"]
)
t1 = time.time()
log("%s:%s:%s" % (_file_path, t1 - t0, time.time()))
if r:
print r
else:
print "done"
# print event_string
break
else:
def _getSleeptime():
try:
return float(open("sleeptime.conf").read().split()[0])
except:
return 4.0 # seconds
def _saveSleeptime(sleeptime):
open("sleeptime.conf", "w").write(str(sleeptime))
delta = 0.0
product_info = {}
while 1:
try:
sleeptime = _getSleeptime()
for product_path, config in products_by_path.items():
# t0=time.time()
difference, info = mtimer.run(
product_path, prev_info=product_info.get(product_path, None), **mtimer_options
)
product_info[product_path] = info
if difference == "Different":
t0 = time.time()
refresh_product(config["objects"], config["pre_run_command"], config["post_run_command"]),
t1 = time.time()
print "(%s)" % (seconds2time(c))
log("%s:%s:%s" % (product_path, t1 - t0, time.time()))
delta = 0.0
# print time.time()-t0, "in", product_path
if run_once:
break
# print "Sleep(%s)" % sleeptime
time.sleep(sleeptime + delta) # ; print "sleep", sleeptime + delta
# every time there's. For every loop, increase the sleeptime with
# a tiny delta that can't be bigger than the sleeptime.
# this min() makes sure the extra time never exceeds the original
# sleeptime. This means that if you leave this running for a very
# long time the (sleeptime+delta) === sleeptime*2
delta = min(delta + sleeptime / 200.0, sleeptime)
c += sleeptime
except KeyboardInterrupt:
break
class InotifyPoll(FilePollBase):
def __init__(self, config, section):
super(InotifyPoll, self).__init__(config, section)
self.inotify = None
self._break_flag = True
def on_file_modify(self, event):
return event & IN_MODIFY
def on_file_remove(self, event):
return event & IN_DELETE | event & IN_MOVED_FROM
def on_file_new(self, event):
return event & IN_CREATE | event & IN_MOVED_TO
def on_directory_delete(self, event):
return event & IN_DELETE_SELF
def monitor_file(self):
signal.signal(signal.SIGTERM, self.exit)
while not self.exit_flag:
try:
self.handlers = self.get_handlers()
self._load_pickle()
self.inotify = Inotify()
self.inotify.add_watch(self.pool_path, IN_CREATE | IN_MOVED_FROM | IN_MOVED_TO |
IN_MODIFY | IN_DELETE | IN_DELETE_SELF)
_epoll = select.epoll()
_epoll.register(self.inotify.fileno(), select.EPOLLIN)
self.pre_process()
self._break_flag = True
while not self.exit_flag and self._break_flag:
try:
for fd, epoll_event in _epoll.poll(-1):
if fd == self.inotify.fileno():
for inotify_event in self.inotify.read_events():
self.process_event(inotify_event)
except IOError as e:
if e.errno != EINTR:
raise e
except Exception:
self.log.error(traceback.format_exc())
finally:
self.clean_handlers()
_epoll.close()
def process_event(self, inotify_event):
(watch_path, mask, cookie, filename) = inotify_event
if self.on_file_new(mask):
if self.is_need_monitor(filename):
full_filename = os.path.join(watch_path, filename)
if os.path.isfile(full_filename):
_new_file_handler = open(full_filename, 'r')
self.monitor_dict[full_filename] = _new_file_handler
self.handlers.append(_new_file_handler)
self._update_pickle(_new_file_handler)
elif self.on_file_modify(mask):
_in_process_file_handler = self.monitor_dict.get(os.path.join(watch_path, filename))
if _in_process_file_handler:
_data = self._get_log_data_by_handler(_in_process_file_handler)
if _data:
for _line in _data:
msg = self.reformate(filename=_in_process_file_handler.name, data=_line)
if msg:
self.channel.put(msg)
# 数据完整性由collector来保证
self._update_pickle(_in_process_file_handler)
elif self.on_file_remove(mask):
_delete_file_handler = self.monitor_dict.get(os.path.join(watch_path, filename))
if _delete_file_handler:
self.monitor_dict.pop(os.path.join(watch_path, filename))
_delete_file_handler.close()
self.handlers.remove(_delete_file_handler)
self._reset_pickle([_delete_file_handler.name])
elif self.on_directory_delete(mask):
parent_path = os.path.normpath(os.path.join(self.pool_path, os.pardir))
basename = os.path.basename(os.path.normpath(self.pool_path))
self.inotify.remove_watch(self.pool_path)
self.inotify.add_watch(parent_path, IN_CREATE | IN_MOVED_TO)
self.log.debug('pool_path is deleted, waiting for pool_path to be created')
wait_flag = False if os.path.exists(self.pool_path) else True
while not self.exit_flag and wait_flag:
try:
for self.inotify_event in self.inotify.read_events():
(watch_path, mask, cookie, filename) = self.inotify_event
if filename == basename:
self.inotify.add_watch(self.pool_path, IN_CREATE | IN_MOVED_FROM | IN_MOVED_TO |
IN_MODIFY | IN_DELETE | IN_DELETE_SELF)
wait_flag = False
except IOError as e:
if e.errno != EINTR:
raise e
self._break_flag = False
self.inotify.remove_watch(parent_path)
self.log.debug('stop waiting for pool_path to be created')
def exit(self, *args, **kwargs):
self.log.info('Received sigterm, agent[{}] is going down.'.format(self.name))
self.exit_flag = True
