def reloader():
import gevent_inotifyx as inotifyx
fd = inotifyx.init()
# NOTE: .git/logs/HEAD is the last thing updated after a git pull/merge
inotifyx.add_watch(fd, "../.git/logs/HEAD", inotifyx.IN_MODIFY)
inotifyx.add_watch(fd, ".reloader", inotifyx.IN_MODIFY | inotifyx.IN_ATTRIB)
inotifyx.get_events(fd)
logging.warn("Shutting down due to new update")
SERVER.stop_accepting()
logging.warn("Free Count[%d] (will kill outstanding processes in 120 sec.)" % SERVER.pool.free_count())
SERVER.pool.join(timeout=120)
SERVER.close()
logging.warn("Shut down successful")
def _work(args, jobs):
if args.raven:
import raven
RAVEN = raven.Client(args.raven)
else:
RAVEN = None
import gevent_inotifyx as inotifyx
fd = inotifyx.init()
# NOTE: .git/logs/HEAD is the last thing updated after a git pull/merge
inotifyx.add_watch(fd, '../.git/logs/HEAD', inotifyx.IN_MODIFY)
inotifyx.add_watch(fd, '.reloader',
inotifyx.IN_MODIFY | inotifyx.IN_ATTRIB)
db = THRIFT_CONSTRUCTOR()
while 1:
try:
work = jobs.get_work(args.queues, timeout=5)
if work:
jobs.add_work(True, 'old' + work[0], **work[1])
job_worker(db=db, **work[1])
if inotifyx.get_events(fd, 0):
print('Shutting down due to new update')
break
except:
if RAVEN:
RAVEN.captureException()
raise
def reloader():
import gevent_inotifyx as inotifyx
fd = inotifyx.init()
# NOTE: .git/logs/HEAD is the last thing updated after a git pull/merge
inotifyx.add_watch(fd, '../.git/logs/HEAD', inotifyx.IN_MODIFY)
inotifyx.add_watch(fd, '.reloader',
inotifyx.IN_MODIFY | inotifyx.IN_ATTRIB)
inotifyx.get_events(fd)
logging.warn('Shutting down due to new update')
SERVER.stop_accepting()
logging.warn(
'Free Count[%d] (will kill outstanding processes in 120 sec.)' %
SERVER.pool.free_count())
SERVER.pool.join(timeout=120)
SERVER.close()
logging.warn('Shut down successful')
def _work(args, jobs):
if args.raven:
import raven
RAVEN = raven.Client(args.raven)
else:
RAVEN = None
import gevent_inotifyx as inotifyx
fd = inotifyx.init()
# NOTE: .git/logs/HEAD is the last thing updated after a git pull/merge
inotifyx.add_watch(fd, '../.git/logs/HEAD', inotifyx.IN_MODIFY)
inotifyx.add_watch(fd, '.reloader', inotifyx.IN_MODIFY | inotifyx.IN_ATTRIB)
db = THRIFT_CONSTRUCTOR()
while 1:
try:
work = jobs.get_work(args.queues, timeout=5)
if work:
jobs.add_work(True, 'old' + work[0], **work[1])
job_worker(db=db, **work[1])
if inotifyx.get_events(fd, 0):
print('Shutting down due to new update')
break
except:
if RAVEN:
RAVEN.captureException()
raise
def event_producer(self):
while True:
events = inotify.get_events(self.fd)
for event in events:
self.q.put(event)
if event.name == pth.basename(self.path):
return
def test_get_events(self):
wd = inotify.add_watch(self.fd, self.testdir, inotify.IN_CREATE)
self._create_file('foo')
try:
events = inotify.get_events(self.fd)
self.assertEqual(len(events), 1)
self.assertEqual(events[0].mask, inotify.IN_CREATE)
self.assertEqual(events[0].name, 'foo')
finally:
os.unlink('foo')
def start(self):
self.fd = inotify.init()
self.ddirs = {}
for startpath in self.paths:
for ddir in j.system.fs.listDirsInDir(startpath, True):
ddirname = j.system.fs.getDirName(ddir + "/", True)
# print ddirname
ok = True
if ddirname.find(".") == 0 or ddir.find(".cache") != -1:
ok = False
if ok:
try:
wd = inotify.add_watch(
self.fd, ddir,
inotify.IN_CREATE | inotify.IN_DELETE
| inotify.IN_DELETE_SELF | inotify.IN_MODIFY
| inotify.IN_MOVE_SELF | inotify.IN_MOVED_FROM
| inotify.IN_MOVED_TO)
# print "add notify %s" % ddir
self.ddirs[wd] = ddir
except:
print "ERROR COULD NOT ADD notify %s" % ddir
print "started"
while True:
events = inotify.get_events(self.fd)
for event in events:
# j.put(event)
if event.name != None:
path = self.ddirs[event.wd] + "/" + event.name
else:
path = self.ddirs[event.wd]
print "restarted for %s" % path
self.start()
if path.find(".goutputstream") != -1:
continue
print "received event:", event.get_mask_description(), path
if j.system.fs.isFile(path):
if j.system.fs.getFileExtension(path).lower() in [
"wiki", "py", "spec"
]:
try:
self.contentmgr.notifychange(path)
print "notified %s" % path
except:
print "could not notify change for %s" % path
elif j.system.fs.isDir(path):
print "restarted for dir %s" % path
self.start()
def start(self):
self.fd = inotify.init()
self.ddirs = {}
for startpath in self.paths:
for ddir in j.system.fs.listDirsInDir(startpath, True):
ddirname = j.system.fs.getDirName(ddir + "/", True)
# print ddirname
ok = True
if ddirname.find(".") == 0 or ddir.find(".cache") != -1:
ok = False
if ok:
try:
wd = inotify.add_watch(self.fd, ddir, inotify.IN_CREATE | inotify.IN_DELETE
| inotify.IN_DELETE_SELF | inotify.IN_MODIFY
| inotify.IN_MOVE_SELF | inotify.IN_MOVED_FROM
| inotify.IN_MOVED_TO)
# print "add notify %s" % ddir
self.ddirs[wd] = ddir
except:
print "ERROR COULD NOT ADD notify %s" % ddir
print "started"
while True:
events = inotify.get_events(self.fd)
for event in events:
# j.put(event)
if event.name != None:
path = self.ddirs[event.wd] + "/" + event.name
else:
path = self.ddirs[event.wd]
print "restarted for %s" % path
self.start()
if path.find(".goutputstream") != -1:
continue
print "received event:", event.get_mask_description(), path
if j.system.fs.isFile(path):
if j.system.fs.getFileExtension(path).lower() in ["wiki", "py", "spec"]:
try:
self.contentmgr.notifychange(path)
print "notified %s" % path
except:
print "could not notify change for %s" % path
elif j.system.fs.isDir(path):
print "restarted for dir %s" % path
self.start()
def watch(self):
fd = inotifyx.init()
inotifyx.add_watch(fd, self.pickup_dir, inotifyx.IN_CLOSE_WRITE | inotifyx.IN_MOVE)
while self.keep_running:
try:
events = inotifyx.get_events(fd, 1.0)
for event in events:
self.pickup_event_processor.process(event)
sleep(0.1)
except KeyboardInterrupt:
self.keep_running = False
os.close(fd)
def watch(self):
fd = inotifyx.init()
inotifyx.add_watch(fd, self.pickup_dir, inotifyx.IN_CLOSE_WRITE | inotifyx.IN_MOVE)
while self.keep_running:
try:
events = inotifyx.get_events(fd, 1.0)
for event in events:
self.pickup_event_processor.process(event)
sleep(0.1)
except KeyboardInterrupt:
self.keep_running = False
os.close(fd)
def monitorDirectory(self):
'''Monitors the given directory for changes.'''
fd = inotify.init()
wb = inotify.add_watch(fd, self.directory,
inotify.IN_CLOSE_WRITE + inotify.IN_DELETE)
self.__rules = self.readDirectory()
while True:
self.wait.clear()
events = inotify.get_events(fd)
current_rules = self.readDirectory()
if cmp(current_rules, self.__rules) != 0:
self.__rules = current_rules
self.wait.set()
def inotify_event(file_path, rules):
fd = inotify.init()
(config_path, config_name) = os.path.split(file_path)
if not config_path:
config_path = os.getcwd()
file_name = config_path + '/' + config_name
wd = inotify.add_watch(
fd, config_path, inotify.IN_MODIFY | inotify.IN_CREATE)
rules.update(reload_config(file_name))
while True:
events = inotify.get_events(fd)
update = False
for event in events:
if event.name == config_name:
update = True
if update:
rules.update(reload_config(file_name))
def wait_for_plugin_changes(self):
import gevent_inotifyx as inotify
fd = inotify.init()
inotify.add_watch(fd, 'rowboat/plugins/', inotify.IN_MODIFY)
while True:
events = inotify.get_events(fd)
for event in events:
# Can't reload core.py sadly
if event.name == 'core.py':
continue
plugin_name = '{}Plugin'.format(
event.name.split('.', 1)[0].title())
if plugin_name in self.bot.plugins:
self.log.info('Detected change in %s, reloading...',
plugin_name)
self.bot.plugins[plugin_name].reload()
class Pickup(object):
def __init__(self, pickup_event_processor=None):
self._pickup_dirs = []
self.pickup_event_processor = pickup_event_processor or PickupEventProcessor(
)
self.keep_running = True
# A dict of inotify watch descriptors to directory names.
# Needed because inotify returns only the name of a file in events.
self.wd_to_name = {}
@property
def pickup_dir(self):
return self._pickup_dirs
@pickup_dir.setter
def pickup_dir(self, value):
self._pickup_dirs = [value] if not isinstance(value, list) else value
def watch(self):
fd = inotifyx.init()
for name in self.pickup_dir:
try:
self.wd_to_name[inotifyx.add_watch(
fd, name,
inotifyx.IN_CLOSE_WRITE | inotifyx.IN_MOVE)] = name
except IOError, e:
logger.warn('Caught IOError `%s`, name:`%s`', format_exc(e),
name)
while self.keep_running:
try:
events = inotifyx.get_events(fd, 1.0)
for event in events:
self.pickup_event_processor.process(
os.path.join(self.wd_to_name[event.wd], event.name),
event)
sleep(0.1)
except KeyboardInterrupt:
self.keep_running = False
os.close(fd)
def _run(self):
if not use_inotify:
log.warn("gevent_inotifyx not loaded; not using inotify")
return
fd = inotify.init()
wd = inotify.add_watch(fd, self.opts.spool_dir,
inotify.IN_CLOSE_WRITE | inotify.IN_MOVED_TO)
while True:
events = inotify.get_events(fd)
for event in events:
path = os.path.sep.join([self.opts.spool_dir,
event.name])
# Filter out inotify events generated for files that
# have been already unlinked from the filesystem
# (IN_EXCL_UNLINK emulation)
if os.path.exists(path):
self.on_find(path)
def wait_for_plugin_changes(self):
import gevent_inotifyx as inotify
fd = inotify.init()
inotify.add_watch(fd, 'rowboat/plugins/', inotify.IN_MODIFY)
while True:
events = inotify.get_events(fd)
for event in events:
# Can't reload core.py sadly
if event.name.startswith('core.py'):
continue
plugin_name = '{}Plugin'.format(event.name.split('.', 1)[0].title())
plugin = next((v for k, v in self.bot.plugins.items() if k.lower() == plugin_name.lower()), None)
if plugin:
self.log.info('Detected change in %s, reloading...', plugin_name)
try:
plugin.reload()
except Exception:
self.log.exception('Failed to reload: ')
def event_producer(fd, server):
while True:
restart = False
events = inotify.get_events(fd, None)
for event in events:
ignore = False
if call(['git', 'check-ignore', event.name]) == 0 or fnmatch.fnmatch(event.name, '.git/*'):
ignore = True
break
if not ignore:
print "File changed:", event.name, event.get_mask_description()
restart = True
if restart:
print 'Restarting %s ...\n' % sys.argv
server.stop(timeout=1)
server.close()
print 'Waiting...'
time.sleep(3)
print 'Restart.'
os.execvp(sys.argv[0], sys.argv)
def do(self):
events = inotify.get_events(self._inotify_fd)
for ev in events:
filename = os.path.join(self.watch_path, ev.name)
file_ext = os.path.splitext(filename)[1].lower()
if file_ext not in self._file_exts:
logger.debug('Invalid file[%s] extension[%s] %s' %
(filename, file_ext, self._file_exts))
continue
with open(filename, 'r') as f:
job = {
'source': 'directory',
'type': 'image', # or video
'filename': os.path.split(filename)[1],
'ctime': datetime.fromtimestamp(os.path.getmtime(filename)),
'content': f.read()
}
logger.info('Get a file: %s' % filename)
self.send(job)
os.remove(filename)
def watchGreenlet(self,fd):
gevent_inotifyx.get_events(fd)
self._logger.warn("shutting down due to change in %s" % (gConfig.shutdownFile))
gConfig.kill()
def waitForChanges(self):
inotify.get_events(self.fd)
return self.readDirectory()
def run(self):
try:
for path in self.callback_config:
if not os.path.exists(path):
raise Exception('Path does not exist `{}`'.format(path))
self.wd_to_path[infx.add_watch(
self.infx_fd, path,
infx.IN_CLOSE_WRITE | infx.IN_MOVE)] = path
while self.keep_running:
try:
events = infx.get_events(self.infx_fd, 1.0)
for event in events:
pe = PickupEvent()
try:
pe.base_dir = self.wd_to_path[event.wd]
config = self.callback_config[pe.base_dir]
if not self.should_pick_up(event.name,
config.patterns):
continue
pe.file_name = event.name
pe.stanza = config.stanza
pe.full_path = os.path.join(
pe.base_dir, event.name)
# If we are deploying services, the path is different than for other resources
if config.is_service_hot_deploy:
spawn_greenlet(hot_deploy, self.server,
pe.file_name, pe.full_path,
config.delete_after_pick_up)
continue
if config.read_on_pickup:
f = open(pe.full_path, 'rb')
pe.raw_data = f.read()
pe.has_raw_data = True
f.close()
if config.parse_on_pickup:
try:
pe.data = self.get_parser(
config.parse_with)(pe.raw_data)
pe.has_data = True
except Exception, e:
pe.parse_error = e
else:
pe.data = pe.raw_data
spawn_greenlet(self.invoke_callbacks, pe,
config.services, config.topics)
self.post_handle(pe.full_path, config)
except Exception, e:
logger.warn(format_exc(e))
except KeyboardInterrupt:
self.keep_running = False
def watchGreenlet(self, fd):
gevent_inotifyx.get_events(fd)
self._logger.warn("shutting down due to change in %s" %
(gConfig.shutdownFile))
gConfig.kill()
def event_producer(fd, q):
while True:
events = inotify.get_events(fd)
for event in events:
q.put(event)
def waitForChanges(self, grace_period=5):
events = inotify.get_events(self.fd)
return self.readDirectory()
def notification_loop(self, interval):
while True:
gevent.sleep(interval)
inotify.get_events(self.notifier)
self.read()
