def _work(args, jobs):
if args.raven:
import raven
RAVEN = raven.Client(args.raven)
else:
RAVEN = None
import gevent_inotifyx as inotifyx
fd = inotifyx.init()
# NOTE: .git/logs/HEAD is the last thing updated after a git pull/merge
inotifyx.add_watch(fd, '../.git/logs/HEAD', inotifyx.IN_MODIFY)
inotifyx.add_watch(fd, '.reloader', inotifyx.IN_MODIFY | inotifyx.IN_ATTRIB)
db = THRIFT_CONSTRUCTOR()
while 1:
try:
work = jobs.get_work(args.queues, timeout=5)
if work:
jobs.add_work(True, 'old' + work[0], **work[1])
job_worker(db=db, **work[1])
if inotifyx.get_events(fd, 0):
print('Shutting down due to new update')
break
except:
if RAVEN:
RAVEN.captureException()
raise
def main():
if len(sys.argv) < 2:
print 'Usage: %s CONFIG_FILE' % sys.argv[0]
sys.exit(1)
signal.signal(signal.SIGINT, sigint_handler)
from iris_api.api import get_api, load_config_file
config = load_config_file(sys.argv[1])
app = get_api(config)
server = config['server']
print 'LISTENING: %(host)s:%(port)d' % server
server = WSGIServer((server['host'], server['port']), app)
if HAS_INOTIFY:
fd = inotify.init()
for dirname, subfolders, _ in os.walk('.'):
if '.git' in subfolders:
subfolders.remove('.git')
inotify.add_watch(fd, dirname, inotify.IN_MODIFY)
gevent.spawn(event_producer, fd, server)
else:
print 'Missing inotify, disable watch support.'
server.serve_forever()
def _work(args, jobs):
if args.raven:
import raven
RAVEN = raven.Client(args.raven)
else:
RAVEN = None
import gevent_inotifyx as inotifyx
fd = inotifyx.init()
# NOTE: .git/logs/HEAD is the last thing updated after a git pull/merge
inotifyx.add_watch(fd, '../.git/logs/HEAD', inotifyx.IN_MODIFY)
inotifyx.add_watch(fd, '.reloader',
inotifyx.IN_MODIFY | inotifyx.IN_ATTRIB)
db = THRIFT_CONSTRUCTOR()
while 1:
try:
work = jobs.get_work(args.queues, timeout=5)
if work:
jobs.add_work(True, 'old' + work[0], **work[1])
job_worker(db=db, **work[1])
if inotifyx.get_events(fd, 0):
print('Shutting down due to new update')
break
except:
if RAVEN:
RAVEN.captureException()
raise
def __init__(self, config_manager, filename, interval=3):
# TODO: check file existance and read-permission.
self.filename = filename
self.interval = interval
self.config_manager = config_manager
self.notifier = inotify.init()
inotify.add_watch(self.notifier, filename, inotify.IN_CLOSE_WRITE)
def wait(self):
try:
inotify.add_watch(self.fd, self.parent, inotify.IN_CREATE)
eventer = gevent.spawn(self.event_producer)
while True:
event = self.q.get()
if event.name == pth.basename(self.path):
break
finally:
gevent.wait((eventer, ))
os.close(self.fd)
return True
def reloader():
import gevent_inotifyx as inotifyx
fd = inotifyx.init()
# NOTE: .git/logs/HEAD is the last thing updated after a git pull/merge
inotifyx.add_watch(fd, "../.git/logs/HEAD", inotifyx.IN_MODIFY)
inotifyx.add_watch(fd, ".reloader", inotifyx.IN_MODIFY | inotifyx.IN_ATTRIB)
inotifyx.get_events(fd)
logging.warn("Shutting down due to new update")
SERVER.stop_accepting()
logging.warn("Free Count[%d] (will kill outstanding processes in 120 sec.)" % SERVER.pool.free_count())
SERVER.pool.join(timeout=120)
SERVER.close()
logging.warn("Shut down successful")
def watch(self):
fd = inotifyx.init()
inotifyx.add_watch(fd, self.pickup_dir, inotifyx.IN_CLOSE_WRITE | inotifyx.IN_MOVE)
while self.keep_running:
try:
events = inotifyx.get_events(fd, 1.0)
for event in events:
self.pickup_event_processor.process(event)
sleep(0.1)
except KeyboardInterrupt:
self.keep_running = False
os.close(fd)
def watch(self):
fd = inotifyx.init()
inotifyx.add_watch(fd, self.pickup_dir, inotifyx.IN_CLOSE_WRITE | inotifyx.IN_MOVE)
while self.keep_running:
try:
events = inotifyx.get_events(fd, 1.0)
for event in events:
self.pickup_event_processor.process(event)
sleep(0.1)
except KeyboardInterrupt:
self.keep_running = False
os.close(fd)
def reloader():
import gevent_inotifyx as inotifyx
fd = inotifyx.init()
# NOTE: .git/logs/HEAD is the last thing updated after a git pull/merge
inotifyx.add_watch(fd, '../.git/logs/HEAD', inotifyx.IN_MODIFY)
inotifyx.add_watch(fd, '.reloader',
inotifyx.IN_MODIFY | inotifyx.IN_ATTRIB)
inotifyx.get_events(fd)
logging.warn('Shutting down due to new update')
SERVER.stop_accepting()
logging.warn(
'Free Count[%d] (will kill outstanding processes in 120 sec.)' %
SERVER.pool.free_count())
SERVER.pool.join(timeout=120)
SERVER.close()
logging.warn('Shut down successful')
def watch(self):
fd = inotifyx.init()
for name in self.pickup_dir:
try:
self.wd_to_name[inotifyx.add_watch(fd, name, inotifyx.IN_CLOSE_WRITE | inotifyx.IN_MOVE)] = name
except IOError, e:
logger.warn('Caught IOError `%s`, name:`%s`', format_exc(e), name)
def __init__(self, logger, directory="rules/"):
self.logger = logger
self.directory = directory
if not os.access(self.directory, os.R_OK):
raise Exception("Directory '%s' is not readable. Please verify." % (self.directory))
self.fd = inotify.init()
self.wb = inotify.add_watch(self.fd, self.directory, inotify.IN_CLOSE_WRITE + inotify.IN_CREATE + inotify.IN_DELETE + inotify.IN_MODIFY + inotify.IN_MOVE)
def wait_for_plugin_changes(self):
import gevent_inotifyx as inotify
fd = inotify.init()
inotify.add_watch(fd, 'rowboat/plugins/', inotify.IN_MODIFY)
while True:
events = inotify.get_events(fd)
for event in events:
# Can't reload core.py sadly
if event.name == 'core.py':
continue
plugin_name = '{}Plugin'.format(
event.name.split('.', 1)[0].title())
if plugin_name in self.bot.plugins:
self.log.info('Detected change in %s, reloading...',
plugin_name)
self.bot.plugins[plugin_name].reload()
def test_get_events(self):
wd = inotify.add_watch(self.fd, self.testdir, inotify.IN_CREATE)
self._create_file('foo')
try:
events = inotify.get_events(self.fd)
self.assertEqual(len(events), 1)
self.assertEqual(events[0].mask, inotify.IN_CREATE)
self.assertEqual(events[0].name, 'foo')
finally:
os.unlink('foo')
def watch(self):
fd = inotifyx.init()
for name in self.pickup_dir:
try:
self.wd_to_name[inotifyx.add_watch(
fd, name,
inotifyx.IN_CLOSE_WRITE | inotifyx.IN_MOVE)] = name
except IOError, e:
logger.warn('Caught IOError `%s`, name:`%s`', format_exc(e),
name)
def wait_for_plugin_changes(self):
import gevent_inotifyx as inotify
fd = inotify.init()
inotify.add_watch(fd, 'rowboat/plugins/', inotify.IN_MODIFY)
while True:
events = inotify.get_events(fd)
for event in events:
# Can't reload core.py sadly
if event.name.startswith('core.py'):
continue
plugin_name = '{}Plugin'.format(event.name.split('.', 1)[0].title())
plugin = next((v for k, v in self.bot.plugins.items() if k.lower() == plugin_name.lower()), None)
if plugin:
self.log.info('Detected change in %s, reloading...', plugin_name)
try:
plugin.reload()
except Exception:
self.log.exception('Failed to reload: ')
def start(self):
self.fd = inotify.init()
self.ddirs = {}
for startpath in self.paths:
for ddir in j.system.fs.listDirsInDir(startpath, True):
ddirname = j.system.fs.getDirName(ddir + "/", True)
# print ddirname
ok = True
if ddirname.find(".") == 0 or ddir.find(".cache") != -1:
ok = False
if ok:
try:
wd = inotify.add_watch(
self.fd, ddir,
inotify.IN_CREATE | inotify.IN_DELETE
| inotify.IN_DELETE_SELF | inotify.IN_MODIFY
| inotify.IN_MOVE_SELF | inotify.IN_MOVED_FROM
| inotify.IN_MOVED_TO)
# print "add notify %s" % ddir
self.ddirs[wd] = ddir
except:
print "ERROR COULD NOT ADD notify %s" % ddir
print "started"
while True:
events = inotify.get_events(self.fd)
for event in events:
# j.put(event)
if event.name != None:
path = self.ddirs[event.wd] + "/" + event.name
else:
path = self.ddirs[event.wd]
print "restarted for %s" % path
self.start()
if path.find(".goutputstream") != -1:
continue
print "received event:", event.get_mask_description(), path
if j.system.fs.isFile(path):
if j.system.fs.getFileExtension(path).lower() in [
"wiki", "py", "spec"
]:
try:
self.contentmgr.notifychange(path)
print "notified %s" % path
except:
print "could not notify change for %s" % path
elif j.system.fs.isDir(path):
print "restarted for dir %s" % path
self.start()
def start(self):
self.fd = inotify.init()
self.ddirs = {}
for startpath in self.paths:
for ddir in j.system.fs.listDirsInDir(startpath, True):
ddirname = j.system.fs.getDirName(ddir + "/", True)
# print ddirname
ok = True
if ddirname.find(".") == 0 or ddir.find(".cache") != -1:
ok = False
if ok:
try:
wd = inotify.add_watch(self.fd, ddir, inotify.IN_CREATE | inotify.IN_DELETE
| inotify.IN_DELETE_SELF | inotify.IN_MODIFY
| inotify.IN_MOVE_SELF | inotify.IN_MOVED_FROM
| inotify.IN_MOVED_TO)
# print "add notify %s" % ddir
self.ddirs[wd] = ddir
except:
print "ERROR COULD NOT ADD notify %s" % ddir
print "started"
while True:
events = inotify.get_events(self.fd)
for event in events:
# j.put(event)
if event.name != None:
path = self.ddirs[event.wd] + "/" + event.name
else:
path = self.ddirs[event.wd]
print "restarted for %s" % path
self.start()
if path.find(".goutputstream") != -1:
continue
print "received event:", event.get_mask_description(), path
if j.system.fs.isFile(path):
if j.system.fs.getFileExtension(path).lower() in ["wiki", "py", "spec"]:
try:
self.contentmgr.notifychange(path)
print "notified %s" % path
except:
print "could not notify change for %s" % path
elif j.system.fs.isDir(path):
print "restarted for dir %s" % path
self.start()
def __init__(self, logger, directory="rules/"):
self.logging = logger
self.directory = directory
if not os.access(self.directory, os.R_OK):
raise Exception("Directory '%s' is not readable. Please verify." %
(self.directory))
self.fd = inotify.init()
self.wb = inotify.add_watch(
self.fd, self.directory,
inotify.IN_CLOSE_WRITE + inotify.IN_CREATE + inotify.IN_DELETE +
inotify.IN_MODIFY + inotify.IN_MOVE)
self.logging.info("Monitoring directory '%s' for changes" %
(directory))
def test_get_events_async(self):
wd = inotify.add_watch(self.fd, self.testdir, inotify.IN_CREATE)
task1 = gevent.spawn(inotify.get_events, self.fd, 10)
task2 = gevent.spawn(self._create_file, 'bar')
try:
# in the synchronous case we would block here for 10s and the
# return value in task1.value would be empty
task1.join()
task2.join()
events = task1.value
self.assertEqual(len(events), 1)
self.assertEqual(events[0].mask, inotify.IN_CREATE)
self.assertEqual(events[0].name, 'bar')
finally:
os.unlink('bar')
def monitorDirectory(self):
'''Monitors the given directory for changes.'''
fd = inotify.init()
wb = inotify.add_watch(fd, self.directory,
inotify.IN_CLOSE_WRITE + inotify.IN_DELETE)
self.__rules = self.readDirectory()
while True:
self.wait.clear()
events = inotify.get_events(fd)
current_rules = self.readDirectory()
if cmp(current_rules, self.__rules) != 0:
self.__rules = current_rules
self.wait.set()
def run(self):
glt = ezRPGreenlet.EzReverseProxyGreenlet(self._logger, self._sfh)
self._logger.info("starting greenlet for thrift service...")
clientGreenlet = gevent.spawn(glt.clientServiceGreenlet)
self._logger.info("started")
self._logger.info("starting greenlet to monitor zookeeper...")
kzGreenlet = gevent.spawn(glt.kzMonitorGreenlet)
self._logger.info("started")
self._logger.info(
"starting greenlet to write out nginx configuration changes...")
cfgProcessorGreenlet = gevent.spawn(
glt.configurationChangeQueueGreenlet)
self._logger.info("started")
self._logger.info("starting greenlet to monitor for shutdown...")
fd = gevent_inotifyx.init()
wd = gevent_inotifyx.add_watch(fd, gConfig.shutdownFile,
gevent_inotifyx.IN_DELETE)
wGreenlet = gevent.spawn(glt.watchGreenlet, fd)
self._logger.info("started")
gConfig.addGreenlets(clientGreenlet, kzGreenlet, cfgProcessorGreenlet,
wGreenlet)
gevent.joinall([clientGreenlet])
self._logger.warn("joined thrift service greenlet")
gevent.joinall([kzGreenlet])
self._logger.warn("joined zookeeper monitoring greenlet")
gConfig.run = False
while not gConfig.configurationChangeQueue.empty():
print "queue not empty"
gConfig.configurationChangeQueue.get()
print "got"
gConfig.configurationChangeQueue.task_done()
print "joining conf queue"
gConfig.configurationChangeQueue.join()
self._logger.warn("joined configuration queue")
gevent.joinall([cfgProcessorGreenlet])
self._logger.warn("joined configuration change greenlet")
gConfig.wGreenlet.join()
self._logger.warn("joined shutdown monitor greenlet")
ServiceDiscoveryClient(gConfig.zk).unregister_endpoint(
gConfig.appName, EzFrontendServiceName, gConfig.internal_hostname,
gConfig.thriftPort)
self._logger.warn("unregistered from discovery service")
def inotify_event(file_path, rules):
fd = inotify.init()
(config_path, config_name) = os.path.split(file_path)
if not config_path:
config_path = os.getcwd()
file_name = config_path + '/' + config_name
wd = inotify.add_watch(
fd, config_path, inotify.IN_MODIFY | inotify.IN_CREATE)
rules.update(reload_config(file_name))
while True:
events = inotify.get_events(fd)
update = False
for event in events:
if event.name == config_name:
update = True
if update:
rules.update(reload_config(file_name))
def run(self):
glt = ezRPGreenlet.EzReverseProxyGreenlet(self._logger, self._sfh)
self._logger.info("starting greenlet for thrift service...")
clientGreenlet = gevent.spawn(glt.clientServiceGreenlet)
self._logger.info("started")
self._logger.info("starting greenlet to monitor zookeeper...")
kzGreenlet = gevent.spawn(glt.kzMonitorGreenlet)
self._logger.info("started")
self._logger.info("starting greenlet to write out nginx configuration changes...")
cfgProcessorGreenlet = gevent.spawn(glt.configurationChangeQueueGreenlet)
self._logger.info("started")
self._logger.info("starting greenlet to monitor for shutdown...")
fd = gevent_inotifyx.init()
wd = gevent_inotifyx.add_watch(fd, gConfig.shutdownFile, gevent_inotifyx.IN_DELETE)
wGreenlet = gevent.spawn(glt.watchGreenlet, fd)
self._logger.info("started")
gConfig.addGreenlets(clientGreenlet, kzGreenlet, cfgProcessorGreenlet, wGreenlet)
gevent.joinall([clientGreenlet])
self._logger.warn("joined thrift service greenlet")
gevent.joinall([kzGreenlet])
self._logger.warn("joined zookeeper monitoring greenlet")
gConfig.run = False
while not gConfig.configurationChangeQueue.empty():
print "queue not empty"
gConfig.configurationChangeQueue.get()
print "got"
gConfig.configurationChangeQueue.task_done()
print "joining conf queue"
gConfig.configurationChangeQueue.join()
self._logger.warn("joined configuration queue")
gevent.joinall([cfgProcessorGreenlet])
self._logger.warn("joined configuration change greenlet")
gConfig.wGreenlet.join()
self._logger.warn("joined shutdown monitor greenlet")
ServiceDiscoveryClient(gConfig.zk).unregister_endpoint(gConfig.appName,
EzFrontendServiceName,
gConfig.internal_hostname,
gConfig.thriftPort)
self._logger.warn("unregistered from discovery service")
def _run(self):
if not use_inotify:
log.warn("gevent_inotifyx not loaded; not using inotify")
return
fd = inotify.init()
wd = inotify.add_watch(fd, self.opts.spool_dir,
inotify.IN_CLOSE_WRITE | inotify.IN_MOVED_TO)
while True:
events = inotify.get_events(fd)
for event in events:
path = os.path.sep.join([self.opts.spool_dir,
event.name])
# Filter out inotify events generated for files that
# have been already unlinked from the filesystem
# (IN_EXCL_UNLINK emulation)
if os.path.exists(path):
self.on_find(path)
def run(self):
try:
for path in self.callback_config:
if not os.path.exists(path):
raise Exception('Path does not exist `{}`'.format(path))
self.wd_to_path[infx.add_watch(
self.infx_fd, path,
infx.IN_CLOSE_WRITE | infx.IN_MOVE)] = path
while self.keep_running:
try:
events = infx.get_events(self.infx_fd, 1.0)
for event in events:
pe = PickupEvent()
try:
pe.base_dir = self.wd_to_path[event.wd]
config = self.callback_config[pe.base_dir]
if not self.should_pick_up(event.name,
config.patterns):
continue
pe.file_name = event.name
pe.stanza = config.stanza
pe.full_path = os.path.join(
pe.base_dir, event.name)
# If we are deploying services, the path is different than for other resources
if config.is_service_hot_deploy:
spawn_greenlet(hot_deploy, self.server,
pe.file_name, pe.full_path,
config.delete_after_pick_up)
continue
if config.read_on_pickup:
f = open(pe.full_path, 'rb')
pe.raw_data = f.read()
pe.has_raw_data = True
f.close()
if config.parse_on_pickup:
try:
pe.data = self.get_parser(
config.parse_with)(pe.raw_data)
pe.has_data = True
except Exception, e:
pe.parse_error = e
else:
pe.data = pe.raw_data
spawn_greenlet(self.invoke_callbacks, pe,
config.services, config.topics)
self.post_handle(pe.full_path, config)
except Exception, e:
logger.warn(format_exc(e))
except KeyboardInterrupt:
self.keep_running = False
elif o in ("-r", "--run"):
if a == 'live':
STREAM_SERVER = '216.18.184.22:1935'
STREAM_USER = '******'
PIC_PATH = '/srv/LIVE_model_imgs_ramfs/'
ONLINE_MODELS_URL = 'http://www.seeme.com/onlinemodels'
PID_FILE = '/tmp/LIVE_ffmpeg_websocket_server.pid'
PORT = 8023
FLASH_PORT = 10843
else:
assert False, "unhandled option"
q = Queue()
q.maxsize = 1000
fd = inotify.init()
inotify.add_watch(fd, PIC_PATH, inotify.IN_CREATE)
stream_dumper = StreamDumper()
gevent.spawn(event_producer, fd, q)
signal.signal(signal.SIGTERM, exit_cleanup)
signal.signal(signal.SIGINT, exit_cleanup)
signal.signal(signal.SIGQUIT, exit_cleanup)
atexit.register(exit_cleanup)
if DEBUG: print("writing pid file: %s" % PID_FILE)
with open(PID_FILE, 'w') as f:
f.write(str(os.getpid()))
# start ffmpeg processes for online models
r = requests.get(ONLINE_MODELS_URL)
if r.status_code == 200:
def prepare(self):
super(FileWatcher, self).prepare()
self._inotify_fd = inotify.init()
self._wd = inotify.add_watch(self._inotify_fd, self.watch_path,
inotify.IN_CLOSE_WRITE)
logger.info("Listening path: %s" % self.watch_path)