Exemple #1
0
 def _work(args, jobs):
     if args.raven:
         import raven
         RAVEN = raven.Client(args.raven)
     else:
         RAVEN = None
     import gevent_inotifyx as inotifyx
     fd = inotifyx.init()
     # NOTE: .git/logs/HEAD is the last thing updated after a git pull/merge
     inotifyx.add_watch(fd, '../.git/logs/HEAD', inotifyx.IN_MODIFY)
     inotifyx.add_watch(fd, '.reloader', inotifyx.IN_MODIFY | inotifyx.IN_ATTRIB)
     db = THRIFT_CONSTRUCTOR()
     while 1:
         try:
             work = jobs.get_work(args.queues, timeout=5)
             if work:
                 jobs.add_work(True, 'old' + work[0], **work[1])
                 job_worker(db=db, **work[1])
             if inotifyx.get_events(fd, 0):
                 print('Shutting down due to new update')
                 break
         except:
             if RAVEN:
                 RAVEN.captureException()
             raise
Exemple #2
0
def main():
    if len(sys.argv) < 2:
        print 'Usage: %s CONFIG_FILE' % sys.argv[0]
        sys.exit(1)

    signal.signal(signal.SIGINT, sigint_handler)

    from iris_api.api import get_api, load_config_file

    config = load_config_file(sys.argv[1])
    app = get_api(config)

    server = config['server']
    print 'LISTENING: %(host)s:%(port)d' % server
    server = WSGIServer((server['host'], server['port']), app)

    if HAS_INOTIFY:
        fd = inotify.init()

        for dirname, subfolders, _ in os.walk('.'):
            if '.git' in subfolders:
                subfolders.remove('.git')
            inotify.add_watch(fd, dirname, inotify.IN_MODIFY)

        gevent.spawn(event_producer, fd, server)
    else:
        print 'Missing inotify, disable watch support.'
    server.serve_forever()
Exemple #3
0
 def _work(args, jobs):
     if args.raven:
         import raven
         RAVEN = raven.Client(args.raven)
     else:
         RAVEN = None
     import gevent_inotifyx as inotifyx
     fd = inotifyx.init()
     # NOTE: .git/logs/HEAD is the last thing updated after a git pull/merge
     inotifyx.add_watch(fd, '../.git/logs/HEAD', inotifyx.IN_MODIFY)
     inotifyx.add_watch(fd, '.reloader',
                        inotifyx.IN_MODIFY | inotifyx.IN_ATTRIB)
     db = THRIFT_CONSTRUCTOR()
     while 1:
         try:
             work = jobs.get_work(args.queues, timeout=5)
             if work:
                 jobs.add_work(True, 'old' + work[0], **work[1])
                 job_worker(db=db, **work[1])
             if inotifyx.get_events(fd, 0):
                 print('Shutting down due to new update')
                 break
         except:
             if RAVEN:
                 RAVEN.captureException()
             raise
Exemple #4
0
    def __init__(self, config_manager, filename, interval=3):
        # TODO: check file existance and read-permission.
        self.filename = filename
        self.interval = interval

        self.config_manager = config_manager
        self.notifier = inotify.init()
        inotify.add_watch(self.notifier, filename, inotify.IN_CLOSE_WRITE)
Exemple #5
0
    def wait(self):
        try:
            inotify.add_watch(self.fd, self.parent, inotify.IN_CREATE)
            eventer = gevent.spawn(self.event_producer)
            while True:
                event = self.q.get()
                if event.name == pth.basename(self.path):
                    break
        finally:
            gevent.wait((eventer, ))
            os.close(self.fd)

        return True
Exemple #6
0
    def reloader():
        import gevent_inotifyx as inotifyx

        fd = inotifyx.init()
        # NOTE: .git/logs/HEAD is the last thing updated after a git pull/merge
        inotifyx.add_watch(fd, "../.git/logs/HEAD", inotifyx.IN_MODIFY)
        inotifyx.add_watch(fd, ".reloader", inotifyx.IN_MODIFY | inotifyx.IN_ATTRIB)
        inotifyx.get_events(fd)
        logging.warn("Shutting down due to new update")
        SERVER.stop_accepting()
        logging.warn("Free Count[%d] (will kill outstanding processes in 120 sec.)" % SERVER.pool.free_count())
        SERVER.pool.join(timeout=120)
        SERVER.close()
        logging.warn("Shut down successful")
Exemple #7
0
    def watch(self):
        fd = inotifyx.init()
        inotifyx.add_watch(fd, self.pickup_dir, inotifyx.IN_CLOSE_WRITE | inotifyx.IN_MOVE)
        
        while self.keep_running:
            try:
                events = inotifyx.get_events(fd, 1.0)
                for event in events:
                    self.pickup_event_processor.process(event)

                sleep(0.1)
            except KeyboardInterrupt:
                self.keep_running = False

        os.close(fd)
Exemple #8
0
    def watch(self):
        fd = inotifyx.init()
        inotifyx.add_watch(fd, self.pickup_dir, inotifyx.IN_CLOSE_WRITE | inotifyx.IN_MOVE)

        while self.keep_running:
            try:
                events = inotifyx.get_events(fd, 1.0)
                for event in events:
                    self.pickup_event_processor.process(event)

                sleep(0.1)
            except KeyboardInterrupt:
                self.keep_running = False

        os.close(fd)
Exemple #9
0
 def reloader():
     import gevent_inotifyx as inotifyx
     fd = inotifyx.init()
     # NOTE: .git/logs/HEAD is the last thing updated after a git pull/merge
     inotifyx.add_watch(fd, '../.git/logs/HEAD', inotifyx.IN_MODIFY)
     inotifyx.add_watch(fd, '.reloader',
                        inotifyx.IN_MODIFY | inotifyx.IN_ATTRIB)
     inotifyx.get_events(fd)
     logging.warn('Shutting down due to new update')
     SERVER.stop_accepting()
     logging.warn(
         'Free Count[%d] (will kill outstanding processes in 120 sec.)' %
         SERVER.pool.free_count())
     SERVER.pool.join(timeout=120)
     SERVER.close()
     logging.warn('Shut down successful')
Exemple #10
0
    def watch(self):
        fd = inotifyx.init()

        for name in self.pickup_dir:
            try:
                self.wd_to_name[inotifyx.add_watch(fd, name, inotifyx.IN_CLOSE_WRITE | inotifyx.IN_MOVE)] = name
            except IOError, e:
                logger.warn('Caught IOError `%s`, name:`%s`', format_exc(e), name)
Exemple #11
0
    def __init__(self, logger, directory="rules/"):
        self.logger = logger
        self.directory = directory

        if not os.access(self.directory, os.R_OK):
            raise Exception("Directory '%s' is not readable. Please verify." % (self.directory))

        self.fd = inotify.init()
        self.wb = inotify.add_watch(self.fd, self.directory, inotify.IN_CLOSE_WRITE + inotify.IN_CREATE + inotify.IN_DELETE + inotify.IN_MODIFY + inotify.IN_MOVE)
Exemple #12
0
    def wait_for_plugin_changes(self):
        import gevent_inotifyx as inotify

        fd = inotify.init()
        inotify.add_watch(fd, 'rowboat/plugins/', inotify.IN_MODIFY)
        while True:
            events = inotify.get_events(fd)
            for event in events:
                # Can't reload core.py sadly
                if event.name == 'core.py':
                    continue

                plugin_name = '{}Plugin'.format(
                    event.name.split('.', 1)[0].title())
                if plugin_name in self.bot.plugins:
                    self.log.info('Detected change in %s, reloading...',
                                  plugin_name)
                    self.bot.plugins[plugin_name].reload()
 def test_get_events(self):
     wd = inotify.add_watch(self.fd, self.testdir, inotify.IN_CREATE)
     self._create_file('foo')
     try:
         events = inotify.get_events(self.fd)
         self.assertEqual(len(events), 1)
         self.assertEqual(events[0].mask, inotify.IN_CREATE)
         self.assertEqual(events[0].name, 'foo')
     finally:
         os.unlink('foo')
Exemple #14
0
    def watch(self):
        fd = inotifyx.init()

        for name in self.pickup_dir:
            try:
                self.wd_to_name[inotifyx.add_watch(
                    fd, name,
                    inotifyx.IN_CLOSE_WRITE | inotifyx.IN_MOVE)] = name
            except IOError, e:
                logger.warn('Caught IOError `%s`, name:`%s`', format_exc(e),
                            name)
Exemple #15
0
    def wait_for_plugin_changes(self):
        import gevent_inotifyx as inotify

        fd = inotify.init()
        inotify.add_watch(fd, 'rowboat/plugins/', inotify.IN_MODIFY)
        while True:
            events = inotify.get_events(fd)
            for event in events:
                # Can't reload core.py sadly
                if event.name.startswith('core.py'):
                    continue

                plugin_name = '{}Plugin'.format(event.name.split('.', 1)[0].title())
                plugin = next((v for k, v in self.bot.plugins.items() if k.lower() == plugin_name.lower()), None)
                if plugin:
                    self.log.info('Detected change in %s, reloading...', plugin_name)
                    try:
                        plugin.reload()
                    except Exception:
                        self.log.exception('Failed to reload: ')
    def start(self):
        self.fd = inotify.init()
        self.ddirs = {}
        for startpath in self.paths:
            for ddir in j.system.fs.listDirsInDir(startpath, True):
                ddirname = j.system.fs.getDirName(ddir + "/", True)
                # print ddirname
                ok = True
                if ddirname.find(".") == 0 or ddir.find(".cache") != -1:
                    ok = False

                if ok:
                    try:
                        wd = inotify.add_watch(
                            self.fd, ddir,
                            inotify.IN_CREATE | inotify.IN_DELETE
                            | inotify.IN_DELETE_SELF | inotify.IN_MODIFY
                            | inotify.IN_MOVE_SELF | inotify.IN_MOVED_FROM
                            | inotify.IN_MOVED_TO)
                        # print "add notify %s" % ddir
                        self.ddirs[wd] = ddir
                    except:
                        print "ERROR COULD NOT ADD notify %s" % ddir

        print "started"
        while True:
            events = inotify.get_events(self.fd)
            for event in events:
                # j.put(event)
                if event.name != None:
                    path = self.ddirs[event.wd] + "/" + event.name
                else:
                    path = self.ddirs[event.wd]
                    print "restarted for %s" % path
                    self.start()

                if path.find(".goutputstream") != -1:
                    continue

                print "received event:", event.get_mask_description(), path

                if j.system.fs.isFile(path):
                    if j.system.fs.getFileExtension(path).lower() in [
                            "wiki", "py", "spec"
                    ]:
                        try:
                            self.contentmgr.notifychange(path)
                            print "notified %s" % path
                        except:
                            print "could not notify change for %s" % path
                elif j.system.fs.isDir(path):
                    print "restarted for dir %s" % path
                    self.start()
    def start(self):
        self.fd = inotify.init()
        self.ddirs = {}
        for startpath in self.paths:
            for ddir in j.system.fs.listDirsInDir(startpath, True):
                ddirname = j.system.fs.getDirName(ddir + "/", True)
                # print ddirname
                ok = True
                if ddirname.find(".") == 0 or ddir.find(".cache") != -1:
                    ok = False

                if ok:
                    try:
                        wd = inotify.add_watch(self.fd, ddir, inotify.IN_CREATE | inotify.IN_DELETE
                                               | inotify.IN_DELETE_SELF | inotify.IN_MODIFY
                                               | inotify.IN_MOVE_SELF | inotify.IN_MOVED_FROM
                                               | inotify.IN_MOVED_TO)
                        # print "add notify %s" % ddir
                        self.ddirs[wd] = ddir
                    except:
                        print "ERROR COULD NOT ADD notify %s" % ddir

        print "started"
        while True:
            events = inotify.get_events(self.fd)
            for event in events:
                # j.put(event)
                if event.name != None:
                    path = self.ddirs[event.wd] + "/" + event.name
                else:
                    path = self.ddirs[event.wd]
                    print "restarted for %s" % path
                    self.start()

                if path.find(".goutputstream") != -1:
                    continue

                print "received event:", event.get_mask_description(), path

                if j.system.fs.isFile(path):
                    if j.system.fs.getFileExtension(path).lower() in ["wiki", "py", "spec"]:
                        try:
                            self.contentmgr.notifychange(path)
                            print "notified %s" % path
                        except:
                            print "could not notify change for %s" % path
                elif j.system.fs.isDir(path):
                    print "restarted for dir %s" % path
                    self.start()
Exemple #18
0
    def __init__(self, logger, directory="rules/"):
        self.logging = logger
        self.directory = directory

        if not os.access(self.directory, os.R_OK):
            raise Exception("Directory '%s' is not readable. Please verify." %
                            (self.directory))

        self.fd = inotify.init()
        self.wb = inotify.add_watch(
            self.fd, self.directory,
            inotify.IN_CLOSE_WRITE + inotify.IN_CREATE + inotify.IN_DELETE +
            inotify.IN_MODIFY + inotify.IN_MOVE)
        self.logging.info("Monitoring directory '%s' for changes" %
                          (directory))
 def test_get_events_async(self):
     wd = inotify.add_watch(self.fd, self.testdir, inotify.IN_CREATE)
     task1 = gevent.spawn(inotify.get_events, self.fd, 10)
     task2 = gevent.spawn(self._create_file, 'bar')
     try:
         # in the synchronous case we would block here for 10s and the
         # return value in task1.value would be empty
         task1.join()
         task2.join()
         events = task1.value
         self.assertEqual(len(events), 1)
         self.assertEqual(events[0].mask, inotify.IN_CREATE)
         self.assertEqual(events[0].name, 'bar')
     finally:
         os.unlink('bar')
Exemple #20
0
    def monitorDirectory(self):
        '''Monitors the given directory for changes.'''

        fd = inotify.init()
        wb = inotify.add_watch(fd, self.directory,
                               inotify.IN_CLOSE_WRITE + inotify.IN_DELETE)
        self.__rules = self.readDirectory()

        while True:
            self.wait.clear()
            events = inotify.get_events(fd)
            current_rules = self.readDirectory()

            if cmp(current_rules, self.__rules) != 0:
                self.__rules = current_rules
                self.wait.set()
Exemple #21
0
    def run(self):
        glt = ezRPGreenlet.EzReverseProxyGreenlet(self._logger, self._sfh)
        self._logger.info("starting greenlet for thrift service...")
        clientGreenlet = gevent.spawn(glt.clientServiceGreenlet)
        self._logger.info("started")

        self._logger.info("starting greenlet to monitor zookeeper...")
        kzGreenlet = gevent.spawn(glt.kzMonitorGreenlet)
        self._logger.info("started")

        self._logger.info(
            "starting greenlet to write out nginx configuration changes...")
        cfgProcessorGreenlet = gevent.spawn(
            glt.configurationChangeQueueGreenlet)
        self._logger.info("started")

        self._logger.info("starting greenlet to monitor for shutdown...")
        fd = gevent_inotifyx.init()
        wd = gevent_inotifyx.add_watch(fd, gConfig.shutdownFile,
                                       gevent_inotifyx.IN_DELETE)
        wGreenlet = gevent.spawn(glt.watchGreenlet, fd)
        self._logger.info("started")

        gConfig.addGreenlets(clientGreenlet, kzGreenlet, cfgProcessorGreenlet,
                             wGreenlet)

        gevent.joinall([clientGreenlet])
        self._logger.warn("joined thrift service greenlet")
        gevent.joinall([kzGreenlet])
        self._logger.warn("joined zookeeper monitoring greenlet")
        gConfig.run = False
        while not gConfig.configurationChangeQueue.empty():
            print "queue not empty"
            gConfig.configurationChangeQueue.get()
            print "got"
            gConfig.configurationChangeQueue.task_done()
        print "joining conf queue"
        gConfig.configurationChangeQueue.join()
        self._logger.warn("joined configuration queue")
        gevent.joinall([cfgProcessorGreenlet])
        self._logger.warn("joined configuration change greenlet")
        gConfig.wGreenlet.join()
        self._logger.warn("joined shutdown monitor greenlet")
        ServiceDiscoveryClient(gConfig.zk).unregister_endpoint(
            gConfig.appName, EzFrontendServiceName, gConfig.internal_hostname,
            gConfig.thriftPort)
        self._logger.warn("unregistered from discovery service")
Exemple #22
0
def inotify_event(file_path, rules):
    fd = inotify.init()
    (config_path, config_name) = os.path.split(file_path)
    if not config_path:
        config_path = os.getcwd()
    file_name = config_path + '/' + config_name
    wd = inotify.add_watch(
        fd, config_path, inotify.IN_MODIFY | inotify.IN_CREATE)
    rules.update(reload_config(file_name))
    while True:
        events = inotify.get_events(fd)
        update = False
        for event in events:
            if event.name == config_name:
                update = True
        if update:
            rules.update(reload_config(file_name))
    def run(self):
      glt = ezRPGreenlet.EzReverseProxyGreenlet(self._logger, self._sfh)
      self._logger.info("starting greenlet for thrift service...")
      clientGreenlet = gevent.spawn(glt.clientServiceGreenlet)
      self._logger.info("started")

      self._logger.info("starting greenlet to monitor zookeeper...")
      kzGreenlet = gevent.spawn(glt.kzMonitorGreenlet)
      self._logger.info("started")

      self._logger.info("starting greenlet to write out nginx configuration changes...")
      cfgProcessorGreenlet = gevent.spawn(glt.configurationChangeQueueGreenlet)
      self._logger.info("started")

      self._logger.info("starting greenlet to monitor for shutdown...")
      fd = gevent_inotifyx.init()
      wd = gevent_inotifyx.add_watch(fd, gConfig.shutdownFile, gevent_inotifyx.IN_DELETE)
      wGreenlet = gevent.spawn(glt.watchGreenlet, fd)
      self._logger.info("started")

      gConfig.addGreenlets(clientGreenlet, kzGreenlet, cfgProcessorGreenlet, wGreenlet)


      gevent.joinall([clientGreenlet])
      self._logger.warn("joined thrift service greenlet")
      gevent.joinall([kzGreenlet])
      self._logger.warn("joined zookeeper monitoring greenlet")
      gConfig.run = False
      while not gConfig.configurationChangeQueue.empty():
          print "queue not empty"
          gConfig.configurationChangeQueue.get()
          print "got"
          gConfig.configurationChangeQueue.task_done()
      print "joining conf queue"
      gConfig.configurationChangeQueue.join()
      self._logger.warn("joined configuration queue")
      gevent.joinall([cfgProcessorGreenlet])
      self._logger.warn("joined configuration change greenlet")
      gConfig.wGreenlet.join()
      self._logger.warn("joined shutdown monitor greenlet")
      ServiceDiscoveryClient(gConfig.zk).unregister_endpoint(gConfig.appName,
                                                             EzFrontendServiceName,
                                                             gConfig.internal_hostname,
                                                             gConfig.thriftPort)
      self._logger.warn("unregistered from discovery service")
Exemple #24
0
    def _run(self):
        if not use_inotify:
            log.warn("gevent_inotifyx not loaded; not using inotify")
            return

        fd = inotify.init()
        wd = inotify.add_watch(fd, self.opts.spool_dir,
                inotify.IN_CLOSE_WRITE | inotify.IN_MOVED_TO)
        while True:
            events = inotify.get_events(fd)
            for event in events:
                path = os.path.sep.join([self.opts.spool_dir,
                                         event.name])

                # Filter out inotify events generated for files that
                # have been already unlinked from the filesystem
                # (IN_EXCL_UNLINK emulation)
                if os.path.exists(path):
                    self.on_find(path)
Exemple #25
0
    def run(self):

        try:

            for path in self.callback_config:
                if not os.path.exists(path):
                    raise Exception('Path does not exist `{}`'.format(path))

                self.wd_to_path[infx.add_watch(
                    self.infx_fd, path,
                    infx.IN_CLOSE_WRITE | infx.IN_MOVE)] = path

            while self.keep_running:
                try:
                    events = infx.get_events(self.infx_fd, 1.0)

                    for event in events:
                        pe = PickupEvent()

                        try:

                            pe.base_dir = self.wd_to_path[event.wd]
                            config = self.callback_config[pe.base_dir]

                            if not self.should_pick_up(event.name,
                                                       config.patterns):
                                continue

                            pe.file_name = event.name
                            pe.stanza = config.stanza
                            pe.full_path = os.path.join(
                                pe.base_dir, event.name)

                            # If we are deploying services, the path is different than for other resources
                            if config.is_service_hot_deploy:
                                spawn_greenlet(hot_deploy, self.server,
                                               pe.file_name, pe.full_path,
                                               config.delete_after_pick_up)
                                continue

                            if config.read_on_pickup:

                                f = open(pe.full_path, 'rb')
                                pe.raw_data = f.read()
                                pe.has_raw_data = True
                                f.close()

                                if config.parse_on_pickup:

                                    try:
                                        pe.data = self.get_parser(
                                            config.parse_with)(pe.raw_data)
                                        pe.has_data = True
                                    except Exception, e:
                                        pe.parse_error = e

                                else:
                                    pe.data = pe.raw_data

                            spawn_greenlet(self.invoke_callbacks, pe,
                                           config.services, config.topics)
                            self.post_handle(pe.full_path, config)

                        except Exception, e:
                            logger.warn(format_exc(e))

                except KeyboardInterrupt:
                    self.keep_running = False
Exemple #26
0
    elif o in ("-r", "--run"):
        if a == 'live':
            STREAM_SERVER = '216.18.184.22:1935'
            STREAM_USER = '******'
            PIC_PATH = '/srv/LIVE_model_imgs_ramfs/'
            ONLINE_MODELS_URL = 'http://www.seeme.com/onlinemodels'
            PID_FILE = '/tmp/LIVE_ffmpeg_websocket_server.pid'
            PORT = 8023
            FLASH_PORT = 10843
    else:
        assert False, "unhandled option"

q = Queue()
q.maxsize = 1000
fd = inotify.init()
inotify.add_watch(fd, PIC_PATH, inotify.IN_CREATE)
stream_dumper = StreamDumper()
gevent.spawn(event_producer, fd, q)

signal.signal(signal.SIGTERM, exit_cleanup)
signal.signal(signal.SIGINT, exit_cleanup)
signal.signal(signal.SIGQUIT, exit_cleanup)
atexit.register(exit_cleanup)

if DEBUG: print("writing pid file: %s" % PID_FILE)
with open(PID_FILE, 'w') as f:
    f.write(str(os.getpid()))

# start ffmpeg processes for online models
r = requests.get(ONLINE_MODELS_URL)
if r.status_code == 200:
Exemple #27
0
 def prepare(self):
     super(FileWatcher, self).prepare()
     self._inotify_fd = inotify.init()
     self._wd = inotify.add_watch(self._inotify_fd, self.watch_path,
                                  inotify.IN_CLOSE_WRITE)
     logger.info("Listening path: %s" % self.watch_path)