Example #1
0
    def start_guest(self, id, config, details=None):
        """
        Start a new guest process on this node.

        :param config: The guest process configuration.
        :type config: obj

        :returns: int -- The PID of the new process.
        """
        # prohibit starting a worker twice
        #
        if id in self._workers:
            emsg = "Could not start worker: a worker with ID '{}' is already running (or starting)".format(
                id)
            self.log.error(emsg)
            raise ApplicationError(u'crossbar.error.worker_already_running',
                                   emsg)

        try:
            checkconfig.check_guest(config)
        except Exception as e:
            raise ApplicationError(
                u'crossbar.error.invalid_configuration',
                'invalid guest worker configuration: {}'.format(e))

        options = config.get('options', {})

        # guest process working directory
        #
        workdir = self._node._cbdir
        if 'workdir' in options:
            workdir = os.path.join(workdir, options['workdir'])
        workdir = os.path.abspath(workdir)

        # guest process executable and command line arguments
        #

        # first try to configure the fully qualified path for the guest
        # executable by joining workdir and configured exectuable ..
        exe = os.path.abspath(os.path.join(workdir, config['executable']))

        if check_executable(exe):
            self.log.info(
                "Using guest worker executable '{exe}' (executable path taken from configuration)",
                exe=exe)
        else:
            # try to detect the fully qualified path for the guest
            # executable by doing a "which" on the configured executable name
            exe = shutil.which(config['executable'])
            if exe is not None and check_executable(exe):
                self.log.info(
                    "Using guest worker executable '{exe}' (executable path detected from environment)",
                    exe=exe)
            else:
                emsg = "Could not start worker: could not find and executable for '{}'".format(
                    config['executable'])
                self.log.error(emsg)
                raise ApplicationError(u'crossbar.error.invalid_configuration',
                                       emsg)

        # guest process command line arguments
        #
        args = [exe]
        args.extend(config.get('arguments', []))

        # guest process environment
        #
        worker_env = create_process_env(options)

        # log name of worker
        #
        worker_logname = 'Guest'

        # topic URIs used (later)
        #
        starting_topic = 'crossbar.node.{}.on_guest_starting'.format(
            self._node_id)
        started_topic = 'crossbar.node.{}.on_guest_started'.format(
            self._node_id)

        # add worker tracking instance to the worker map ..
        #
        worker = GuestWorkerProcess(self,
                                    id,
                                    details.caller,
                                    keeplog=options.get('traceback', None))

        self._workers[id] = worker

        # create a (custom) process endpoint
        #
        ep = WorkerProcessEndpoint(self._node._reactor,
                                   exe,
                                   args,
                                   path=workdir,
                                   env=worker_env,
                                   worker=worker)

        # ready handling
        #
        def on_ready_success(proto):

            worker.pid = proto.transport.pid
            worker.status = 'started'
            worker.started = datetime.utcnow()

            self.log.info("{worker} with ID '{id}' and PID {pid} started",
                          worker=worker_logname,
                          id=worker.id,
                          pid=worker.pid)

            self._node._reactor.addSystemEventTrigger(
                'before',
                'shutdown',
                self._cleanup_worker,
                self._node._reactor,
                worker,
            )

            # directory watcher
            #
            if 'watch' in options:

                if HAS_FSNOTIFY:

                    # assemble list of watched directories
                    watched_dirs = []
                    for d in options['watch'].get('directories', []):
                        watched_dirs.append(
                            os.path.abspath(os.path.join(self._node._cbdir,
                                                         d)))

                    worker.watch_timeout = options['watch'].get('timeout', 1)

                    # create a directory watcher
                    worker.watcher = DirWatcher(dirs=watched_dirs,
                                                notify_once=True)

                    # make sure to stop the background thread running inside the
                    # watcher upon Twisted being shut down
                    def on_shutdown():
                        worker.watcher.stop()

                    self._node._reactor.addSystemEventTrigger(
                        'before', 'shutdown', on_shutdown)

                    # this handler will get fired by the watcher upon detecting an FS event
                    def on_fsevent(evt):
                        worker.watcher.stop()
                        proto.signal('TERM')

                        if options['watch'].get('action', None) == 'restart':
                            self.log.info("Restarting guest ..")
                            # Add a timeout large enough (perhaps add a config option later)
                            self._node._reactor.callLater(
                                worker.watch_timeout, self.start_guest, id,
                                config, details)
                            # Shut the worker down, after the restart event is scheduled
                            worker.stop()

                    # now run the watcher on a background thread
                    deferToThread(worker.watcher.loop, on_fsevent)

                else:
                    self.log.warn(
                        "Warning: cannot watch directory for changes - feature DirWatcher unavailable"
                    )

            # assemble guest worker startup information
            #
            started_info = {
                'id': worker.id,
                'status': worker.status,
                'started': utcstr(worker.started),
                'who': worker.who
            }

            self.publish(started_topic,
                         started_info,
                         options=PublishOptions(exclude=[details.caller]))

            return started_info

        def on_ready_error(err):
            del self._workers[worker.id]

            emsg = 'Failed to start guest worker: {}'.format(err.value)
            self.log.error(emsg)
            raise ApplicationError(u"crossbar.error.cannot_start", emsg,
                                   ep.getlog())

        worker.ready.addCallbacks(on_ready_success, on_ready_error)

        def on_exit_success(res):
            self.log.info("Guest {id} exited with success", id=worker.id)
            del self._workers[worker.id]

        def on_exit_error(err):
            self.log.error("Guest {id} exited with error {err.value}",
                           id=worker.id,
                           err=err)
            del self._workers[worker.id]

        worker.exit.addCallbacks(on_exit_success, on_exit_error)

        # create a transport factory for talking WAMP to the native worker
        #
        transport_factory = create_guest_worker_client_factory(
            config, worker.ready, worker.exit)
        transport_factory.noisy = False
        self._workers[id].factory = transport_factory

        # now (immediately before actually forking) signal the starting of the worker
        #
        starting_info = {
            'id': id,
            'status': worker.status,
            'created': utcstr(worker.created),
            'who': worker.who
        }

        # the caller gets a progressive result ..
        if details.progress:
            details.progress(starting_info)

        # .. while all others get an event
        self.publish(starting_topic,
                     starting_info,
                     options=PublishOptions(exclude=[details.caller]))

        # now actually fork the worker ..
        #
        self.log.info("Starting {worker} with ID '{id}'...",
                      worker=worker_logname,
                      id=id)
        self.log.debug("{worker} '{id}' using command line '{cli}'...",
                       worker=worker_logname,
                       id=id,
                       cli=' '.join(args))

        d = ep.connect(transport_factory)

        def on_connect_success(proto):

            # this seems to be called immediately when the child process
            # has been forked. even if it then immediately fails because
            # e.g. the executable doesn't even exist. in other words,
            # I'm not sure under what conditions the deferred will
            # errback - probably only if the forking of a new process fails
            # at OS level due to out of memory conditions or such.

            pid = proto.transport.pid
            self.log.debug("Guest worker process connected with PID {pid}",
                           pid=pid)

            worker.pid = pid

            # proto is an instance of GuestWorkerClientProtocol
            worker.proto = proto

            worker.status = 'connected'
            worker.connected = datetime.utcnow()

        def on_connect_error(err):

            # not sure when this errback is triggered at all .. see above.
            self.log.error(
                "Internal error: connection to forked guest worker failed ({})"
                .format(err))

            # in any case, forward the error ..
            worker.ready.errback(err)

        d.addCallbacks(on_connect_success, on_connect_error)

        return worker.ready
Example #2
0
    def start_guest(self, id, config, details=None):
        """
        Start a new guest process on this node.

        :param config: The guest process configuration.
        :type config: obj

        :returns: int -- The PID of the new process.
        """
        # prohibit starting a worker twice
        #
        if id in self._workers:
            emsg = "ERROR: could not start worker - a worker with ID '{}' is already running (or starting)".format(id)
            log.msg(emsg)
            raise ApplicationError('crossbar.error.worker_already_running', emsg)

        try:
            checkconfig.check_guest(config)
        except Exception as e:
            raise ApplicationError('crossbar.error.invalid_configuration', 'invalid guest worker configuration: {}'.format(e))

        options = config.get('options', {})

        # guest process working directory
        #
        workdir = self._node._cbdir
        if 'workdir' in options:
            workdir = os.path.join(workdir, options['workdir'])
        workdir = os.path.abspath(workdir)

        # guest process executable and command line arguments
        #

        # first try to configure the fully qualified path for the guest
        # executable by joining workdir and configured exectuable ..
        exe = os.path.abspath(os.path.join(workdir, config['executable']))

        if check_executable(exe):
            log.msg("Using guest worker executable '{}' (executable path taken from configuration)".format(exe))
        else:
            # try to detect the fully qualified path for the guest
            # executable by doing a "which" on the configured executable name
            exe = shutil.which(config['executable'])
            if exe is not None and check_executable(exe):
                log.msg("Using guest worker executable '{}' (executable path detected from environment)".format(exe))
            else:
                emsg = "ERROR: could not start worker - could not find and executable for '{}'".format(config['executable'])
                log.msg(emsg)
                raise ApplicationError('crossbar.error.invalid_configuration', emsg)

        # guest process command line arguments
        #
        args = [exe]
        args.extend(config.get('arguments', []))

        # guest process environment
        #
        worker_env = create_process_env(options)

        # log name of worker
        #
        worker_logname = 'Guest'

        # topic URIs used (later)
        #
        starting_topic = 'crossbar.node.{}.on_guest_starting'.format(self._node_id)
        started_topic = 'crossbar.node.{}.on_guest_started'.format(self._node_id)

        # add worker tracking instance to the worker map ..
        #
        worker = GuestWorkerProcess(self, id, details.caller, keeplog=options.get('traceback', None))

        self._workers[id] = worker

        # create a (custom) process endpoint
        #
        ep = WorkerProcessEndpoint(self._node._reactor, exe, args, path=workdir, env=worker_env, worker=worker)

        # ready handling
        #
        def on_ready_success(proto):

            worker.pid = proto.transport.pid
            worker.status = 'started'
            worker.started = datetime.utcnow()

            log.msg("{} with ID '{}' and PID {} started".format(worker_logname, worker.id, worker.pid))

            # directory watcher
            #
            if 'watch' in options:

                if HAS_FSNOTIFY:

                    # assemble list of watched directories
                    watched_dirs = []
                    for d in options['watch'].get('directories', []):
                        watched_dirs.append(os.path.abspath(os.path.join(self._node._cbdir, d)))

                    # create a directory watcher
                    worker.watcher = DirWatcher(dirs=watched_dirs, notify_once=True)

                    # make sure to stop the background thread running inside the
                    # watcher upon Twisted being shut down
                    def on_shutdown():
                        worker.watcher.stop()

                    reactor.addSystemEventTrigger('before', 'shutdown', on_shutdown)

                    # this handler will get fired by the watcher upon detecting an FS event
                    def on_fsevent(evt):
                        worker.watcher.stop()
                        proto.signal('TERM')

                        if options['watch'].get('action', None) == 'restart':
                            log.msg("Restarting guest ..")
                            reactor.callLater(0.1, self.start_guest, id, config, details)

                    # now run the watcher on a background thread
                    deferToThread(worker.watcher.loop, on_fsevent)

                else:
                    log.msg("Warning: cannot watch directory for changes - feature DirWatcher unavailable")

            # assemble guest worker startup information
            #
            started_info = {
                'id': worker.id,
                'status': worker.status,
                'started': utcstr(worker.started),
                'who': worker.who
            }

            self.publish(started_topic, started_info, options=PublishOptions(exclude=[details.caller]))

            return started_info

        def on_ready_error(err):
            del self._workers[worker.id]

            emsg = 'ERROR: failed to start guest worker - {}'.format(err.value)
            log.msg(emsg)
            raise ApplicationError("crossbar.error.cannot_start", emsg, ep.getlog())

        worker.ready.addCallbacks(on_ready_success, on_ready_error)

        def on_exit_success(res):
            log.msg("Guest excited with success")
            del self._workers[worker.id]

        def on_exit_error(err):
            log.msg("Guest excited with error", err)
            del self._workers[worker.id]

        worker.exit.addCallbacks(on_exit_success, on_exit_error)

        # create a transport factory for talking WAMP to the native worker
        #
        transport_factory = create_guest_worker_client_factory(config, worker.ready, worker.exit)
        transport_factory.noisy = False
        self._workers[id].factory = transport_factory

        # now (immediately before actually forking) signal the starting of the worker
        #
        starting_info = {
            'id': id,
            'status': worker.status,
            'created': utcstr(worker.created),
            'who': worker.who
        }

        # the caller gets a progressive result ..
        if details.progress:
            details.progress(starting_info)

        # .. while all others get an event
        self.publish(starting_topic, starting_info, options=PublishOptions(exclude=[details.caller]))

        # now actually fork the worker ..
        #
        if self.debug:
            log.msg("Starting {} with ID '{}' using command line '{}' ..".format(worker_logname, id, ' '.join(args)))
        else:
            log.msg("Starting {} with ID '{}' ..".format(worker_logname, id))

        d = ep.connect(transport_factory)

        def on_connect_success(proto):

            # this seems to be called immediately when the child process
            # has been forked. even if it then immediately fails because
            # e.g. the executable doesn't even exist. in other words,
            # I'm not sure under what conditions the deferred will
            # errback - probably only if the forking of a new process fails
            # at OS level due to out of memory conditions or such.

            pid = proto.transport.pid
            if self.debug:
                log.msg("Guest worker process connected with PID {}".format(pid))

            worker.pid = pid

            # proto is an instance of GuestWorkerClientProtocol
            worker.proto = proto

            worker.status = 'connected'
            worker.connected = datetime.utcnow()

        def on_connect_error(err):

            # not sure when this errback is triggered at all .. see above.
            if self.debug:
                log.msg("ERROR: Connecting forked guest worker failed - {}".format(err))

            # in any case, forward the error ..
            worker.ready.errback(err)

        d.addCallbacks(on_connect_success, on_connect_error)

        return worker.ready
Example #3
0
    def _start_guest_worker(self, worker_id, worker_config, details=None):
        """
        Start a new guest process on this node.

        :param config: The guest process configuration.
        :type config: obj

        :returns: int -- The PID of the new process.
        """
        # prohibit starting a worker twice
        #
        if worker_id in self._workers:
            emsg = "Could not start worker: a worker with ID '{}' is already running (or starting)".format(
                worker_id)
            self.log.error(emsg)
            raise ApplicationError(u'crossbar.error.worker_already_running',
                                   emsg)

        try:
            checkconfig.check_guest(worker_config)
        except Exception as e:
            raise ApplicationError(
                u'crossbar.error.invalid_configuration',
                'invalid guest worker configuration: {}'.format(e))

        options = worker_config.get('options', {})

        # guest process working directory
        #
        workdir = self._node._cbdir
        if 'workdir' in options:
            workdir = os.path.join(workdir, options['workdir'])
        workdir = os.path.abspath(workdir)

        # guest process executable and command line arguments
        #

        # first try to configure the fully qualified path for the guest
        # executable by joining workdir and configured exectuable ..
        exe = os.path.abspath(
            os.path.join(workdir, worker_config['executable']))

        if check_executable(exe):
            self.log.info(
                "Using guest worker executable '{exe}' (executable path taken from configuration)",
                exe=exe)
        else:
            # try to detect the fully qualified path for the guest
            # executable by doing a "which" on the configured executable name
            exe = which(worker_config['executable'])
            if exe is not None and check_executable(exe):
                self.log.info(
                    "Using guest worker executable '{exe}' (executable path detected from environment)",
                    exe=exe)
            else:
                emsg = "Could not start worker: could not find and executable for '{}'".format(
                    worker_config['executable'])
                self.log.error(emsg)
                raise ApplicationError(u'crossbar.error.invalid_configuration',
                                       emsg)

        # guest process command line arguments
        #
        args = [exe]
        args.extend(worker_config.get('arguments', []))

        # guest process environment
        #
        worker_env = create_process_env(options)

        # log name of worker
        #
        worker_logname = 'Guest'

        # topic URIs used (later)
        #
        starting_topic = u'{}.on_guest_starting'.format(self._uri_prefix)
        started_topic = u'{}.on_guest_started'.format(self._uri_prefix)

        # add worker tracking instance to the worker map ..
        #
        worker = GuestWorkerProcess(self,
                                    worker_id,
                                    details.caller,
                                    keeplog=options.get('traceback', None))

        self._workers[worker_id] = worker

        # create a (custom) process endpoint
        #
        ep = WorkerProcessEndpoint(self._node._reactor,
                                   exe,
                                   args,
                                   path=workdir,
                                   env=worker_env,
                                   worker=worker)

        # ready handling
        #
        def on_ready_success(proto):

            self.log.info('{worker_logname} worker "{worker_id}" started',
                          worker_logname=worker_logname,
                          worker_id=worker.id)

            worker.on_worker_started(proto)

            self._node._reactor.addSystemEventTrigger(
                'before',
                'shutdown',
                self._cleanup_worker,
                self._node._reactor,
                worker,
            )

            # directory watcher
            #
            if 'watch' in options:

                if HAS_FS_WATCHER:

                    # assemble list of watched directories
                    watched_dirs = []
                    for d in options['watch'].get('directories', []):
                        watched_dirs.append(
                            os.path.abspath(os.path.join(self._node._cbdir,
                                                         d)))

                    worker.watch_timeout = options['watch'].get('timeout', 1)

                    # create a filesystem watcher
                    worker.watcher = FilesystemWatcher(
                        workdir, watched_dirs=watched_dirs)

                    # make sure to stop the watch upon Twisted being shut down
                    def on_shutdown():
                        worker.watcher.stop()

                    self._node._reactor.addSystemEventTrigger(
                        'before', 'shutdown', on_shutdown)

                    # this handler will get fired by the watcher upon detecting an FS event
                    def on_filesystem_change(fs_event):
                        worker.watcher.stop()
                        proto.signal('TERM')

                        if options['watch'].get('action', None) == 'restart':
                            self.log.info(
                                "Filesystem watcher detected change {fs_event} - restarting guest in {watch_timeout} seconds ..",
                                fs_event=fs_event,
                                watch_timeout=worker.watch_timeout)
                            # Add a timeout large enough (perhaps add a config option later)
                            self._node._reactor.callLater(
                                worker.watch_timeout, self.start_guest,
                                worker_id, worker_config, details)
                            # Shut the worker down, after the restart event is scheduled
                            # FIXME: all workers should have a stop() method ..
                            # -> 'GuestWorkerProcess' object has no attribute 'stop'
                            # worker.stop()
                        else:
                            self.log.info(
                                "Filesystem watcher detected change {fs_event} - no action taken!",
                                fs_event=fs_event)

                    # now start watching ..
                    worker.watcher.start(on_filesystem_change)

                else:
                    self.log.warn(
                        "Cannot watch directories for changes - feature not available"
                    )

            # assemble guest worker startup information
            #
            started_info = {
                u'id': worker.id,
                u'status': worker.status,
                u'started': utcstr(worker.started),
                u'who': worker.who,
            }

            self.publish(started_topic,
                         started_info,
                         options=PublishOptions(exclude=details.caller))

            return started_info

        def on_ready_error(err):
            del self._workers[worker.id]

            emsg = 'Failed to start guest worker: {}'.format(err.value)
            self.log.error(emsg)
            raise ApplicationError(u"crossbar.error.cannot_start", emsg,
                                   ep.getlog())

        worker.ready.addCallbacks(on_ready_success, on_ready_error)

        def on_exit_success(res):
            self.log.info("Guest {worker_id} exited with success",
                          worker_id=worker.id)
            del self._workers[worker.id]

        def on_exit_error(err):
            self.log.error("Guest {worker_id} exited with error {err.value}",
                           worker_id=worker.id,
                           err=err)
            del self._workers[worker.id]

        worker.exit.addCallbacks(on_exit_success, on_exit_error)

        # create a transport factory for talking WAMP to the native worker
        #
        transport_factory = create_guest_worker_client_factory(
            worker_config, worker.ready, worker.exit)
        transport_factory.noisy = False
        self._workers[worker_id].factory = transport_factory

        # now (immediately before actually forking) signal the starting of the worker
        #
        starting_info = {
            u'id': worker_id,
            u'status': worker.status,
            u'created': utcstr(worker.created),
            u'who': worker.who,
        }

        # the caller gets a progressive result ..
        if details.progress:
            details.progress(starting_info)

        # .. while all others get an event
        self.publish(starting_topic,
                     starting_info,
                     options=PublishOptions(exclude=details.caller))

        # now actually fork the worker ..
        #
        self.log.info('{worker_logname} "{worker_id}" process starting ..',
                      worker_logname=worker_logname,
                      worker_id=worker_id)
        self.log.debug(
            '{worker_logname} "{worker_id}" process using command line "{cli}" ..',
            worker_logname=worker_logname,
            worker_id=worker_id,
            cli=' '.join(args))

        d = ep.connect(transport_factory)

        def on_connect_success(proto):

            # this seems to be called immediately when the child process
            # has been forked. even if it then immediately fails because
            # e.g. the executable doesn't even exist. in other words,
            # I'm not sure under what conditions the deferred will
            # errback - probably only if the forking of a new process fails
            # at OS level due to out of memory conditions or such.
            self.log.debug('{worker_logname} "{worker_id}" connected',
                           worker_logname=worker_logname,
                           worker_id=worker_id)

            # do not comment this: it will lead to on_worker_started being called
            # _before_ on_worker_connected, and we don't need it!
            # worker.on_worker_connected(proto)

        def on_connect_error(err):

            # not sure when this errback is triggered at all .. see above.
            self.log.failure(
                "Internal error: connection to forked guest worker failed ({log_failure.value})",
            )

            # in any case, forward the error ..
            worker.ready.errback(err)

        d.addCallbacks(on_connect_success, on_connect_error)

        return worker.ready
Example #4
0
    def start_guest(self, id, config, details=None):
        """
        Start a new guest process on this node.

        :param config: The guest process configuration.
        :type config: obj

        :returns: int -- The PID of the new process.
        """
        # prohibit starting a worker twice
        #
        if id in self._workers:
            emsg = "Could not start worker: a worker with ID '{}' is already running (or starting)".format(id)
            self.log.error(emsg)
            raise ApplicationError(u'crossbar.error.worker_already_running', emsg)

        try:
            checkconfig.check_guest(config)
        except Exception as e:
            raise ApplicationError(u'crossbar.error.invalid_configuration', 'invalid guest worker configuration: {}'.format(e))

        options = config.get('options', {})

        # guest process working directory
        #
        workdir = self._node._cbdir
        if 'workdir' in options:
            workdir = os.path.join(workdir, options['workdir'])
        workdir = os.path.abspath(workdir)

        # guest process executable and command line arguments
        #

        # first try to configure the fully qualified path for the guest
        # executable by joining workdir and configured exectuable ..
        exe = os.path.abspath(os.path.join(workdir, config['executable']))

        if check_executable(exe):
            self.log.info("Using guest worker executable '{exe}' (executable path taken from configuration)",
                          exe=exe)
        else:
            # try to detect the fully qualified path for the guest
            # executable by doing a "which" on the configured executable name
            exe = shutil.which(config['executable'])
            if exe is not None and check_executable(exe):
                self.log.info("Using guest worker executable '{exe}' (executable path detected from environment)",
                              exe=exe)
            else:
                emsg = "Could not start worker: could not find and executable for '{}'".format(config['executable'])
                self.log.error(emsg)
                raise ApplicationError(u'crossbar.error.invalid_configuration', emsg)

        # guest process command line arguments
        #
        args = [exe]
        args.extend(config.get('arguments', []))

        # guest process environment
        #
        worker_env = create_process_env(options)

        # log name of worker
        #
        worker_logname = 'Guest'

        # topic URIs used (later)
        #
        starting_topic = 'crossbar.node.{}.on_guest_starting'.format(self._node_id)
        started_topic = 'crossbar.node.{}.on_guest_started'.format(self._node_id)

        # add worker tracking instance to the worker map ..
        #
        worker = GuestWorkerProcess(self, id, details.caller, keeplog=options.get('traceback', None))

        self._workers[id] = worker

        # create a (custom) process endpoint
        #
        ep = WorkerProcessEndpoint(self._node._reactor, exe, args, path=workdir, env=worker_env, worker=worker)

        # ready handling
        #
        def on_ready_success(proto):

            worker.pid = proto.transport.pid
            worker.status = 'started'
            worker.started = datetime.utcnow()

            self.log.info("{worker} with ID '{id}' and PID {pid} started",
                          worker=worker_logname, id=worker.id, pid=worker.pid)

            self._node._reactor.addSystemEventTrigger(
                'before', 'shutdown',
                self._cleanup_worker, self._node._reactor, worker,
            )

            # directory watcher
            #
            if 'watch' in options:

                if HAS_FS_WATCHER:

                    # assemble list of watched directories
                    watched_dirs = []
                    for d in options['watch'].get('directories', []):
                        watched_dirs.append(os.path.abspath(os.path.join(self._node._cbdir, d)))

                    worker.watch_timeout = options['watch'].get('timeout', 1)

                    # create a filesystem watcher
                    worker.watcher = FilesystemWatcher(workdir, watched_dirs=watched_dirs)

                    # make sure to stop the watch upon Twisted being shut down
                    def on_shutdown():
                        worker.watcher.stop()

                    self._node._reactor.addSystemEventTrigger('before', 'shutdown', on_shutdown)

                    # this handler will get fired by the watcher upon detecting an FS event
                    def on_filesystem_change(fs_event):
                        worker.watcher.stop()
                        proto.signal('TERM')

                        if options['watch'].get('action', None) == 'restart':
                            self.log.info("Filesystem watcher detected change {fs_event} - restarting guest in {watch_timeout} seconds ..", fs_event=fs_event, watch_timeout=worker.watch_timeout)
                            # Add a timeout large enough (perhaps add a config option later)
                            self._node._reactor.callLater(worker.watch_timeout, self.start_guest, id, config, details)
                            # Shut the worker down, after the restart event is scheduled
                            # FIXME: all workers should have a stop() method ..
                            # -> 'GuestWorkerProcess' object has no attribute 'stop'
                            # worker.stop()
                        else:
                            self.log.info("Filesystem watcher detected change {fs_event} - no action taken!", fs_event=fs_event)

                    # now start watching ..
                    worker.watcher.start(on_filesystem_change)

                else:
                    self.log.warn("Cannot watch directories for changes - feature not available")

            # assemble guest worker startup information
            #
            started_info = {
                u'id': worker.id,
                u'status': worker.status,
                u'started': utcstr(worker.started),
                u'who': worker.who,
            }

            self.publish(started_topic, started_info, options=PublishOptions(exclude=details.caller))

            return started_info

        def on_ready_error(err):
            del self._workers[worker.id]

            emsg = 'Failed to start guest worker: {}'.format(err.value)
            self.log.error(emsg)
            raise ApplicationError(u"crossbar.error.cannot_start", emsg, ep.getlog())

        worker.ready.addCallbacks(on_ready_success, on_ready_error)

        def on_exit_success(res):
            self.log.info("Guest {id} exited with success", id=worker.id)
            del self._workers[worker.id]

        def on_exit_error(err):
            self.log.error("Guest {id} exited with error {err.value}",
                           id=worker.id, err=err)
            del self._workers[worker.id]

        worker.exit.addCallbacks(on_exit_success, on_exit_error)

        # create a transport factory for talking WAMP to the native worker
        #
        transport_factory = create_guest_worker_client_factory(config, worker.ready, worker.exit)
        transport_factory.noisy = False
        self._workers[id].factory = transport_factory

        # now (immediately before actually forking) signal the starting of the worker
        #
        starting_info = {
            u'id': id,
            u'status': worker.status,
            u'created': utcstr(worker.created),
            u'who': worker.who,
        }

        # the caller gets a progressive result ..
        if details.progress:
            details.progress(starting_info)

        # .. while all others get an event
        self.publish(starting_topic, starting_info, options=PublishOptions(exclude=details.caller))

        # now actually fork the worker ..
        #
        self.log.info("Starting {worker} with ID '{id}'...",
                      worker=worker_logname, id=id)
        self.log.debug("{worker} '{id}' using command line '{cli}'...",
                       worker=worker_logname, id=id, cli=' '.join(args))

        d = ep.connect(transport_factory)

        def on_connect_success(proto):

            # this seems to be called immediately when the child process
            # has been forked. even if it then immediately fails because
            # e.g. the executable doesn't even exist. in other words,
            # I'm not sure under what conditions the deferred will
            # errback - probably only if the forking of a new process fails
            # at OS level due to out of memory conditions or such.

            pid = proto.transport.pid
            self.log.debug("Guest worker process connected with PID {pid}",
                           pid=pid)

            worker.pid = pid

            # proto is an instance of GuestWorkerClientProtocol
            worker.proto = proto

            worker.status = 'connected'
            worker.connected = datetime.utcnow()

        def on_connect_error(err):

            # not sure when this errback is triggered at all .. see above.
            self.log.failure(
                "Internal error: connection to forked guest worker failed ({log_failure.value})",
            )

            # in any case, forward the error ..
            worker.ready.errback(err)

        d.addCallbacks(on_connect_success, on_connect_error)

        return worker.ready
Example #5
0
    def start_guest(self, id, config, details=None):
        """
      Start a new guest process on this node.

      :param config: The guest process configuration.
      :type config: obj

      :returns: int -- The PID of the new process.
      """
        ## prohibit starting a worker twice
        ##
        if id in self._workers:
            emsg = "ERROR: could not start worker - a worker with ID '{}'' is already running (or starting)".format(
                id)
            log.msg(emsg)
            raise ApplicationError('crossbar.error.worker_already_running',
                                   emsg)

        try:
            checkconfig.check_guest(config)
        except Exception as e:
            raise ApplicationError(
                'crossbar.error.invalid_configuration',
                'invalid guest worker configuration: {}'.format(e))

        options = config.get('options', {})

        ## guest process executable and command line arguments
        ##
        exe = config['executable']
        args = [exe]
        args.extend(config.get('arguments', []))

        ## guest process working directory
        ##
        workdir = self._node._cbdir
        if 'workdir' in options:
            workdir = os.path.join(workdir, options['workdir'])
        workdir = os.path.abspath(workdir)

        ## guest process environment
        ##
        worker_env = create_process_env(options)

        ## log name of worker
        ##
        worker_logname = 'Guest'

        ## topic URIs used (later)
        ##
        starting_topic = 'crossbar.node.{}.on_guest_starting'.format(
            self._node_id)
        started_topic = 'crossbar.node.{}.on_guest_started'.format(
            self._node_id)

        ## add worker tracking instance to the worker map ..
        ##
        worker = GuestWorkerProcess(self,
                                    id,
                                    details.authid,
                                    keeplog=options.get('traceback', None))

        self._workers[id] = worker

        ## create a (custom) process endpoint
        ##
        ep = WorkerProcessEndpoint(self._node._reactor,
                                   exe,
                                   args,
                                   path=workdir,
                                   env=worker_env,
                                   worker=worker)

        ## ready handling
        ##
        def on_ready_success(proto):

            worker.pid = proto.transport.pid
            worker.status = 'started'
            worker.started = datetime.utcnow()

            log.msg("{} with ID '{}' and PID {} started".format(
                worker_logname, worker.id, worker.pid))

            ## directory watcher
            ##
            if 'watch' in options:

                if _HAS_FSNOTIFY:

                    ## assemble list of watched directories
                    watched_dirs = []
                    for d in options['watch'].get('directories', []):
                        watched_dirs.append(
                            os.path.abspath(os.path.join(self._node._cbdir,
                                                         d)))

                    ## create a directory watcher
                    worker.watcher = DirWatcher(dirs=watched_dirs,
                                                notify_once=True)

                    ## make sure to stop the background thread running inside the
                    ## watcher upon Twisted being shut down
                    def on_shutdown():
                        worker.watcher.stop()

                    reactor.addSystemEventTrigger('before', 'shutdown',
                                                  on_shutdown)

                    ## this handler will get fired by the watcher upon detecting an FS event
                    def on_fsevent(evt):
                        worker.watcher.stop()
                        proto.signal('TERM')

                        if options['watch'].get('action', None) == 'restart':
                            log.msg("Restarting guest ..")
                            reactor.callLater(0.1, self.start_guest, id,
                                              config, details)

                    ## now run the watcher on a background thread
                    deferToThread(worker.watcher.loop, on_fsevent)

                else:
                    log.msg(
                        "Warning: cannot watch directory for changes - feature DirWatcher unavailable"
                    )

            ## assemble guest worker startup information
            ##
            started_info = {
                'id': worker.id,
                'status': worker.status,
                'started': utcstr(worker.started),
                'who': worker.who
            }

            self.publish(started_topic,
                         started_info,
                         options=PublishOptions(exclude=[details.caller]))

            return started_info

        def on_ready_error(err):
            del self._workers[worker.id]

            emsg = 'ERROR: failed to start guest worker - {}'.format(err.value)
            log.msg(emsg)
            raise ApplicationError("crossbar.error.cannot_start", emsg,
                                   ep.getlog())

        worker.ready.addCallbacks(on_ready_success, on_ready_error)

        def on_exit_success(res):
            log.msg("Guest excited with success")
            del self._workers[worker.id]

        def on_exit_error(err):
            log.msg("Guest excited with error", err)
            del self._workers[worker.id]

        worker.exit.addCallbacks(on_exit_success, on_exit_error)

        ## create a transport factory for talking WAMP to the native worker
        ##
        transport_factory = create_guest_worker_client_factory(
            config, worker.ready, worker.exit)
        transport_factory.noisy = False
        self._workers[id].factory = transport_factory

        ## now (immediately before actually forking) signal the starting of the worker
        ##
        starting_info = {
            'id': id,
            'status': worker.status,
            'created': utcstr(worker.created),
            'who': worker.who
        }

        ## the caller gets a progressive result ..
        if details.progress:
            details.progress(starting_info)

        ## .. while all others get an event
        self.publish(starting_topic,
                     starting_info,
                     options=PublishOptions(exclude=[details.caller]))

        ## now actually fork the worker ..
        ##
        if self.debug:
            log.msg(
                "Starting {} with ID '{}' using command line '{}' ..".format(
                    worker_logname, id, ' '.join(args)))
        else:
            log.msg("Starting {} with ID '{}' ..".format(worker_logname, id))

        d = ep.connect(transport_factory)

        def on_connect_success(proto):

            ## this seems to be called immediately when the child process
            ## has been forked. even if it then immediately fails because
            ## e.g. the executable doesn't even exist. in other words,
            ## I'm not sure under what conditions the deferred will
            ## errback - probably only if the forking of a new process fails
            ## at OS level due to out of memory conditions or such.

            pid = proto.transport.pid
            if self.debug:
                log.msg(
                    "Guest worker process connected with PID {}".format(pid))

            worker.pid = pid

            ## proto is an instance of GuestWorkerClientProtocol
            worker.proto = proto

            worker.status = 'connected'
            worker.connected = datetime.utcnow()

        def on_connect_error(err):

            ## not sure when this errback is triggered at all .. see above.
            if self.debug:
                log.msg(
                    "ERROR: Connecting forked guest worker failed - {}".format(
                        err))

            ## in any case, forward the error ..
            worker.ready.errback(err)

        d.addCallbacks(on_connect_success, on_connect_error)

        return worker.ready
Example #6
0
    def start_guest(self, id, config, details=None):
        """
        Start a new guest process on this node.

        :param config: The guest process configuration.
        :type config: obj

        :returns: int -- The PID of the new process.
        """
        # prohibit starting a worker twice
        #
        if id in self._workers:
            emsg = "Could not start worker: a worker with ID '{}' is already running (or starting)".format(id)
            self.log.error(emsg)
            raise ApplicationError(u"crossbar.error.worker_already_running", emsg)

        try:
            checkconfig.check_guest(config)
        except Exception as e:
            raise ApplicationError(
                u"crossbar.error.invalid_configuration", "invalid guest worker configuration: {}".format(e)
            )

        options = config.get("options", {})

        # guest process working directory
        #
        workdir = self._node._cbdir
        if "workdir" in options:
            workdir = os.path.join(workdir, options["workdir"])
        workdir = os.path.abspath(workdir)

        # guest process executable and command line arguments
        #

        # first try to configure the fully qualified path for the guest
        # executable by joining workdir and configured exectuable ..
        exe = os.path.abspath(os.path.join(workdir, config["executable"]))

        if check_executable(exe):
            self.log.info("Using guest worker executable '{exe}' (executable path taken from configuration)", exe=exe)
        else:
            # try to detect the fully qualified path for the guest
            # executable by doing a "which" on the configured executable name
            exe = shutil.which(config["executable"])
            if exe is not None and check_executable(exe):
                self.log.info(
                    "Using guest worker executable '{exe}' (executable path detected from environment)", exe=exe
                )
            else:
                emsg = "Could not start worker: could not find and executable for '{}'".format(config["executable"])
                self.log.error(emsg)
                raise ApplicationError(u"crossbar.error.invalid_configuration", emsg)

        # guest process command line arguments
        #
        args = [exe]
        args.extend(config.get("arguments", []))

        # guest process environment
        #
        worker_env = create_process_env(options)

        # log name of worker
        #
        worker_logname = "Guest"

        # topic URIs used (later)
        #
        starting_topic = "crossbar.node.{}.on_guest_starting".format(self._node_id)
        started_topic = "crossbar.node.{}.on_guest_started".format(self._node_id)

        # add worker tracking instance to the worker map ..
        #
        worker = GuestWorkerProcess(self, id, details.caller, keeplog=options.get("traceback", None))

        self._workers[id] = worker

        # create a (custom) process endpoint
        #
        ep = WorkerProcessEndpoint(self._node._reactor, exe, args, path=workdir, env=worker_env, worker=worker)

        # ready handling
        #
        def on_ready_success(proto):

            worker.pid = proto.transport.pid
            worker.status = "started"
            worker.started = datetime.utcnow()

            self.log.info(
                "{worker} with ID '{id}' and PID {pid} started", worker=worker_logname, id=worker.id, pid=worker.pid
            )

            self._node._reactor.addSystemEventTrigger(
                "before", "shutdown", self._cleanup_worker, self._node._reactor, worker
            )

            # directory watcher
            #
            if "watch" in options:

                if HAS_FSNOTIFY:

                    # assemble list of watched directories
                    watched_dirs = []
                    for d in options["watch"].get("directories", []):
                        watched_dirs.append(os.path.abspath(os.path.join(self._node._cbdir, d)))

                    worker.watch_timeout = options["watch"].get("timeout", 1)

                    # create a directory watcher
                    worker.watcher = DirWatcher(dirs=watched_dirs, notify_once=True)

                    # make sure to stop the background thread running inside the
                    # watcher upon Twisted being shut down
                    def on_shutdown():
                        worker.watcher.stop()

                    self._node._reactor.addSystemEventTrigger("before", "shutdown", on_shutdown)

                    # this handler will get fired by the watcher upon detecting an FS event
                    def on_fsevent(evt):
                        worker.watcher.stop()
                        proto.signal("TERM")

                        if options["watch"].get("action", None) == "restart":
                            self.log.info("Restarting guest ..")
                            # Add a timeout large enough (perhaps add a config option later)
                            self._node._reactor.callLater(worker.watch_timeout, self.start_guest, id, config, details)
                            # Shut the worker down, after the restart event is scheduled
                            worker.stop()

                    # now run the watcher on a background thread
                    deferToThread(worker.watcher.loop, on_fsevent)

                else:
                    self.log.warn("Warning: cannot watch directory for changes - feature DirWatcher unavailable")

            # assemble guest worker startup information
            #
            started_info = {
                u"id": worker.id,
                u"status": worker.status,
                u"started": utcstr(worker.started),
                u"who": worker.who,
            }

            self.publish(started_topic, started_info, options=PublishOptions(exclude=details.caller))

            return started_info

        def on_ready_error(err):
            del self._workers[worker.id]

            emsg = "Failed to start guest worker: {}".format(err.value)
            self.log.error(emsg)
            raise ApplicationError(u"crossbar.error.cannot_start", emsg, ep.getlog())

        worker.ready.addCallbacks(on_ready_success, on_ready_error)

        def on_exit_success(res):
            self.log.info("Guest {id} exited with success", id=worker.id)
            del self._workers[worker.id]

        def on_exit_error(err):
            self.log.error("Guest {id} exited with error {err.value}", id=worker.id, err=err)
            del self._workers[worker.id]

        worker.exit.addCallbacks(on_exit_success, on_exit_error)

        # create a transport factory for talking WAMP to the native worker
        #
        transport_factory = create_guest_worker_client_factory(config, worker.ready, worker.exit)
        transport_factory.noisy = False
        self._workers[id].factory = transport_factory

        # now (immediately before actually forking) signal the starting of the worker
        #
        starting_info = {u"id": id, u"status": worker.status, u"created": utcstr(worker.created), u"who": worker.who}

        # the caller gets a progressive result ..
        if details.progress:
            details.progress(starting_info)

        # .. while all others get an event
        self.publish(starting_topic, starting_info, options=PublishOptions(exclude=details.caller))

        # now actually fork the worker ..
        #
        self.log.info("Starting {worker} with ID '{id}'...", worker=worker_logname, id=id)
        self.log.debug(
            "{worker} '{id}' using command line '{cli}'...", worker=worker_logname, id=id, cli=" ".join(args)
        )

        d = ep.connect(transport_factory)

        def on_connect_success(proto):

            # this seems to be called immediately when the child process
            # has been forked. even if it then immediately fails because
            # e.g. the executable doesn't even exist. in other words,
            # I'm not sure under what conditions the deferred will
            # errback - probably only if the forking of a new process fails
            # at OS level due to out of memory conditions or such.

            pid = proto.transport.pid
            self.log.debug("Guest worker process connected with PID {pid}", pid=pid)

            worker.pid = pid

            # proto is an instance of GuestWorkerClientProtocol
            worker.proto = proto

            worker.status = "connected"
            worker.connected = datetime.utcnow()

        def on_connect_error(err):

            # not sure when this errback is triggered at all .. see above.
            self.log.error("Internal error: connection to forked guest worker failed ({})".format(err))

            # in any case, forward the error ..
            worker.ready.errback(err)

        d.addCallbacks(on_connect_success, on_connect_error)

        return worker.ready