Esempio n. 1
0
    def run(self):
        """Entry point for the VELES execution engine.
        """
        veles.validate_environment()

        ret = self._process_special_args()
        if ret is not None:
            return ret
        parser = Main.init_parser()
        args = parser.parse_args(self.argv)
        self._apply_args(args)

        self.setup_logging(args.verbosity)
        self._print_logo(args)
        for name in filter(str.strip, args.debug.split(',')):
            logging.getLogger(name).setLevel(logging.DEBUG)
        self._seed_random(args.random_seed)
        if args.debug_pickle:
            setup_pickle_debug()
        ThreadPool.reset()
        self._register_print_max_rss()

        if self.logger.isEnabledFor(logging.DEBUG):
            self._print_config(root)
        wm = self._load_model(self.workflow_file)
        self._apply_config(self.config_file)
        self._override_config(args.config_list)
        if self.logger.isEnabledFor(logging.DEBUG):
            self._print_config(root)

        self._run_core(wm)

        if not self.interactive:
            self.info("End of job")
        else:
            self.info("\033[1;35mReturned the control\033[0m")
        return Main.EXIT_SUCCESS
Esempio n. 2
0
    def run(self):
        """Entry point for the VELES execution engine.
        """
        veles.validate_environment()

        ret = self._process_special_args()
        if ret is not None:
            return ret
        parser = Main.init_parser()
        args = parser.parse_args(self.argv)
        self._apply_args(args)

        self.setup_logging(args.verbosity)
        self._print_logo(args)
        for name in filter(str.strip, args.debug.split(',')):
            logging.getLogger(name).setLevel(logging.DEBUG)
        self._seed_random(args.random_seed)
        if args.debug_pickle:
            setup_pickle_debug()
        ThreadPool.reset()
        self._register_print_max_rss()

        if self.logger.isEnabledFor(logging.DEBUG):
            self._print_config(root)
        wm = self._load_model(self.workflow_file)
        self._apply_config(self.config_file)
        self._override_config(args.config_list)
        if self.logger.isEnabledFor(logging.DEBUG):
            self._print_config(root)

        self._run_core(wm)

        if not self.interactive:
            self.info("End of job")
        else:
            self.info("\033[1;35mReturned the control\033[0m")
        return Main.EXIT_SUCCESS
Esempio n. 3
0
    def initialize(self, snapshot=False, **kwargs):
        # Ensure reactor stops in some rare cases when it does not normally
        if not self.interactive:
            self.workflow.thread_pool.register_on_shutdown(
                Launcher._reactor_shutdown)
        else:
            self._interactive_shutdown_ref = self._interactive_shutdown
            ThreadPool.register_atexit(self._interactive_shutdown_ref)
        if self.is_slave:
            self._agent = SlaveManager(self.args.master_address, self.workflow)

            def on_id_received(node_id, log_id):
                self.id = node_id
                self.log_id = log_id
                if self.logs_to_mongo:
                    logger.Logger.duplicate_all_logging_to_mongo(
                        self.args.log_mongo, self.log_id, node_id)

            self.agent.on_id_received = on_id_received
        else:
            if self.reports_web_status:
                timeout = self._notify_update_interval / 2
                self._web_status_agent = Agent(
                    reactor, pool=HTTPConnectionPool(reactor),
                    connectTimeout=timeout)
                # Launch the status server if it's not been running yet
                self._launch_status()
            if self.workflow.plotters_are_enabled and \
                    (not self.interactive or Launcher.graphics_client is None):
                try:
                    Launcher.graphics_server, Launcher.graphics_client = \
                        graphics_server.GraphicsServer.launch(
                            self.workflow.thread_pool,
                            self.matplotlib_backend,
                            self._set_webagg_port,
                            self.args.no_graphics_client)
                except Exception as e:
                    self.error("Failed to create the graphics server and/or "
                               "client. Try to completely disable plotting "
                               "with -p ''.")
                    raise from_none(e)
            elif self.args.no_graphics_client:
                self.warning("Plotters are disabled. --no-graphics-client has "
                             "no effect.")
            if self.is_master:
                try:
                    self._agent = MasterManager(self.args.listen_address,
                                                self.workflow)
                    # Launch the nodes described in the command line or config
                    self._launch_nodes()
                except Exception as e:
                    self._stop_graphics()
                    raise from_none(e)
        # The last moment when we can do this, because OpenCL device curses
        # new process creation
        try:
            self._generate_workflow_graphs()
        except Exception as e:
            self.error("Failed to generate the workflow graph(s)")
            self._stop_graphics()
            raise from_none(e)
        try:
            if not self.is_master and not kwargs.get("no_device", False):
                self._device = Device()
        except Exception as e:
            self.error("Failed to create the OpenCL device")
            self._stop_graphics()
            raise from_none(e)
        if "no_device" in kwargs:
            del kwargs["no_device"]
        self.workflow.reset_thread_pool()

        def greet_reactor():
            def set_thread_ident():
                self._reactor_thread_ident = threading.current_thread().ident

            reactor.callWhenRunning(self.info, "Reactor is running")
            reactor.callWhenRunning(set_thread_ident)

        def initialize_workflow():
            try:
                self.workflow.initialize(device=self.device, snapshot=snapshot,
                                         **kwargs)
            except Exception as ie:
                self.error("Failed to initialize the workflow")
                self._stop_graphics()
                self.device_thread_pool_detach()
                raise from_none(ie)

        if not self.interactive:
            # delay greet_reactor() until everything else is initialized
            initialize_workflow()
        else:
            greet_reactor()
            reactor.callWhenRunning(initialize_workflow)

        if not self.is_standalone:
            self._agent.initialize()
        if not self.interactive:
            trigger = reactor.addSystemEventTrigger
            trigger('before', 'shutdown', self._on_stop)
            trigger('after', 'shutdown', self._print_stats)
            trigger('after', 'shutdown', self.event, "work", "end", height=0.1)
        else:
            register = self.workflow.thread_pool.register_on_shutdown
            self._on_stop_ref = self._on_stop
            register(self._on_stop_ref)
            self._print_stats_ref = self._print_stats
            register(self._print_stats_ref)

            def work_end():
                self.event("work", "end", height=0.1)
            self._work_end = work_end
            register(self._work_end)
        for unit in self.workflow:
            if isinstance(unit, Plotter):
                unit.graphics_server = Launcher.graphics_server
        greet_reactor()
        self._initialized = True
Esempio n. 4
0
    def initialize(self, snapshot=False, **kwargs):
        # Ensure reactor stops in some rare cases when it does not normally
        if not self.interactive:
            self.workflow.thread_pool.register_on_shutdown(
                Launcher._reactor_shutdown)
        else:
            self._interactive_shutdown_ref = self._interactive_shutdown
            ThreadPool.register_atexit(self._interactive_shutdown_ref)
        if self.is_slave:
            self._agent = SlaveManager(self.args.master_address, self.workflow)

            def on_id_received(node_id, log_id):
                self.id = node_id
                self.log_id = log_id
                if self.logs_to_mongo:
                    logger.Logger.duplicate_all_logging_to_mongo(
                        self.args.log_mongo, self.log_id, node_id)

            self.agent.on_id_received = on_id_received
        else:
            if self.reports_web_status:
                timeout = self._notify_update_interval / 2
                self._web_status_agent = Agent(
                    reactor,
                    pool=HTTPConnectionPool(reactor),
                    connectTimeout=timeout)
                # Launch the status server if it's not been running yet
                self._launch_status()
            if self.workflow.plotters_are_enabled and \
                    (not self.interactive or Launcher.graphics_client is None):
                try:
                    Launcher.graphics_server, Launcher.graphics_client = \
                        graphics_server.GraphicsServer.launch(
                            self.workflow.thread_pool,
                            self.matplotlib_backend,
                            self._set_webagg_port,
                            self.args.no_graphics_client)
                except Exception as e:
                    self.error("Failed to create the graphics server and/or "
                               "client. Try to completely disable plotting "
                               "with -p ''.")
                    raise from_none(e)
            elif self.args.no_graphics_client:
                self.warning("Plotters are disabled. --no-graphics-client has "
                             "no effect.")
            if self.is_master:
                try:
                    self._agent = MasterManager(self.args.listen_address,
                                                self.workflow)
                    # Launch the nodes described in the command line or config
                    self._launch_nodes()
                except Exception as e:
                    self._stop_graphics()
                    raise from_none(e)
        # The last moment when we can do this, because OpenCL device curses
        # new process creation
        try:
            self._generate_workflow_graphs()
        except Exception as e:
            self.error("Failed to generate the workflow graph(s)")
            self._stop_graphics()
            raise from_none(e)
        try:
            if not self.is_master and not kwargs.get("no_device", False):
                self._device = Device()
        except Exception as e:
            self.error("Failed to create the OpenCL device")
            self._stop_graphics()
            raise from_none(e)
        if "no_device" in kwargs:
            del kwargs["no_device"]
        self.workflow.reset_thread_pool()

        def greet_reactor():
            def set_thread_ident():
                self._reactor_thread_ident = threading.current_thread().ident

            reactor.callWhenRunning(self.info, "Reactor is running")
            reactor.callWhenRunning(set_thread_ident)

        def initialize_workflow():
            try:
                self.workflow.initialize(device=self.device,
                                         snapshot=snapshot,
                                         **kwargs)
            except Exception as ie:
                self.error("Failed to initialize the workflow")
                self._stop_graphics()
                self.device_thread_pool_detach()
                raise from_none(ie)

        if not self.interactive:
            # delay greet_reactor() until everything else is initialized
            initialize_workflow()
        else:
            greet_reactor()
            reactor.callWhenRunning(initialize_workflow)

        if not self.is_standalone:
            self._agent.initialize()
        if not self.interactive:
            trigger = reactor.addSystemEventTrigger
            trigger('before', 'shutdown', self._on_stop)
            trigger('after', 'shutdown', self._print_stats)
            trigger('after', 'shutdown', self.event, "work", "end", height=0.1)
        else:
            register = self.workflow.thread_pool.register_on_shutdown
            self._on_stop_ref = self._on_stop
            register(self._on_stop_ref)
            self._print_stats_ref = self._print_stats
            register(self._print_stats_ref)

            def work_end():
                self.event("work", "end", height=0.1)

            self._work_end = work_end
            register(self._work_end)
        for unit in self.workflow:
            if isinstance(unit, Plotter):
                unit.graphics_server = Launcher.graphics_server
        greet_reactor()
        self._initialized = True