Esempio n. 1
0
 def _load(self, Workflow, **kwargs):
     self.debug("load() was called from run(), workflow class is %s",
                str(Workflow))
     self.load_called = True
     try:
         self.launcher = Launcher(self.interactive)
         self.launcher.workflow_file = self.workflow_file
         self.launcher.config_file = self.config_file
         self.launcher.seeds = self.seeds
     except:
         self.exception("Failed to create the launcher")
         sys.exit(Main.EXIT_FAILURE)
     try:
         self.workflow = self._load_workflow(self.snapshot_file_name)
         snapshot = self.workflow is not None
         if not snapshot:
             wfkw = self._get_interactive_locals()
             wfkw.update(kwargs)
             self.workflow = Workflow(self.launcher, **wfkw)
             self.info("Created %s", self.workflow)
         else:
             self.info("Loaded the workflow snapshot from %s: %s",
                       self.snapshot_file_name, self.workflow)
             if self._visualization_mode:
                 self.workflow.plotters_are_enabled = True
             self.workflow.workflow = self.launcher
     except:
         self.exception("Failed to create the workflow")
         self.launcher.stop()
         sys.exit(Main.EXIT_FAILURE)
     if self._workflow_graph:
         self.workflow.generate_graph(filename=self._workflow_graph,
                                      with_data_links=True,
                                      background='white')
     return self.workflow, snapshot
Esempio n. 2
0
 def _load(self, Workflow, **kwargs):
     self.debug("load() was called from run(), workflow class is %s",
                str(Workflow))
     self.load_called = True
     try:
         self.launcher = Launcher(self.interactive)
         self.launcher.workflow_file = self.workflow_file
         self.launcher.config_file = self.config_file
         self.launcher.seeds = self.seeds
     except:
         self.exception("Failed to create the launcher")
         sys.exit(Main.EXIT_FAILURE)
     try:
         self.workflow = self._load_workflow(self.snapshot_file_name)
         self.snapshot = self.workflow is not None
         if not self.snapshot:
             wfkw = self._get_interactive_locals()
             wfkw.update(kwargs)
             self.workflow = Workflow(self.launcher, **wfkw)
             self.info("Created %s", self.workflow)
         else:
             self.info("Loaded the workflow snapshot from %s: %s",
                       self.snapshot_file_name, self.workflow)
             if self._visualization_mode:
                 self.workflow.plotters_are_enabled = True
             self.workflow.workflow = self.launcher
     except:
         self.exception("Failed to create the workflow")
         self.launcher.stop()
         sys.exit(Main.EXIT_FAILURE)
     if self._workflow_graph:
         self.workflow.generate_graph(filename=self._workflow_graph,
                                      with_data_links=True,
                                      background='white')
     return self.workflow, self.snapshot
Esempio n. 3
0
class Main(Logger, CommandLineBase):
    """
    Entry point of any VELES engine executions.
    """

    EXIT_SUCCESS = 0
    EXIT_FAILURE = 1
    registered_print_max_rss = False

    def __init__(self, interactive=False, *args, **kwargs):
        Main.setup_argv(not interactive, True, *args, **kwargs)
        super(Main, self).__init__()
        self._interactive = interactive
        self._ensemble_train = None
        self._ensemble_test = None
        self._optimization = None

    @property
    def interactive(self):
        return self._interactive

    @property
    def optimization(self):
        return self._optimization

    @optimization.setter
    def optimization(self, value):
        if value is None:
            self._optimization = None
            return
        if not isinstance(value, tuple) or len(value) != 2:
            raise TypeError(
                "Invalid optimization value: expected a tuple of length 2, got"
                " %s" % type(value))
        if not isinstance(value[0], int) or \
                not isinstance(value[1], (int, type(None))):
            raise TypeError("Invalid optimization value types: %s", value)
        if value[0] < 5:
            raise ValueError("Population size may not be less than 5")
        if value[1] is not None and value[1] < 1:
            raise ValueError("At least 1 generation must be set")
        self._optimization = OptimizationTuple(*value)

    @property
    def ensemble_train(self):
        return self._ensemble_train

    @ensemble_train.setter
    def ensemble_train(self, value):
        if value is None:
            self._ensemble_train = None
            return
        if not isinstance(value, tuple) or len(value) != 2 or \
                not isinstance(value[0], int) or \
                not isinstance(value[1], (float, int)):
            raise ValueError("Invalid ensemble_train value")
        ratio = value[1]
        if ratio <= 0 or ratio > 1:
            raise ValueError("Training set ratio must be in (0, 1] (got %s)" %
                             ratio)
        self._ensemble_train = EnsembleTrainTuple(*value)

    @property
    def ensemble_test(self):
        return self._ensemble_test

    @ensemble_test.setter
    def ensemble_test(self, value):
        if value is None:
            self._ensemble_test = None
            return
        if not isinstance(value, string_types):
            raise TypeError("ensemble_test must be a string (got %s)" %
                            type(value))
        self._ensemble_test = EnsembleTestTuple(value)

    @property
    def regular(self):
        return not self.optimization and not self.ensemble_train and \
            not self.ensemble_test

    def _process_special_args(self):
        if "--frontend" in sys.argv:
            try:
                self._open_frontend()
            except KeyboardInterrupt:
                return Main.EXIT_FAILURE
            return self._process_special_args()
        if self.interactive:
            for opt in "forge", "--version", "--help", "--dump-config":
                if opt in self.argv:
                    raise ValueError(
                        "\"%s\" is not supported in interactive mode" % opt)
            return None
        if len(sys.argv) > 1 and sys.argv[1] == "forge":
            from veles.forge.forge_client import __run__ as forge_run
            del sys.argv[1]
            action = sys.argv[1]
            try:
                forge_run()
                return Main.EXIT_SUCCESS
            except Exception as e:
                if isinstance(e, SystemExit):
                    raise from_none(e)
                self.exception("Failed to run forge %s", action)
                return Main.EXIT_FAILURE
        if "--version" in sys.argv:
            self._print_version()
            return Main.EXIT_SUCCESS
        if "--html-help" in sys.argv:
            veles.__html__()
            return Main.EXIT_SUCCESS
        if "--help" in sys.argv:
            # help text requires UTF-8, but the default codec is ascii over ssh
            Logger.ensure_utf8_streams()
        if "--dump-config" in sys.argv:
            self.info("Scanning for the plugins...")
            self.debug("Loaded plugins: %s", veles.__plugins__)
            root.print_()
            return Main.EXIT_SUCCESS
        return None

    def _open_frontend(self):
        from multiprocessing import Process, SimpleQueue

        connection = SimpleQueue()
        frontend = Process(
            target=self._open_frontend_process,
            args=(connection, [k for k in sys.argv[1:] if k != "--frontend"]))
        frontend.start()
        cmdline = connection.get()
        frontend.join()
        if self.interactive:
            argv_backup = list(sys.argv)
        sys.argv[1:] = cmdline.split()
        Main.setup_argv(True, True)
        if self.interactive:
            sys.argv = argv_backup
        print("Running with the following command line: %s" % sys.argv)

    def _open_frontend_process(self, connection, argv):
        if not os.path.exists(
                os.path.join(root.common.web.root, "frontend.html")):
            self.info("frontend.html was not found, generating it...")
            from veles.scripts.generate_frontend import main

            main()
            gc.collect()

        from random import randint

        port = randint(1024, 65535)
        self.info("Launching the web server on localhost:%d...", port)

        from tornado.escape import json_decode
        from tornado.ioloop import IOLoop
        import tornado.web as web

        cmdline = [""]

        class CmdlineHandler(web.RequestHandler):
            def post(self):
                try:
                    data = json_decode(self.request.body)
                    cmdline[0] = data.get("cmdline")
                    IOLoop.instance().add_callback(IOLoop.instance().stop)
                except:
                    self.exception("Frontend cmdline POST")

        class FrontendHandler(web.RequestHandler):
            def get(self):
                self.render("frontend.html", cmdline=" ".join(argv))

        app = web.Application(
            [("/cmdline", CmdlineHandler),
             (r"/((js|css|fonts|img|maps)/.*)", web.StaticFileHandler, {
                 'path': root.common.web.root
             }), (r"/frontend\.html", FrontendHandler),
             ("/", web.RedirectHandler, {
                 "url": "/frontend.html",
                 "permanent": True
             }),
             ("", web.RedirectHandler, {
                 "url": "/frontend.html",
                 "permanent": True
             })],
            template_path=root.common.web.root)
        app.listen(port)

        def open_browser():
            from veles.portable import show_file

            show_file("http://localhost:%d" % port)

        IOLoop.instance().add_callback(open_browser)
        try:
            IOLoop.instance().start()
        except KeyboardInterrupt:
            sys.stderr.write("KeyboardInterrupt\n")
        finally:
            connection.put(cmdline[0])

    def _parse_optimization(self, args):
        if args.optimize is None:
            return
        optparsed = args.optimize.split(':')
        if len(optparsed) > 2:
            raise ValueError("Invalid --optimize value: %s" % args.optimize)
        try:
            self.optimization = int(optparsed[0]), \
                int(optparsed[1]) if len(optparsed) == 2 else None
        except ValueError:
            raise from_none(
                ValueError("\"%s\" is not a valid --optimize value" %
                           args.optimize))

    def _parse_ensemble_train(self, args):
        if args.ensemble_train is None:
            return

        optparsed = args.ensemble_train.split(":")
        if len(optparsed) != 2:
            raise ValueError("--ensemble-train must be specified as"
                             "<number of instances>:<training set ratio>")
        try:
            self.ensemble_train = int(optparsed[0]), float(optparsed[1])
        except ValueError:
            raise from_none(
                "Failed to parse ensemble parameters from (%s, %s)" %
                optparsed)

    def _parse_ensemble_test(self, args):
        if args.ensemble_test is None:
            return
        if self.ensemble_train is not None:
            raise ValueError(
                "--ensemble-train and --ensemble-test may not be used "
                "together")
        self.ensemble_test = args.ensemble_test

    def _daemonize(self):
        daemon_context = daemon.DaemonContext()
        daemon_context.working_directory = os.getcwd()
        daemon_context.files_preserve = [
            int(fd) for fd in os.listdir("/proc/self/fd") if int(fd) > 2
        ]
        daemon_context.open()  # <- the magic happens here

    @staticmethod
    def _get_interactive_locals():
        """
        If we are running under IPython, extracts the local variables;
        otherwise, returns an empty dict.
        """
        try:
            __IPYTHON__  # pylint: disable=E0602
            from IPython.core.interactiveshell import InteractiveShell
            return {
                k: v
                for k, v in InteractiveShell.instance().user_ns.items()
                if k[0] != '_' and k not in ("Out", "In", "exit", "quit",
                                             "get_ipython")
            }
        except NameError:
            return {}

    def _load_model(self, fname_workflow):
        self.info("Loading workflow \"%s\"...", fname_workflow)
        self.load_called = False
        self.main_called = False
        package_name, module_name = get_file_package_and_module(fname_workflow)
        try:
            return import_file_as_package(fname_workflow)
        except Exception as e:
            self.debug(
                "Failed to import \"%s\" through the parent package "
                "\"%s\": %s", fname_workflow, package_name, e)
            package_import_error = e
        # We failed to load the package => try module approach
        try:
            return import_file_as_module(fname_workflow)
        except FileNotFoundError:
            self.exception("Workflow does not exist: \"%s\"", fname_workflow)
            sys.exit(errno.ENOENT)
        except IsADirectoryError:
            self.exception("Workflow \"%s\" is a directory", fname_workflow)
            sys.exit(errno.EISDIR)
        except PermissionError:
            self.exception("Cannot read workflow \"%s\"", fname_workflow)
            sys.exit(errno.EACCES)
        except Exception as e:
            self.error(
                "Failed to load the workflow \"%s\".\n"
                "Package import error: %s\nModule import error: %s",
                fname_workflow, package_import_error, e)
            sys.exit(Main.EXIT_FAILURE)

    def _apply_config(self, fname_config):
        if not self.config_file:
            self.warning("Configuration path is empty")
            return

        def fail():
            self.exception("Failed to apply the configuration \"%s\"",
                           fname_config)
            sys.exit(Main.EXIT_FAILURE)

        self.info("Applying the configuration from %s...", fname_config)
        try:
            runpy.run_path(fname_config)
        except FileNotFoundError:
            self.exception("Configuration does not exist: \"%s\"",
                           fname_config)
            sys.exit(errno.ENOENT)
        except IsADirectoryError:
            self.exception("Configuration \"%s\" is a directory", fname_config)
            sys.exit(errno.EISDIR)
        except PermissionError:
            self.exception("Cannot read configuration \"%s\"", fname_config)
            sys.exit(errno.EACCES)
        except TypeError as e:
            self.debug("Filed to import \"%s\": %s -> assumed pickle",
                       fname_config, e)
            from veles.pickle2 import pickle
            try:
                with open(fname_config, "rb") as fin:
                    cfg = pickle.load(fin)
            except:
                fail()
            for subcfg in cfg:
                root[subcfg].update(cfg[subcfg])
        except SyntaxError as e:
            self.debug("Filed to import \"%s\": %s -> assumed json",
                       fname_config, e)
            import json
            try:
                with open(fname_config, "r") as fin:
                    cfg = json.load(fin)
            except:
                fail()
            for subcfg in cfg:
                root[subcfg].update(cfg[subcfg])
        except:
            fail()

    def _override_config(self, config_list):
        cmds = "\n".join(config_list)
        self.debug("Overriding the configuration with %s", cmds)
        try:
            exec(cmds)
        except:
            self.exception("Invalid configuration overloads")
            sys.exit(Main.EXIT_FAILURE)

    def _seed_random(self, rndvals):
        self.debug("Seeding with %s", rndvals)
        rndvals_split = rndvals.split(',')
        seeds = []
        for rndval, index in zip(rndvals_split, range(len(rndvals_split))):
            try:
                binvle = binascii.unhexlify(rndval)
                seed = numpy.frombuffer(binvle, dtype=numpy.uint8)
                prng.get(index + 1).seed(seed, dtype=numpy.uint8)
                seeds.append(seed)
                continue
            except (binascii.Error, TypeError):
                pass
            vals = rndval.split(':')
            fname = vals[0]
            if fname == "":
                if index > 1:
                    fname = rndvals_split[0].split(':')[0] + str(index)
                else:
                    self.critical("Random generator file name is empty")
                    sys.exit(errno.ENOENT)
            if fname == "-":
                seeds.append(None)
                try:
                    prng.get(index + 1).seed(None)
                except:
                    self.exception(
                        "Failed to seed the random generator %d "
                        "with the last used seed.", index + 1)
                    sys.exit(Main.EXIT_FAILURE)
                continue
            if not os.path.isabs(fname):
                new_fname = os.path.abspath(fname)
                if os.path.exists(new_fname):
                    fname = new_fname
                else:
                    fname = os.path.join(root.common.dirs.veles, fname)
                    if not os.path.exists(fname):
                        self.critical(
                            "Neither %s nor %s exist. Cannot seed "
                            "the random generator.", new_fname, fname)
                        sys.exit(errno.ENOENT)
            count = int(vals[1]) if len(vals) > 1 else 16
            dtype = numpy.dtype(vals[2]) if len(vals) > 2 else numpy.int32

            self.debug("Seeding with %d samples of type %s from %s to %d",
                       count, dtype, fname, index + 1)
            try:
                seed = (numpy.fromfile(fname, dtype=dtype, count=count))
                prng.get(index + 1).seed(seed, dtype=dtype)
                seeds.append(seed)
            except:
                self.exception("Failed to seed the random generator with %s",
                               fname)
                sys.exit(Main.EXIT_FAILURE)
        self.seeds = seeds

    def _load_workflow(self, fname_snapshot):
        stype = splittype(fname_snapshot)[0]
        if stype == "odbc":
            import pyodbc

            addr = fname_snapshot[7:]
            parsed = addr.split('&')
            try:
                odbc, table, id_, log_id = parsed[:4]
            except TypeError:
                self.warning("Invalid ODBC source format. Here is the "
                             "template: odbc://<odbc data source spec>&"
                             "<table>&<id>&<log id>[&<optional name>]\n"
                             "<table> and <log id> may be empty (\"veles\" and"
                             " <id> value will be used).")
                return None
            if not table:
                table = "veles"
            if not log_id:
                log_id = id_
            if len(parsed) > 4:
                if len(parsed) > 5:
                    self.warning("Invalid ODBC source format")
                    return None
                name = parsed[-1]
            else:
                name = None
            try:
                return SnapshotterToDB.import_(odbc, table, id_, log_id, name)
            except pyodbc.Error as e:
                self.warning(
                    "Failed to load the snapshot from ODBC source: %s", e)
                return None
        elif stype in ("http", "https"):
            try:
                self.info("Downloading %s...", fname_snapshot)
                fname_snapshot = self.snapshot_file_name = wget.download(
                    fname_snapshot, root.common.dirs.snapshots)
                print()
                sys.stdout.flush()
            except:
                self.exception("Failed to fetch the snapshot at \"%s\"",
                               fname_snapshot)
                return None
        try:
            return SnapshotterToFile.import_(fname_snapshot)
        except FileNotFoundError:
            if fname_snapshot.strip() != "":
                self.warning("Workflow snapshot %s does not exist",
                             fname_snapshot)
            return None

    def _load(self, Workflow, **kwargs):
        self.debug("load() was called from run(), workflow class is %s",
                   str(Workflow))
        self.load_called = True
        try:
            self.launcher = Launcher(self.interactive)
            self.launcher.workflow_file = self.workflow_file
            self.launcher.config_file = self.config_file
            self.launcher.seeds = self.seeds
        except:
            self.exception("Failed to create the launcher")
            sys.exit(Main.EXIT_FAILURE)
        try:
            self.workflow = self._load_workflow(self.snapshot_file_name)
            snapshot = self.workflow is not None
            if not snapshot:
                wfkw = self._get_interactive_locals()
                wfkw.update(kwargs)
                self.workflow = Workflow(self.launcher, **wfkw)
                self.info("Created %s", self.workflow)
            else:
                self.info("Loaded the workflow snapshot from %s: %s",
                          self.snapshot_file_name, self.workflow)
                if self._visualization_mode:
                    self.workflow.plotters_are_enabled = True
                self.workflow.workflow = self.launcher
        except:
            self.exception("Failed to create the workflow")
            self.launcher.stop()
            sys.exit(Main.EXIT_FAILURE)
        if self._workflow_graph:
            self.workflow.generate_graph(filename=self._workflow_graph,
                                         with_data_links=True,
                                         background='white')
        return self.workflow, snapshot

    def _main(self, **kwargs):
        if self._dry_run < 2:
            self.launcher.stop()
            return
        self.debug("main() was called from run()")
        if not self.load_called:
            self.critical("Call load() first in run()")
            sys.exit(Main.EXIT_FAILURE)
        self.main_called = True
        kwargs["no_device"] = not self.regular

        try:
            self.launcher.initialize(**kwargs)
        except:
            self.exception("Failed to initialize the launcher.")
            self.launcher.stop()
            sys.exit(Main.EXIT_FAILURE)

        self.debug("Initialization is complete")
        try:
            if self._dump_attrs != "no":
                self._dump_unit_attributes(self._dump_attrs == "all")
            gc.collect()
            if self._dry_run > 2:
                self.debug("Running the launcher")
                self.launcher.run()
            elif self._visualization_mode:
                self.debug("Visualizing the workflow...")
                self._visualize_workflow()
        except:
            self.exception("Failed to run the workflow")
            self.launcher.stop()
            sys.exit(Main.EXIT_FAILURE)
        finally:
            self.launcher.device_thread_pool_detach()

    def _dump_unit_attributes(self, arrays=True):
        import veles.external.prettytable as prettytable
        from veles.workflow import Workflow

        self.debug("Dumping unit attributes of %s...", str(self.workflow))
        table = prettytable.PrettyTable("#", "unit", "attr", "value")
        table.align["#"] = "r"
        table.align["unit"] = "l"
        table.align["attr"] = "l"
        table.align["value"] = "l"
        table.max_width["value"] = 100
        for i, u in enumerate(self.workflow.units_in_dependency_order):
            for k, v in sorted(u.__dict__.items()):
                if k not in Workflow.HIDDEN_UNIT_ATTRS:
                    if (not arrays and hasattr(v, "__len__") and len(v) > 32
                            and not isinstance(v, str)
                            and not isinstance(v, bytes)):
                        strv = "object of class %s of length %d" % (repr(
                            v.__class__.__name__), len(v))
                    else:
                        strv = repr(v)
                    table.add_row(i, u.__class__.__name__, k, strv)
        print(table)

    def _visualize_workflow(self):
        _, file_name = self.workflow.generate_graph(with_data_links=True,
                                                    background='white')
        from veles.portable import show_file

        show_file(file_name)

        import signal

        self.launcher.graphics_client.send_signal(signal.SIGUSR2)
        from twisted.internet import reactor

        reactor.callWhenRunning(self._run_workflow_plotters)
        reactor.callWhenRunning(print_, "Press Ctrl-C when you are done...")
        reactor.run()

    def _run_workflow_plotters(self):
        from veles.plotter import Plotter

        for unit in self.workflow:
            if isinstance(unit, Plotter):
                unit.run()
        # Second loop is needed to finish with PDF
        for unit in self.workflow:
            if isinstance(unit, Plotter):
                unit.last_run_time = 0
                unit.run()
                break

    def _run_core(self, wm):
        if self._dry_run <= 0:
            return
        if not self.optimization:
            from veles.genetics import fix_config
            fix_config(root)
        if self.regular:
            self.run_module(wm)
        elif self.optimization is not None:
            import veles.genetics.optimization_workflow as workflow
            self.run_module(workflow, model=wm, **self.optimization.__dict__)
        elif self.ensemble_train is not None:
            import veles.ensemble.model_workflow as workflow
            self.run_module(workflow, model=wm, **self.ensemble_train.__dict__)
        elif self.ensemble_test is not None:
            import veles.ensemble.test_workflow as workflow
            self.run_module(workflow, model=wm, **self.ensemble_test.__dict__)
        else:
            raise NotImplementedError("Unsupported execution mode")

    def _apply_args(self, args):
        if args.background:
            self._daemonize()
        if not args.workflow:
            raise ValueError("Workflow path may not be empty")
        config_file = args.config
        if config_file == "-":
            config_file = "%s_config%s" % os.path.splitext(args.workflow)
        self.workflow_file = os.path.abspath(args.workflow)
        self.config_file = os.path.abspath(config_file) if config_file else ""
        self._visualization_mode = args.visualize
        self._workflow_graph = args.workflow_graph
        self._dry_run = Main.DRY_RUN_CHOICES.index(args.dry_run)
        self._dump_attrs = args.dump_unit_attributes
        self.snapshot_file_name = args.snapshot
        self._parse_optimization(args)
        self._parse_ensemble_train(args)
        self._parse_ensemble_test(args)

    def _print_logo(self, args):
        if not args.no_logo:
            try:
                print(Main.LOGO)
            except:
                print(Main.LOGO.replace("©", "(c)"))
            sys.stdout.flush()

    def _print_version(self):
        print(veles.__version__, formatdate(veles.__date__, True),
              veles.__git__)

    def _print_config(self, cfg):
        io = StringIO()
        cfg.print_(file=io)
        self.debug("\n%s", io.getvalue().strip())

    def setup_logging(self, verbosity):
        try:
            super(Main, self).setup_logging(Main.LOG_LEVEL_MAP[verbosity])
        except Logger.LoggerHasBeenAlreadySetUp as e:
            if not self.interactive:
                raise from_none(e)

    def _register_print_max_rss(self):
        if not Main.registered_print_max_rss:
            atexit.register(self.print_max_rss)
            Main.registered_print_max_rss = True

    @staticmethod
    def format_decimal(val):
        if val < 1000:
            return str(val)
        d, m = divmod(val, 1000)
        return Main.format_decimal(d) + " %03d" % m

    def print_max_rss(self):
        res = resource.getrusage(resource.RUSAGE_SELF)
        if Watcher.max_mem_in_use > 0:
            self.info("Peak device memory used: %s Kb",
                      self.format_decimal(Watcher.max_mem_in_use // 1000))
        self.info("Peak resident memory used: %s Kb",
                  self.format_decimal(res.ru_maxrss))

    def run_module(self, module, **kwargs):
        self.debug("Calling %s.run()...", module.__name__)
        module.run(self._load, self._main, **kwargs)
        if not self.main_called and self._dry_run > 2:
            self.warning("main() was not called by run() in %s",
                         module.__file__)

    """
    Basically, this is what each workflow module's run() should do.
    """

    def run_workflow(self, Workflow, kwargs_load=None, kwargs_main=None):
        # we should not set "{}" as default values because of the way
        # default values work: the dicts will be reused, not recreated
        if kwargs_load is None:
            kwargs_load = {}
        if kwargs_main is None:
            kwargs_main = {}
        self._load(Workflow, **kwargs_load)
        self._main(**kwargs_main)

    def run(self):
        """Entry point for the VELES execution engine.
        """
        veles.validate_environment()

        ret = self._process_special_args()
        if ret is not None:
            return ret
        parser = Main.init_parser()
        args = parser.parse_args(self.argv)
        self._apply_args(args)

        self.setup_logging(args.verbosity)
        self._print_logo(args)
        for name in filter(str.strip, args.debug.split(',')):
            logging.getLogger(name).setLevel(logging.DEBUG)
        self._seed_random(args.random_seed)
        if args.debug_pickle:
            setup_pickle_debug()
        ThreadPool.reset()
        self._register_print_max_rss()

        if self.logger.isEnabledFor(logging.DEBUG):
            self._print_config(root)
        wm = self._load_model(self.workflow_file)
        self._apply_config(self.config_file)
        self._override_config(args.config_list)
        if self.logger.isEnabledFor(logging.DEBUG):
            self._print_config(root)

        self._run_core(wm)

        if not self.interactive:
            self.info("End of job")
        else:
            self.info("\033[1;35mReturned the control\033[0m")
        return Main.EXIT_SUCCESS
Esempio n. 4
0
class Main(Logger, CommandLineBase):
    """
    Entry point of any VELES engine executions.
    """

    EXIT_SUCCESS = 0
    EXIT_FAILURE = 1
    registered_print_max_rss = False

    def __init__(self, interactive=False, *args, **kwargs):
        Main.setup_argv(not interactive, True, *args, **kwargs)
        super(Main, self).__init__()
        self._interactive = interactive
        self._ensemble_train = None
        self._ensemble_test = None
        self._optimization = None

    @property
    def interactive(self):
        return self._interactive

    @property
    def optimization(self):
        return self._optimization

    @optimization.setter
    def optimization(self, value):
        if value is None:
            self._optimization = None
            return
        if not isinstance(value, tuple) or len(value) != 2:
            raise TypeError(
                "Invalid optimization value: expected a tuple of length 2, got"
                " %s" % type(value))
        if not isinstance(value[0], int) or \
                not isinstance(value[1], (int, type(None))):
            raise TypeError("Invalid optimization value types: %s", value)
        if value[0] < 5:
            raise ValueError("Population size may not be less than 5")
        if value[1] is not None and value[1] < 1:
            raise ValueError("At least 1 generation must be set")
        self._optimization = OptimizationTuple(*value)

    @property
    def ensemble_train(self):
        return self._ensemble_train

    @ensemble_train.setter
    def ensemble_train(self, value):
        if value is None:
            self._ensemble_train = None
            return
        if not isinstance(value, tuple) or len(value) != 2 or \
                not isinstance(value[0], int) or \
                not isinstance(value[1], (float, int)):
            raise ValueError("Invalid ensemble_train value")
        ratio = value[1]
        if ratio <= 0 or ratio > 1:
            raise ValueError(
                "Training set ratio must be in (0, 1] (got %s)" % ratio)
        self._ensemble_train = EnsembleTrainTuple(*value)

    @property
    def ensemble_test(self):
        return self._ensemble_test

    @ensemble_test.setter
    def ensemble_test(self, value):
        if value is None:
            self._ensemble_test = None
            return
        if not isinstance(value, string_types):
            raise TypeError(
                "ensemble_test must be a string (got %s)" % type(value))
        self._ensemble_test = EnsembleTestTuple(value)

    @property
    def regular(self):
        return not self.optimization and not self.ensemble_train and \
            not self.ensemble_test

    def _process_special_args(self):
        if "--frontend" in sys.argv:
            try:
                self._open_frontend()
            except KeyboardInterrupt:
                return Main.EXIT_FAILURE
            return self._process_special_args()
        if self.interactive:
            for opt in "forge", "--version", "--help", "--dump-config":
                if opt in self.argv:
                    raise ValueError(
                        "\"%s\" is not supported in interactive mode" % opt)
            return None
        if len(sys.argv) > 1 and sys.argv[1] == "forge":
            from veles.forge.forge_client import __run__ as forge_run
            del sys.argv[1]
            action = sys.argv[1]
            try:
                forge_run()
                return Main.EXIT_SUCCESS
            except Exception as e:
                if isinstance(e, SystemExit):
                    raise from_none(e)
                self.exception("Failed to run forge %s", action)
                return Main.EXIT_FAILURE
        if "--version" in sys.argv:
            self._print_version()
            return Main.EXIT_SUCCESS
        if "--html-help" in sys.argv:
            veles.__html__()
            return Main.EXIT_SUCCESS
        if "--help" in sys.argv:
            # help text requires UTF-8, but the default codec is ascii over ssh
            Logger.ensure_utf8_streams()
        if "--dump-config" in sys.argv:
            self.info("Scanning for the plugins...")
            self.debug("Loaded plugins: %s", veles.__plugins__)
            root.print_()
            return Main.EXIT_SUCCESS
        return None

    def _open_frontend(self):
        from multiprocessing import Process, SimpleQueue

        connection = SimpleQueue()
        frontend = Process(
            target=self._open_frontend_process,
            args=(connection, [k for k in sys.argv[1:] if k != "--frontend"]))
        frontend.start()
        cmdline = connection.get()
        frontend.join()
        if self.interactive:
            argv_backup = list(sys.argv)
        sys.argv[1:] = cmdline.split()
        Main.setup_argv(True, True)
        if self.interactive:
            sys.argv = argv_backup
        print("Running with the following command line: %s" % sys.argv)

    def _open_frontend_process(self, connection, argv):
        if not os.path.exists(os.path.join(root.common.web.root,
                                           "frontend.html")):
            self.info("frontend.html was not found, generating it...")
            from veles.scripts.generate_frontend import main

            main()
            gc.collect()

        from random import randint

        port = randint(1024, 65535)
        self.info("Launching the web server on localhost:%d...", port)

        from tornado.escape import json_decode
        from tornado.ioloop import IOLoop
        import tornado.web as web

        cmdline = [""]

        class CmdlineHandler(web.RequestHandler):
            def post(self):
                try:
                    data = json_decode(self.request.body)
                    cmdline[0] = data.get("cmdline")
                    IOLoop.instance().add_callback(IOLoop.instance().stop)
                except:
                    self.exception("Frontend cmdline POST")

        class FrontendHandler(web.RequestHandler):
            def get(self):
                self.render("frontend.html", cmdline=" ".join(argv))

        app = web.Application([
            ("/cmdline", CmdlineHandler),
            (r"/((js|css|fonts|img|maps)/.*)",
             web.StaticFileHandler, {'path': root.common.web.root}),
            (r"/frontend\.html", FrontendHandler),
            ("/", web.RedirectHandler, {"url": "/frontend.html",
                                        "permanent": True}),
            ("", web.RedirectHandler, {"url": "/frontend.html",
                                       "permanent": True})
        ], template_path=root.common.web.root)
        app.listen(port)

        def open_browser():
            from veles.portable import show_file

            show_file("http://localhost:%d" % port)

        IOLoop.instance().add_callback(open_browser)
        try:
            IOLoop.instance().start()
        except KeyboardInterrupt:
            sys.stderr.write("KeyboardInterrupt\n")
        finally:
            connection.put(cmdline[0])

    def _parse_optimization(self, args):
        if args.optimize is None:
            return
        optparsed = args.optimize.split(':')
        if len(optparsed) > 2:
            raise ValueError("Invalid --optimize value: %s" % args.optimize)
        try:
            self.optimization = int(optparsed[0]), \
                int(optparsed[1]) if len(optparsed) == 2 else None
        except ValueError:
            raise from_none(ValueError(
                "\"%s\" is not a valid --optimize value" % args.optimize))

    def _parse_ensemble_train(self, args):
        if args.ensemble_train is None:
            return

        optparsed = args.ensemble_train.split(":")
        if len(optparsed) != 2:
            raise ValueError(
                "--ensemble-train must be specified as"
                "<number of instances>:<training set ratio>")
        try:
            self.ensemble_train = int(optparsed[0]), float(optparsed[1])
        except ValueError:
            raise from_none(
                "Failed to parse ensemble parameters from (%s, %s)" %
                optparsed)

    def _parse_ensemble_test(self, args):
        if args.ensemble_test is None:
            return
        if self.ensemble_train is not None:
            raise ValueError(
                "--ensemble-train and --ensemble-test may not be used "
                "together")
        self.ensemble_test = args.ensemble_test

    def _daemonize(self):
        daemon_context = daemon.DaemonContext()
        daemon_context.working_directory = os.getcwd()
        daemon_context.files_preserve = [
            int(fd) for fd in os.listdir("/proc/self/fd")
            if int(fd) > 2]
        daemon_context.open()  # <- the magic happens here

    @staticmethod
    def _get_interactive_locals():
        """
        If we are running under IPython, extracts the local variables;
        otherwise, returns an empty dict.
        """
        try:
            __IPYTHON__  # pylint: disable=E0602
            from IPython.core.interactiveshell import InteractiveShell
            return {k: v for k, v in
                    InteractiveShell.instance().user_ns.items()
                    if k[0] != '_' and k not in (
                        "Out", "In", "exit", "quit", "get_ipython")}
        except NameError:
            return {}

    def _load_model(self, fname_workflow):
        self.info("Loading workflow \"%s\"...", fname_workflow)
        self.load_called = False
        self.main_called = False
        package_name, module_name = get_file_package_and_module(
            fname_workflow)
        try:
            return import_file_as_package(fname_workflow)
        except Exception as e:
            self.debug("Failed to import \"%s\" through the parent package "
                       "\"%s\": %s", fname_workflow, package_name, e)
            package_import_error = e
        # We failed to load the package => try module approach
        try:
            return import_file_as_module(fname_workflow)
        except FileNotFoundError:
            self.exception("Workflow does not exist: \"%s\"", fname_workflow)
            sys.exit(errno.ENOENT)
        except IsADirectoryError:
            self.exception("Workflow \"%s\" is a directory", fname_workflow)
            sys.exit(errno.EISDIR)
        except PermissionError:
            self.exception("Cannot read workflow \"%s\"", fname_workflow)
            sys.exit(errno.EACCES)
        except Exception as e:
            self.error("Failed to load the workflow \"%s\".\n"
                       "Package import error: %s\nModule import error: %s",
                       fname_workflow, package_import_error, e)
            sys.exit(Main.EXIT_FAILURE)

    def _apply_config(self, fname_config):
        if not self.config_file:
            self.warning("Configuration path is empty")
            return

        def fail():
            self.exception("Failed to apply the configuration \"%s\"",
                           fname_config)
            sys.exit(Main.EXIT_FAILURE)

        self.info("Applying the configuration from %s...", fname_config)
        try:
            runpy.run_path(fname_config)
        except FileNotFoundError:
            self.exception("Configuration does not exist: \"%s\"",
                           fname_config)
            sys.exit(errno.ENOENT)
        except IsADirectoryError:
            self.exception("Configuration \"%s\" is a directory", fname_config)
            sys.exit(errno.EISDIR)
        except PermissionError:
            self.exception("Cannot read configuration \"%s\"", fname_config)
            sys.exit(errno.EACCES)
        except TypeError as e:
            self.debug("Filed to import \"%s\": %s -> assumed pickle",
                       fname_config, e)
            from veles.pickle2 import pickle
            try:
                with open(fname_config, "rb") as fin:
                    cfg = pickle.load(fin)
            except:
                fail()
            for subcfg in cfg:
                root[subcfg].update(cfg[subcfg])
        except SyntaxError as e:
            self.debug("Filed to import \"%s\": %s -> assumed json",
                       fname_config, e)
            import json
            try:
                with open(fname_config, "r") as fin:
                    cfg = json.load(fin)
            except:
                fail()
            for subcfg in cfg:
                root[subcfg].update(cfg[subcfg])
        except:
            fail()

    def _override_config(self, config_list):
        cmds = "\n".join(config_list)
        self.debug("Overriding the configuration with %s", cmds)
        try:
            exec(cmds)
        except:
            self.exception("Invalid configuration overloads")
            sys.exit(Main.EXIT_FAILURE)

    def _seed_random(self, rndvals):
        self.debug("Seeding with %s", rndvals)
        rndvals_split = rndvals.split(',')
        seeds = []
        for rndval, index in zip(rndvals_split, range(len(rndvals_split))):
            try:
                binvle = binascii.unhexlify(rndval)
                seed = numpy.frombuffer(binvle, dtype=numpy.uint8)
                prng.get(index + 1).seed(seed, dtype=numpy.uint8)
                seeds.append(seed)
                continue
            except (binascii.Error, TypeError):
                pass
            vals = rndval.split(':')
            fname = vals[0]
            if fname == "":
                if index > 1:
                    fname = rndvals_split[0].split(':')[0] + str(index)
                else:
                    self.critical("Random generator file name is empty")
                    sys.exit(errno.ENOENT)
            if fname == "-":
                seeds.append(None)
                try:
                    prng.get(index + 1).seed(None)
                except:
                    self.exception("Failed to seed the random generator %d "
                                   "with the last used seed.", index + 1)
                    sys.exit(Main.EXIT_FAILURE)
                continue
            if not os.path.isabs(fname):
                new_fname = os.path.abspath(fname)
                if os.path.exists(new_fname):
                    fname = new_fname
                else:
                    fname = os.path.join(root.common.dirs.veles, fname)
                    if not os.path.exists(fname):
                        self.critical("Neither %s nor %s exist. Cannot seed "
                                      "the random generator.", new_fname,
                                      fname)
                        sys.exit(errno.ENOENT)
            count = int(vals[1]) if len(vals) > 1 else 16
            dtype = numpy.dtype(vals[2]) if len(vals) > 2 else numpy.int32

            self.debug("Seeding with %d samples of type %s from %s to %d",
                       count, dtype, fname, index + 1)
            try:
                seed = (numpy.fromfile(fname, dtype=dtype, count=count))
                prng.get(index + 1).seed(seed, dtype=dtype)
                seeds.append(seed)
            except:
                self.exception("Failed to seed the random generator with %s",
                               fname)
                sys.exit(Main.EXIT_FAILURE)
        self.seeds = seeds

    def _load_workflow(self, fname_snapshot):
        stype = splittype(fname_snapshot)[0]
        if stype == "odbc":
            import pyodbc

            addr = fname_snapshot[7:]
            parsed = addr.split('&')
            try:
                odbc, table, id_, log_id = parsed[:4]
            except TypeError:
                self.warning("Invalid ODBC source format. Here is the "
                             "template: odbc://<odbc data source spec>&"
                             "<table>&<id>&<log id>[&<optional name>]\n"
                             "<table> and <log id> may be empty (\"veles\" and"
                             " <id> value will be used).")
                return None
            if not table:
                table = "veles"
            if not log_id:
                log_id = id_
            if len(parsed) > 4:
                if len(parsed) > 5:
                    self.warning("Invalid ODBC source format")
                    return None
                name = parsed[-1]
            else:
                name = None
            try:
                return SnapshotterToDB.import_(odbc, table, id_, log_id, name)
            except pyodbc.Error as e:
                self.warning(
                    "Failed to load the snapshot from ODBC source: %s", e)
                return None
        elif stype in ("http", "https"):
            try:
                self.info("Downloading %s...", fname_snapshot)
                fname_snapshot = self.snapshot_file_name = wget.download(
                    fname_snapshot, root.common.dirs.snapshots)
                print()
                sys.stdout.flush()
            except:
                self.exception("Failed to fetch the snapshot at \"%s\"",
                               fname_snapshot)
                return None
        try:
            return SnapshotterToFile.import_(fname_snapshot)
        except FileNotFoundError:
            if fname_snapshot.strip() != "":
                self.warning("Workflow snapshot %s does not exist",
                             fname_snapshot)
            return None

    def _load(self, Workflow, **kwargs):
        self.debug("load() was called from run(), workflow class is %s",
                   str(Workflow))
        self.load_called = True
        try:
            self.launcher = Launcher(self.interactive)
            self.launcher.workflow_file = self.workflow_file
            self.launcher.config_file = self.config_file
            self.launcher.seeds = self.seeds
        except:
            self.exception("Failed to create the launcher")
            sys.exit(Main.EXIT_FAILURE)
        try:
            self.workflow = self._load_workflow(self.snapshot_file_name)
            self.snapshot = self.workflow is not None
            if not self.snapshot:
                wfkw = self._get_interactive_locals()
                wfkw.update(kwargs)
                self.workflow = Workflow(self.launcher, **wfkw)
                self.info("Created %s", self.workflow)
            else:
                self.info("Loaded the workflow snapshot from %s: %s",
                          self.snapshot_file_name, self.workflow)
                if self._visualization_mode:
                    self.workflow.plotters_are_enabled = True
                self.workflow.workflow = self.launcher
        except:
            self.exception("Failed to create the workflow")
            self.launcher.stop()
            sys.exit(Main.EXIT_FAILURE)
        if self._workflow_graph:
            self.workflow.generate_graph(filename=self._workflow_graph,
                                         with_data_links=True,
                                         background='white')
        return self.workflow, self.snapshot

    def _main(self, **kwargs):
        if self._dry_run < 2:
            self.launcher.stop()
            return
        self.debug("main() was called from run()")
        if not self.load_called:
            self.critical("Call load() first in run()")
            sys.exit(Main.EXIT_FAILURE)
        self.main_called = True
        kwargs["snapshot"] = self.snapshot
        kwargs["no_device"] = not self.regular

        try:
            self.launcher.initialize(**kwargs)
        except:
            self.exception("Failed to initialize the launcher.")
            self.launcher.stop()
            sys.exit(Main.EXIT_FAILURE)

        self.debug("Initialization is complete")
        try:
            if self._dump_attrs != "no":
                self._dump_unit_attributes(self._dump_attrs == "all")
            gc.collect()
            if self._dry_run > 2:
                self.debug("Running the launcher")
                self.launcher.run()
            elif self._visualization_mode:
                self.debug("Visualizing the workflow...")
                self._visualize_workflow()
        except:
            self.exception("Failed to run the workflow")
            self.launcher.stop()
            sys.exit(Main.EXIT_FAILURE)
        finally:
            self.launcher.device_thread_pool_detach()

    def _dump_unit_attributes(self, arrays=True):
        import veles.external.prettytable as prettytable
        from veles.workflow import Workflow

        self.debug("Dumping unit attributes of %s...", str(self.workflow))
        table = prettytable.PrettyTable("#", "unit", "attr", "value")
        table.align["#"] = "r"
        table.align["unit"] = "l"
        table.align["attr"] = "l"
        table.align["value"] = "l"
        table.max_width["value"] = 100
        for i, u in enumerate(self.workflow.units_in_dependency_order):
            for k, v in sorted(u.__dict__.items()):
                if k not in Workflow.HIDDEN_UNIT_ATTRS:
                    if (not arrays and hasattr(v, "__len__") and len(v) > 32
                            and not isinstance(v, str)
                            and not isinstance(v, bytes)):
                        strv = "object of class %s of length %d" % (
                            repr(v.__class__.__name__), len(v))
                    else:
                        strv = repr(v)
                    table.add_row(i, u.__class__.__name__, k, strv)
        print(table)

    def _visualize_workflow(self):
        _, file_name = self.workflow.generate_graph(with_data_links=True,
                                                    background='white')
        from veles.portable import show_file

        show_file(file_name)

        import signal

        self.launcher.graphics_client.send_signal(signal.SIGUSR2)
        from twisted.internet import reactor

        reactor.callWhenRunning(self._run_workflow_plotters)
        reactor.callWhenRunning(print_, "Press Ctrl-C when you are done...")
        reactor.run()

    def _run_workflow_plotters(self):
        from veles.plotter import Plotter

        for unit in self.workflow:
            if isinstance(unit, Plotter):
                unit.run()
        # Second loop is needed to finish with PDF
        for unit in self.workflow:
            if isinstance(unit, Plotter):
                unit.last_run_time = 0
                unit.run()
                break

    def _run_core(self, wm):
        if self._dry_run <= 0:
            return
        if not self.optimization:
            from veles.genetics import fix_config
            fix_config(root)
        if self.regular:
            self.run_module(wm)
        elif self.optimization is not None:
            import veles.genetics.optimization_workflow as workflow
            self.run_module(workflow, model=wm, **self.optimization.__dict__)
        elif self.ensemble_train is not None:
            import veles.ensemble.model_workflow as workflow
            self.run_module(workflow, model=wm, **self.ensemble_train.__dict__)
        elif self.ensemble_test is not None:
            import veles.ensemble.test_workflow as workflow
            self.run_module(workflow, model=wm, **self.ensemble_test.__dict__)
        else:
            raise NotImplementedError("Unsupported execution mode")

    def _apply_args(self, args):
        if args.background:
            self._daemonize()
        if not args.workflow:
            raise ValueError("Workflow path may not be empty")
        config_file = args.config
        if config_file == "-":
            config_file = "%s_config%s" % os.path.splitext(args.workflow)
        self.workflow_file = os.path.abspath(args.workflow)
        self.config_file = os.path.abspath(config_file) if config_file else ""
        self._visualization_mode = args.visualize
        self._workflow_graph = args.workflow_graph
        self._dry_run = Main.DRY_RUN_CHOICES.index(args.dry_run)
        self._dump_attrs = args.dump_unit_attributes
        self.snapshot_file_name = args.snapshot
        self._parse_optimization(args)
        self._parse_ensemble_train(args)
        self._parse_ensemble_test(args)

    def _print_logo(self, args):
        if not args.no_logo:
            try:
                print(Main.LOGO)
            except:
                print(Main.LOGO.replace("©", "(c)"))
            sys.stdout.flush()

    def _print_version(self):
        print(veles.__version__, formatdate(veles.__date__, True),
              veles.__git__)

    def _print_config(self, cfg):
        io = StringIO()
        cfg.print_(file=io)
        self.debug("\n%s", io.getvalue().strip())

    def setup_logging(self, verbosity):
        try:
            super(Main, self).setup_logging(Main.LOG_LEVEL_MAP[verbosity])
        except Logger.LoggerHasBeenAlreadySetUp as e:
            if not self.interactive:
                raise from_none(e)

    def _register_print_max_rss(self):
        if not Main.registered_print_max_rss:
            atexit.register(self.print_max_rss)
            Main.registered_print_max_rss = True

    @staticmethod
    def format_decimal(val):
        if val < 1000:
            return str(val)
        d, m = divmod(val, 1000)
        return Main.format_decimal(d) + " %03d" % m

    def print_max_rss(self):
        res = resource.getrusage(resource.RUSAGE_SELF)
        if Watcher.max_mem_in_use > 0:
            self.info("Peak device memory used: %s Kb",
                      self.format_decimal(Watcher.max_mem_in_use // 1000))
        self.info("Peak resident memory used: %s Kb",
                  self.format_decimal(res.ru_maxrss))

    def run_module(self, module, **kwargs):
        self.debug("Calling %s.run()...", module.__name__)
        module.run(self._load, self._main, **kwargs)
        if not self.main_called and self._dry_run > 2:
            self.warning("main() was not called by run() in %s",
                         module.__file__)

    """
    Basically, this is what each workflow module's run() should do.
    """

    def run_workflow(self, Workflow, kwargs_load=None, kwargs_main=None):
        # we should not set "{}" as default values because of the way
        # default values work: the dicts will be reused, not recreated
        if kwargs_load is None:
            kwargs_load = {}
        if kwargs_main is None:
            kwargs_main = {}
        self._load(Workflow, **kwargs_load)
        self._main(**kwargs_main)

    def run(self):
        """Entry point for the VELES execution engine.
        """
        veles.validate_environment()

        ret = self._process_special_args()
        if ret is not None:
            return ret
        parser = Main.init_parser()
        args = parser.parse_args(self.argv)
        self._apply_args(args)

        self.setup_logging(args.verbosity)
        self._print_logo(args)
        for name in filter(str.strip, args.debug.split(',')):
            logging.getLogger(name).setLevel(logging.DEBUG)
        self._seed_random(args.random_seed)
        if args.debug_pickle:
            setup_pickle_debug()
        ThreadPool.reset()
        self._register_print_max_rss()

        if self.logger.isEnabledFor(logging.DEBUG):
            self._print_config(root)
        wm = self._load_model(self.workflow_file)
        self._apply_config(self.config_file)
        self._override_config(args.config_list)
        if self.logger.isEnabledFor(logging.DEBUG):
            self._print_config(root)

        self._run_core(wm)

        if not self.interactive:
            self.info("End of job")
        else:
            self.info("\033[1;35mReturned the control\033[0m")
        return Main.EXIT_SUCCESS