Ejemplo n.º 1
0
 def request(self, command, message=b''):
     self.event("ZeroMQ", "begin", dir="send", command=command, height=0.5)
     if self.shmem is not None and command == 'update':
         self.shmem.seek(0)
     try:
         pickles_size, delta = timeit(
             self.send,
             self.id,
             command.encode('charmap'),
             message,
             io=self.shmem,
             pickles_compression=self.pickles_compression)
         if command not in self._request_timings:
             self._request_timings[command] = (0.0, 0)
         self._request_timings[command] = (
             self._request_timings[command][0] + delta,
             self._request_timings[command][1] + 1)
     except ZmqConnection.IOOverflow:
         self.shmem = None
         return
     if self.is_ipc and command == 'update' and self.shmem is None:
         self.shmem = SharedIO(
             "veles-update-" + self.id.decode('charmap'),
             int(pickles_size * (1.0 + ZmqDealer.RESERVE_SHMEM_SIZE)))
     self.event("ZeroMQ", "end", dir="send", command=command, height=0.5)
Ejemplo n.º 2
0
 def wrapped_measure_time(*args, **kwargs):
     res, delta = timeit(fn, *args, **kwargs)
     if self.id in storage:
         storage[self.id] += delta
     if self.timings:
         self.debug("%s took %.6f sec", fn.__name__, delta)
     return res
Ejemplo n.º 3
0
 def wrapped(self, *args, **kwargs):
     mt = self._method_time_.get(fn.__name__)
     if mt is None:
         mt = 0
     res, dt = timeit(fn, self, *args, **kwargs)
     mt += dt
     self._method_time_[fn.__name__] = mt
     return res
Ejemplo n.º 4
0
 def wrapped(self, *args, **kwargs):
     mt = self._method_time_.get(fn.__name__)
     if mt is None:
         mt = 0
     res, dt = timeit(fn, self, *args, **kwargs)
     mt += dt
     self._method_time_[fn.__name__] = mt
     return res
Ejemplo n.º 5
0
    def __init__(self, thread_pool=None):
        if self.initialized:
            return
        self.initialized = True
        assert thread_pool is not None, (
            "GraphicsServer was not previously initialized")
        super(GraphicsServer, self).__init__()
        parser = GraphicsServer.init_parser()
        args, _ = parser.parse_known_args(self.argv)
        self._debug_pickle = args.graphics_pickle_debug
        zmq_endpoints = [
            ZmqEndpoint("bind", "inproc://veles-plots"),
            ZmqEndpoint("bind", "rndipc://veles-ipc-plots-:")
        ]
        ifaces = []
        for iface, _ in interfaces():
            if iface in root.common.graphics.blacklisted_ifaces:
                continue
            ifaces.append(iface)
            zmq_endpoints.append(
                ZmqEndpoint(
                    "bind", "rndepgm://%s;%s:1024:65535:1" %
                    (iface, root.common.graphics.multicast_address)))
        self.debug("Trying to bind to %s...", zmq_endpoints)

        try:
            self.zmq_connection, btime = timeit(ZmqPublisher, zmq_endpoints)
        except zmq.error.ZMQError:
            self.exception("Failed to bind to %s", zmq_endpoints)
            raise from_none(GraphicsServer.InitializationError())

        # Important! Save the bound method to variable to avoid dead weak refs
        # See http://stackoverflow.com/questions/19443440/weak-reference-to-python-class-method  # nopep8
        self._shutdown_ = self.shutdown
        thread_pool.register_on_shutdown(self._shutdown_)

        # tmpfn, *ports = self.zmq_connection.rnd_vals
        tmpfn = self.zmq_connection.rnd_vals[0]
        ports = self.zmq_connection.rnd_vals[1:]

        self.endpoints = {
            "inproc": "inproc://veles-plots",
            "ipc": "ipc://" + tmpfn,
            "epgm": []
        }
        for port, iface in zip(ports, ifaces):
            self.endpoints["epgm"].append(
                "epgm://%s;%s:%d" %
                (iface, root.common.graphics.multicast_address, port))
        self.info(
            "Publishing to %s",
            "; ".join([self.endpoints["inproc"], self.endpoints["ipc"]] +
                      self.endpoints["epgm"]))
        if btime > 1:
            self.warning(
                "EPGM bind took %d seconds - consider adding offending "
                "interfaces to root.common.graphics.blacklisted_ifaces or "
                "completely disabling graphics (-p '').", int(btime))
Ejemplo n.º 6
0
    def numpy_run(self):
        def execute(repeats):
            for _ in range(repeats):
                numpy.dot(self._input_A_.mem, self._input_A_.mem,
                          self._input_B_.mem)

        if self.dry_run_first:
            execute(1)

        return timeit(execute, self.repeats)[1]
Ejemplo n.º 7
0
 def run(self):
     super(NNSnapshotterBase, self).run()
     logged = set()
     for u in self.workflow.start_point.dependent_units():
         for attr in ("input", "weights", "bias", "output", "err_output",
                      "err_input"):
             self._log_attr(u, attr, logged)
     del logged
     _, dt = timeit(gc.collect)
     if dt > 1.0:
         self.warning("gc.collect() took %.1f sec", dt)
Ejemplo n.º 8
0
 def run(self):
     if not super(NNSnapshotterBase, self).run():
         return
     logged = set()
     for u in self.workflow.start_point.dependent_units():
         for attr in ("input", "weights", "bias", "output", "err_output", "err_input"):
             self._log_attr(u, attr, logged)
     del logged
     _, dt = timeit(gc.collect)
     if dt > 1.0:
         self.warning("gc.collect() took %.1f sec", dt)
Ejemplo n.º 9
0
    def __init__(self, thread_pool=None):
        if self.initialized:
            return
        self.initialized = True
        assert thread_pool is not None, (
            "GraphicsServer was not previously initialized")
        super(GraphicsServer, self).__init__()
        parser = GraphicsServer.init_parser()
        args, _ = parser.parse_known_args(self.argv)
        self._debug_pickle = args.graphics_pickle_debug
        zmq_endpoints = [ZmqEndpoint("bind", "inproc://veles-plots"),
                         ZmqEndpoint("bind", "rndipc://veles-ipc-plots-:")]
        ifaces = []
        for iface, _ in interfaces():
            if iface in root.common.graphics.blacklisted_ifaces:
                continue
            ifaces.append(iface)
            zmq_endpoints.append(ZmqEndpoint(
                "bind", "rndepgm://%s;%s:1024:65535:1" %
                        (iface, root.common.graphics.multicast_address)))
        self.debug("Trying to bind to %s...", zmq_endpoints)

        try:
            self.zmq_connection, btime = timeit(ZmqPublisher, zmq_endpoints)
        except zmq.error.ZMQError:
            self.exception("Failed to bind to %s", zmq_endpoints)
            raise from_none(GraphicsServer.InitializationError())

        # Important! Save the bound method to variable to avoid dead weak refs
        # See http://stackoverflow.com/questions/19443440/weak-reference-to-python-class-method  # nopep8
        self._shutdown_ = self.shutdown
        thread_pool.register_on_shutdown(self._shutdown_)

        # tmpfn, *ports = self.zmq_connection.rnd_vals
        tmpfn = self.zmq_connection.rnd_vals[0]
        ports = self.zmq_connection.rnd_vals[1:]

        self.endpoints = {"inproc": "inproc://veles-plots",
                          "ipc": "ipc://" + tmpfn,
                          "epgm": []}
        for port, iface in zip(ports, ifaces):
            self.endpoints["epgm"].append(
                "epgm://%s;%s:%d" %
                (iface, root.common.graphics.multicast_address, port))
        self.info("Publishing to %s", "; ".join([self.endpoints["inproc"],
                                                 self.endpoints["ipc"]] +
                                                self.endpoints["epgm"]))
        if btime > 1:
            self.warning(
                "EPGM bind took %d seconds - consider adding offending "
                "interfaces to root.common.graphics.blacklisted_ifaces or "
                "completely disabling graphics (-p '').",
                int(btime))
Ejemplo n.º 10
0
    def ocl_run(self):
        global_size = (roundup(self.size, self.block_size),) * 2
        local_size = (self.block_size,) * 2
        self.device.queue_.flush()
        self.device.queue_.finish()

        def execute(repeats):
            for _ in range(repeats):
                self.execute_kernel(global_size, local_size)
            self.device.queue_.flush()
            self.device.queue_.finish()

        if self.dry_run_first:
            execute(1)

        return timeit(execute, self.repeats)[1]
Ejemplo n.º 11
0
    def cuda_run(self):
        import cuda4py.blas as cublas
        self.device.sync()

        def execute(repeats):
            for _ in range(repeats):
                self.gemm_(
                    self.device.blas, cublas.CUBLAS_OP_T, cublas.CUBLAS_OP_N,
                    self.size, self.size, self.size,
                    self.np_one, self._input_A_.devmem, self._input_A_.devmem,
                    self.np_zero, self._input_B_.devmem)
            self.device.sync()

        if self.dry_run_first:
            execute(1)

        return timeit(execute, self.repeats)[1]
Ejemplo n.º 12
0
 def wrapped(self, *args, **kwargs):
     res, delta = timeit(fn, self, *args, **kwargs)
     if self.is_slave:
         self._run_time_ += delta
     return res
Ejemplo n.º 13
0
 def wrapped(self, *args, **kwargs):
     res, delta = timeit(fn, self, *args, **kwargs)
     if self.is_slave:
         self._run_time_ += delta
     return res