示例#1
0
    def __init__(self,
                 command,
                 stdIOType,
                 command_args=None,
                 cwd=None,
                 **kwargs):
        self.log = logging.getLogger(__name__)
        self.log.debug('foo')

        Callback.__init__(self)

        assert isinstance(stdIOType, list)
        self.timeout = kwargs['timeout']
        self.commandSleepTime = kwargs['commandSleepTime']
        self.stdIOType = stdIOType
        self.stdioData = deque()

        args = [command]
        if command_args:
            if not isinstance(command_args, list):
                command_args = [command_args]
            args.extend(command_args)

        # create configured process pipes
        prockwargs = dict()
        prockwargs['shell'] = False
        prockwargs['stdin'] = PIPE
        prockwargs['stdout'] = PIPE if STDIOTYPE.STDOUT in stdIOType else None
        prockwargs['stderr'] = PIPE if STDIOTYPE.STDERR in stdIOType else None
        prockwargs['cwd'] = cwd

        # create process
        self.proc = Popen(args, **prockwargs)

        # create process handler

        self.stdoutHandle = None
        if STDIOTYPE.STDOUT in stdIOType:
            self.stdoutHandle = _StdOutHandler(stdio=self.proc.stdout,
                                               lineWrapper=OutLineWrapper,
                                               **kwargs)
            self.stdoutHandle.register(self._newDataCallbackFnc)

        self.stderrHandle = None
        if STDIOTYPE.STDERR in stdIOType:
            self.stderrHandle = _StdOutHandler(stdio=self.proc.stderr,
                                               lineWrapper=ErrorLineWrapper,
                                               **kwargs)
            self.stderrHandle.register(self._newDataCallbackFnc)

        self.stdioHandle = [self.stdoutHandle, self.stderrHandle]
示例#2
0
    def __init__(self, **kwargs):
        self.log = logging.getLogger(__name__)
        self.log.debug('init with <%s>', kwargs)

        self.log.debug('call threading.Thread.__init__(self, target=self.run)')
        threading.Thread.__init__(self, target=self.run)
        self.log.debug('call Callback.__init__(self)')
        Callback.__init__(self)

        self.syncThreads = kwargs.get('syncThreads', False)
        self.commandSleepTime = kwargs['commandSleepTime']
        self.timeout = kwargs['timeout']
        self.stdio = kwargs['stdio']
        self.lastDataTime = None
    def __init__(self, **kwargs):
        self.log = logging.getLogger(__name__)
        self.log.debug("init with <%s>", kwargs)

        self.log.debug("call threading.Thread.__init__(self, target=self.run)")
        threading.Thread.__init__(self, target=self.run)
        self.log.debug("call Callback.__init__(self)")
        Callback.__init__(self)

        self.syncThreads = kwargs.get("syncThreads", False)
        self.commandSleepTime = kwargs["commandSleepTime"]
        self.timeout = kwargs["timeout"]
        self.stdio = kwargs["stdio"]
        self.lastDataTime = None
示例#4
0
    def __init__(self, lineWrapper=None, **kwargs):
        self.log = logging.getLogger(__name__)
        self.log.debug('start logging')
        Callback.__init__(self)
        self.lineWrapper = lineWrapper
        self.stdio = kwargs['stdio']
        self.dataCallbackFnc = kwargs.get('dataCallbackFnc', None)
        #self.stdioData = deque()

        # setting stdio-file in non blocking mode
        fcntl.fcntl(self.stdio.fileno(), fcntl.F_SETFL, os.O_NONBLOCK)

        # create data thread and register data callback
        self.stdioThread = _DataThread(**kwargs)
        #self.stdioThread.register(self.stdioData.extend, self)
        self.stdioThread.register(self.storeData, self)
    def __init__(self, lineWrapper=None, **kwargs):
        self.log = logging.getLogger(__name__)
        self.log.debug("start logging")
        Callback.__init__(self)
        self.lineWrapper = lineWrapper
        self.stdio = kwargs["stdio"]
        self.dataCallbackFnc = kwargs.get("dataCallbackFnc", None)
        # self.stdioData = deque()

        # setting stdio-file in non blocking mode
        fcntl.fcntl(self.stdio.fileno(), fcntl.F_SETFL, os.O_NONBLOCK)

        # create data thread and register data callback
        self.stdioThread = _DataThread(**kwargs)
        # self.stdioThread.register(self.stdioData.extend, self)
        self.stdioThread.register(self.storeData, self)
    def __init__(self, command, stdIOType, command_args=None, cwd=None, **kwargs):
        self.log = logging.getLogger(__name__)
        self.log.debug("foo")

        Callback.__init__(self)

        assert isinstance(stdIOType, list)
        self.timeout = kwargs["timeout"]
        self.commandSleepTime = kwargs["commandSleepTime"]
        self.stdIOType = stdIOType
        self.stdioData = deque()

        args = [command]
        if command_args:
            if not isinstance(command_args, list):
                command_args = [command_args]
            args.extend(command_args)

        # create configured process pipes
        prockwargs = dict()
        prockwargs["shell"] = False
        prockwargs["stdin"] = PIPE
        prockwargs["stdout"] = PIPE if STDIOTYPE.STDOUT in stdIOType else None
        prockwargs["stderr"] = PIPE if STDIOTYPE.STDERR in stdIOType else None
        prockwargs["cwd"] = cwd

        # create process
        self.proc = Popen(args, **prockwargs)

        # create process handler

        self.stdoutHandle = None
        if STDIOTYPE.STDOUT in stdIOType:
            self.stdoutHandle = _StdOutHandler(stdio=self.proc.stdout, lineWrapper=OutLineWrapper, **kwargs)
            self.stdoutHandle.register(self._newDataCallbackFnc)

        self.stderrHandle = None
        if STDIOTYPE.STDERR in stdIOType:
            self.stderrHandle = _StdOutHandler(stdio=self.proc.stderr, lineWrapper=ErrorLineWrapper, **kwargs)
            self.stderrHandle.register(self._newDataCallbackFnc)

        self.stdioHandle = [self.stdoutHandle, self.stderrHandle]
示例#7
0
    def __init__(self,
                 server=None,
                 user_id=0,
                 initial_version=None,
                 initial={}):
        self.state = initial
        self.initial_state = initial
        self.history = History()

        self.user_id = user_id
        self.version = VectorClock(user_id, initial_version)

        self.server = server
        if server is not None:
            self.server.on_recieve += self.on_recieve

        self.on_update = Callback()
示例#8
0
    def __init__(self, host, port):
        self.host = host
        self.port = port

        self.listeners_running = False
        self.senders_running = False

        self.in_queue = Queue()
        self.out_queue = Queue()

        self.on_recieve = Callback()

        # initial socket setup
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self.sock.bind((self.host, self.port))
        self.sock.settimeout(60)
        # TODO: increase number of listeners?
        self.sock.listen(5)

        self.clients: List[socket.socket] = [self.sock]
示例#9
0
def computeNewModels(filePath,
                     name,
                     step=10000,
                     maxLines=100001,
                     size=100,
                     min_count=5,
                     iter=5,
                     sg=0,
                     negative=5):

    treated_lines = [0]
    time_array = []
    loss = 0

    for i in range(maxLines):
        if i % step == 0 and i != 0:
            depart_time = time.time()
            print("Ligne " + str(i) + " sur " + str(maxLines))
            partialCorpus = MyPartialCorpus(filePath, i)

            loss_callback = Callback()

            model = gensim.models.Word2Vec(sentences=partialCorpus,
                                           workers=multiprocessing.cpu_count(),
                                           size=size,
                                           min_count=min_count,
                                           iter=iter,
                                           sg=sg,
                                           negative=negative,
                                           compute_loss=True,
                                           callbacks=[loss_callback])
            loss = loss_callback.loss_previous_step / (maxLines - 1)
            print("Temps pour entrainer le model: " +
                  str(time.time() - depart_time))
            time_array.append(time.time() - depart_time)

    writeNewModel(treated_lines, time_array, name)
    return loss
示例#10
0
 def parse_callbacks(self):
     """ Field nodes can have callback subnodes, parse them here """
     from Callback import Callback
     for cb in self.find_children(self.ns(self.NS_CORE, 'callback')):
         self.callbacks.append(Callback(self._namespace, cb))