def __init__(self,
command,
stdIOType,
command_args=None,
cwd=None,
**kwargs):
self.log = logging.getLogger(__name__)
self.log.debug('foo')
Callback.__init__(self)
assert isinstance(stdIOType, list)
self.timeout = kwargs['timeout']
self.commandSleepTime = kwargs['commandSleepTime']
self.stdIOType = stdIOType
self.stdioData = deque()
args = [command]
if command_args:
if not isinstance(command_args, list):
command_args = [command_args]
args.extend(command_args)
# create configured process pipes
prockwargs = dict()
prockwargs['shell'] = False
prockwargs['stdin'] = PIPE
prockwargs['stdout'] = PIPE if STDIOTYPE.STDOUT in stdIOType else None
prockwargs['stderr'] = PIPE if STDIOTYPE.STDERR in stdIOType else None
prockwargs['cwd'] = cwd
# create process
self.proc = Popen(args, **prockwargs)
# create process handler
self.stdoutHandle = None
if STDIOTYPE.STDOUT in stdIOType:
self.stdoutHandle = _StdOutHandler(stdio=self.proc.stdout,
lineWrapper=OutLineWrapper,
**kwargs)
self.stdoutHandle.register(self._newDataCallbackFnc)
self.stderrHandle = None
if STDIOTYPE.STDERR in stdIOType:
self.stderrHandle = _StdOutHandler(stdio=self.proc.stderr,
lineWrapper=ErrorLineWrapper,
**kwargs)
self.stderrHandle.register(self._newDataCallbackFnc)
self.stdioHandle = [self.stdoutHandle, self.stderrHandle]
def __init__(self, **kwargs):
self.log = logging.getLogger(__name__)
self.log.debug('init with <%s>', kwargs)
self.log.debug('call threading.Thread.__init__(self, target=self.run)')
threading.Thread.__init__(self, target=self.run)
self.log.debug('call Callback.__init__(self)')
Callback.__init__(self)
self.syncThreads = kwargs.get('syncThreads', False)
self.commandSleepTime = kwargs['commandSleepTime']
self.timeout = kwargs['timeout']
self.stdio = kwargs['stdio']
self.lastDataTime = None
def __init__(self, **kwargs):
self.log = logging.getLogger(__name__)
self.log.debug("init with <%s>", kwargs)
self.log.debug("call threading.Thread.__init__(self, target=self.run)")
threading.Thread.__init__(self, target=self.run)
self.log.debug("call Callback.__init__(self)")
Callback.__init__(self)
self.syncThreads = kwargs.get("syncThreads", False)
self.commandSleepTime = kwargs["commandSleepTime"]
self.timeout = kwargs["timeout"]
self.stdio = kwargs["stdio"]
self.lastDataTime = None
def __init__(self, lineWrapper=None, **kwargs):
self.log = logging.getLogger(__name__)
self.log.debug('start logging')
Callback.__init__(self)
self.lineWrapper = lineWrapper
self.stdio = kwargs['stdio']
self.dataCallbackFnc = kwargs.get('dataCallbackFnc', None)
#self.stdioData = deque()
# setting stdio-file in non blocking mode
fcntl.fcntl(self.stdio.fileno(), fcntl.F_SETFL, os.O_NONBLOCK)
# create data thread and register data callback
self.stdioThread = _DataThread(**kwargs)
#self.stdioThread.register(self.stdioData.extend, self)
self.stdioThread.register(self.storeData, self)
def __init__(self, lineWrapper=None, **kwargs):
self.log = logging.getLogger(__name__)
self.log.debug("start logging")
Callback.__init__(self)
self.lineWrapper = lineWrapper
self.stdio = kwargs["stdio"]
self.dataCallbackFnc = kwargs.get("dataCallbackFnc", None)
# self.stdioData = deque()
# setting stdio-file in non blocking mode
fcntl.fcntl(self.stdio.fileno(), fcntl.F_SETFL, os.O_NONBLOCK)
# create data thread and register data callback
self.stdioThread = _DataThread(**kwargs)
# self.stdioThread.register(self.stdioData.extend, self)
self.stdioThread.register(self.storeData, self)
def __init__(self, command, stdIOType, command_args=None, cwd=None, **kwargs):
self.log = logging.getLogger(__name__)
self.log.debug("foo")
Callback.__init__(self)
assert isinstance(stdIOType, list)
self.timeout = kwargs["timeout"]
self.commandSleepTime = kwargs["commandSleepTime"]
self.stdIOType = stdIOType
self.stdioData = deque()
args = [command]
if command_args:
if not isinstance(command_args, list):
command_args = [command_args]
args.extend(command_args)
# create configured process pipes
prockwargs = dict()
prockwargs["shell"] = False
prockwargs["stdin"] = PIPE
prockwargs["stdout"] = PIPE if STDIOTYPE.STDOUT in stdIOType else None
prockwargs["stderr"] = PIPE if STDIOTYPE.STDERR in stdIOType else None
prockwargs["cwd"] = cwd
# create process
self.proc = Popen(args, **prockwargs)
# create process handler
self.stdoutHandle = None
if STDIOTYPE.STDOUT in stdIOType:
self.stdoutHandle = _StdOutHandler(stdio=self.proc.stdout, lineWrapper=OutLineWrapper, **kwargs)
self.stdoutHandle.register(self._newDataCallbackFnc)
self.stderrHandle = None
if STDIOTYPE.STDERR in stdIOType:
self.stderrHandle = _StdOutHandler(stdio=self.proc.stderr, lineWrapper=ErrorLineWrapper, **kwargs)
self.stderrHandle.register(self._newDataCallbackFnc)
self.stdioHandle = [self.stdoutHandle, self.stderrHandle]
def __init__(self,
server=None,
user_id=0,
initial_version=None,
initial={}):
self.state = initial
self.initial_state = initial
self.history = History()
self.user_id = user_id
self.version = VectorClock(user_id, initial_version)
self.server = server
if server is not None:
self.server.on_recieve += self.on_recieve
self.on_update = Callback()
def __init__(self, host, port):
self.host = host
self.port = port
self.listeners_running = False
self.senders_running = False
self.in_queue = Queue()
self.out_queue = Queue()
self.on_recieve = Callback()
# initial socket setup
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.sock.bind((self.host, self.port))
self.sock.settimeout(60)
# TODO: increase number of listeners?
self.sock.listen(5)
self.clients: List[socket.socket] = [self.sock]
def computeNewModels(filePath,
name,
step=10000,
maxLines=100001,
size=100,
min_count=5,
iter=5,
sg=0,
negative=5):
treated_lines = [0]
time_array = []
loss = 0
for i in range(maxLines):
if i % step == 0 and i != 0:
depart_time = time.time()
print("Ligne " + str(i) + " sur " + str(maxLines))
partialCorpus = MyPartialCorpus(filePath, i)
loss_callback = Callback()
model = gensim.models.Word2Vec(sentences=partialCorpus,
workers=multiprocessing.cpu_count(),
size=size,
min_count=min_count,
iter=iter,
sg=sg,
negative=negative,
compute_loss=True,
callbacks=[loss_callback])
loss = loss_callback.loss_previous_step / (maxLines - 1)
print("Temps pour entrainer le model: " +
str(time.time() - depart_time))
time_array.append(time.time() - depart_time)
writeNewModel(treated_lines, time_array, name)
return loss
def parse_callbacks(self):
""" Field nodes can have callback subnodes, parse them here """
from Callback import Callback
for cb in self.find_children(self.ns(self.NS_CORE, 'callback')):
self.callbacks.append(Callback(self._namespace, cb))
