def process_worker(task_queue):
while True:
tid, task_info = task_queue.get()
task_queue.put((-1, ("Working", tid)))
(task, argv, kwargs) = Husky.loads(task_info)
start_time = time.time()
res = task(*argv, **kwargs)
used_time = time.time() - start_time
finish_time = time.time()
task_queue.put((tid, Husky.dumps((res, start_time, finish_time))))
task_queue.put((-1, ("Idle", None)))
def process_worker(task_queue):
while True:
tid, task_info = task_queue.get()
task_queue.put((-1, ("Working", tid)))
(task, argv, kwargs) = Husky.loads(task_info)
start_time = time.time()
res = task(*argv, **kwargs)
used_time = time.time() - start_time
finish_time = time.time()
task_queue.put((tid, Husky.dumps((res, start_time, finish_time))))
task_queue.put((-1, ("Idle", None)))
if not header: return False
length = struct.unpack(self.HEADER_STRUCT, header)[0]
chunks = []
while length:
recv = safe_recv(self._sock, length)
if not recv: return False
chunks.append(recv)
length -= len(recv)
return "".join(chunks)
def write(self, bytes):
msg = struct.pack(self.HEADER_STRUCT, len(bytes)) + bytes
return safe_send(self._sock, msg)
def close(self):
self._sock.shutdown(SHUT_RDWR)
self._sock.close()
dumps = lambda x: snappy.compress(Husky.dumps(x))
loads = lambda x: Husky.loads(snappy.decompress(x))
class ObjPort(Port):
def read(self):
res = super(ObjPort, self).read()
if res:
res = loads(res)
return res
def write(self, bytes):
return super(ObjPort, self).write(dumps(bytes))
def run(self, tid, task, *argv, **kwargs):
task_info = Husky.dumps((task, argv, kwargs))
self.puppet.submit_task(tid, task_info)
res = self.puppet.fetch_result(tid)
self.queue.discard(tid)
return Husky.loads(res)
def reg_mod(self, name, mod):
self.redis.set("mod_%s" % name, Husky.dumps(mod))
def put(self, value):
self.id = uuid1().hex
value = Husky.dumps(value)
kvstore.set(self.id, value, serialize=False)
def get(self, key):
res = kvstore.get(key, serialize=False)
# while not res:
# sleep(0.01)
# res = kvstore.get(key, serialize=False)
return Husky.loads(res)
def __getstate__(self):
return Husky.dumps(self.f)
def __setstate__(self, state):
self.f = Husky.loads(state)
def lpush(self, item):
self.redis.lpush(self.name, Husky.dumps(item))
return True
def rpop(self):
return Husky.loads(self.redis.brpop(self.name, 0)[1])
def lpush(self, item):
self.redis.lpush(self.name, Husky.dumps(item))
return True
def rpop(self):
return Husky.loads(self.redis.brpop(self.name, 0)[1])
def reg_mod(self, name, mod):
self.redis.set("mod_%s" % name, Husky.dumps(mod))
