def _test_if_pipe_blocks(self, buffer_class): r, w = self.pipe() # set nbytes such that for sure it is > maximum pipe buffer nbytes = 1000000 block = b'x' * 4096 buf = buffer_class(block) # Lack of "nonlocal" keyword in Python 2.x: bytesread = [0] byteswritten = [0] def produce(): while byteswritten[0] != nbytes: bytesleft = nbytes - byteswritten[0] byteswritten[0] += self.write(w, buf[:min(bytesleft, 4096)]) def consume(): while bytesread[0] != nbytes: bytesleft = nbytes - bytesread[0] bytesread[0] += len(self.read(r, min(bytesleft, 4096))) producer = Greenlet(produce) producer.start() consumer = Greenlet(consume) consumer.start_later(1) # If patching was not succesful, the producer will have filled # the pipe before the consumer starts, and would block the entire # process. Therefore the next line would never finish. joinall([producer, consumer]) self.assertEqual(bytesread[0], nbytes) self.assertEqual(bytesread[0], byteswritten[0])
def run(self): dispatcher = downloader.init() stopper = Greenlet(downloader.stop) stopper.start_later(5) dispatcher.join()
def test_if_pipe_blocks(self): r, w = self.pipe() # set nbytes such that for sure it is > maximum pipe buffer nbytes = 1000000 block = six.b('x') * 4096 buf = buffer(block) # Lack of "nonlocal" keyword in Python 2.x: bytesread = [0] byteswritten = [0] def produce(): while byteswritten[0] != nbytes: bytesleft = nbytes - byteswritten[0] byteswritten[0] += self.write(w, buf[:min(bytesleft, 4096)]) def consume(): while bytesread[0] != nbytes: bytesleft = nbytes - bytesread[0] bytesread[0] += len(self.read(r, min(bytesleft, 4096))) producer = Greenlet(produce) producer.start() consumer = Greenlet(consume) consumer.start_later(1) # If patching was not succesful, the producer will have filled # the pipe before the consumer starts, and would block the entire # process. Therefore the next line would never finish. joinall([producer, consumer]) assert bytesread[0] == nbytes assert bytesread[0] == byteswritten[0]
def random_delay_broadcast1(inputs, t): maxdelay = 0.01 N = len(inputs) buffers = map(lambda _: Queue(1), inputs) # Instantiate the "broadcast" instruction def makeBroadcast(i): def _broadcast(v): def _deliver(j): buffers[j].put((i,v)) for j in range(N): Greenlet(_deliver, j).start_later(random.random()*maxdelay) return _broadcast def makeOutput(i): def _output(v): print '[%d]' % i, 'output:', v return _output ts = [] for i in range(N): bc = makeBroadcast(i) recv = buffers[i].get outp = makeOutput(i) inp = bv_broadcast(i, N, t, bc, recv, outp) th = Greenlet(inp, inputs[i]) th.start_later(random.random()*maxdelay) ts.append(th) try: gevent.joinall(ts) except gevent.hub.LoopExit: pass
def produce(self, session, time): goods = [] if not self.type.from_good: return self.time_produce(time) for type in self.type.from_good: for good in self.owner.goods: if type == good.type: goods.append(good) break if not goods: return False quantity = min(goods, key=lambda g: g.quantity).quantity quantity += self.quantitylevel for good in goods: session.delete(good) session.commit() self.producing = True gl = Greenlet(self.create_product, session, quantity) gl.start_later(quantity * time) return True
def CompleteIn(n): """Returns an AsyncResult that completes in <n> seconds Args: n - The number of seconds to wait before completing. """ ar = AsyncResult() def helper(): ar.set() g = Greenlet(helper) g.start_later(float(n)) return ar
def _broadcast(v): def _deliver(j): random_delay_multivalue_consensus.msgCount += 1 tmpCount = random_delay_multivalue_consensus.msgCount mylog(bcolors.OKGREEN + "MSG: [%d] -[%d]-> [%d]: %s" % (i, tmpCount, j, repr(v)) + bcolors.ENDC) buffers[j].put((i,v)) mylog(bcolors.OKGREEN + " [%d] -[%d]-> [%d]: Finish" % (i, tmpCount, j) + bcolors.ENDC) for j in range(N): g = Greenlet(_deliver, j) g.start_later(random.random()*maxdelay) msgThreads.append(g) # Keep reference
def random_delay_multivalue_consensus(N, t, inputs): maxdelay = 0.01 msgThreads = [] buffers = map(lambda _: Queue(1), range(N)) random_delay_multivalue_consensus.msgCount = 0 # Instantiate the "broadcast" instruction def makeBroadcast(i): def _broadcast(v): def _deliver(j): random_delay_multivalue_consensus.msgCount += 1 tmpCount = random_delay_multivalue_consensus.msgCount mylog(bcolors.OKGREEN + "MSG: [%d] -[%d]-> [%d]: %s" % (i, tmpCount, j, repr(v)) + bcolors.ENDC) buffers[j].put((i, v)) mylog(bcolors.OKGREEN + " [%d] -[%d]-> [%d]: Finish" % (i, tmpCount, j) + bcolors.ENDC) for j in range(N): g = Greenlet(_deliver, j) g.start_later(random.random() * maxdelay) msgThreads.append(g) # Keep reference return _broadcast ts = [] #cid = 1 for i in range(N): bc = makeBroadcast(i) recv = buffers[i].get vi = inputs[i] th = Greenlet(mv84consensus, i, N, t, vi, bc, recv) th.start_later(random.random() * maxdelay) ts.append(th) try: gevent.joinall(ts) except gevent.hub.LoopExit: # Manual fix for early stop agreed = "" for key, value in globalState.items(): if globalState[key] != "": agreed = globalState[key] for key, value in globalState.items(): if globalState[key] == "": globalState[key] = agreed if globalState[key] != agreed: print "Consensus Error" print globalState
def _broadcast(v): def _deliver(j): random_delay_multivalue_consensus.msgCount += 1 tmpCount = random_delay_multivalue_consensus.msgCount mylog(bcolors.OKGREEN + "MSG: [%d] -[%d]-> [%d]: %s" % (i, tmpCount, j, repr(v)) + bcolors.ENDC) buffers[j].put((i, v)) mylog(bcolors.OKGREEN + " [%d] -[%d]-> [%d]: Finish" % (i, tmpCount, j) + bcolors.ENDC) for j in range(N): g = Greenlet(_deliver, j) g.start_later(random.random() * maxdelay) msgThreads.append(g) # Keep reference
def _queue_keepalive(self, persona, timeout=900): """ Queue keepalive for persona in @param timeout seconds (default 15 minutes) """ buf = 10 # seconds remaining = (self._get_session(persona)['timeout'] - datetime.datetime.utcnow()).seconds - buf if (remaining - buf) < 0: remaining = 2 self.logger.debug("Next keepalive for {} queued in {} seconds".format(persona, remaining)) ping = Greenlet(self._keepalive, persona) ping.start_later(remaining)
def check_users_online(): """check for expired users and send a message they left the room""" before_timestamp = int((time.time()) - (USER_TIMEOUT)) logging.info("checking users online, purging before %s" % before_timestamp) if using_redis: redis_check_users_online(before_timestamp) else: list_check_users_online(before_timestamp) ## setup our next check g = Greenlet(check_users_online) g.start_later(USER_TIMEOUT_INTERVAL)
def queue_keepalive(self, persona): """ Send keep-alive some time before the session times out """ if persona.id not in self.sessions: send_in_seconds = 2 else: buf = 30 # seconds timeout = dateutil_parse(self.sessions[persona.id]['timeout']) send_in_seconds = (timeout - datetime.datetime.now()).seconds - buf if send_in_seconds < 0: send_in_seconds = 2 ping = Greenlet(self.keep_alive, persona) ping.start_later(send_in_seconds)
def myelin_receive(self, recipient_id, interval=None): """ Request Vesicles directed at recipient from Myelin and pass them on to Synapse for handling. Parameters: recipient_id (String) The ID of the Persona for which to listen interval (int) If set to an amount of seconds, the function will repeatedly be called again in this interval """ recipient = Persona.query.get(recipient_id) if not recipient: self.logger.error("Could not find Persona {}".format(recipient_id)) return self.logger.debug("Updating Myelin of {} at {} second intervals".format(recipient, interval)) params = dict() # Determine offset offset = recipient.myelin_offset if offset is not None: self.logger.debug("Last Vesicle for {} received {} ({})".format( recipient, naturaltime(datetime.datetime.utcnow() - offset), offset)) params["offset"] = str(recipient.myelin_offset) resp, errors = self._request_resource("GET", ["myelin", "recipient", recipient.id], params, None) if errors: self._log_errors("Error receiving from Myelin", errors) else: for v in resp["vesicles"]: vesicle = self.synapse.handle_vesicle(v) if vesicle is not None: myelin_modified = iso8601.parse_date( resp["meta"]["myelin_modified"][vesicle.id]).replace(tzinfo=None) if offset is None or myelin_modified > offset: offset = myelin_modified # Update recipient's offset if a more recent Vesicle has been received if offset is not None: if recipient.myelin_offset is None or offset > recipient.myelin_offset: recipient.myelin_offset = offset session = create_session() session.add(recipient) session.commit() # session.close() # Schedule this method to be called in again in interval seconds if interval is not None: update = Greenlet(self.myelin_receive, recipient_id, interval) update.start_later(interval)
def random_delay_multivalue_consensus(N, t, inputs): maxdelay = 0.01 msgThreads = [] buffers = map(lambda _: Queue(1), range(N)) random_delay_multivalue_consensus.msgCount = 0 # Instantiate the "broadcast" instruction def makeBroadcast(i): def _broadcast(v): def _deliver(j): random_delay_multivalue_consensus.msgCount += 1 tmpCount = random_delay_multivalue_consensus.msgCount mylog(bcolors.OKGREEN + "MSG: [%d] -[%d]-> [%d]: %s" % (i, tmpCount, j, repr(v)) + bcolors.ENDC) buffers[j].put((i,v)) mylog(bcolors.OKGREEN + " [%d] -[%d]-> [%d]: Finish" % (i, tmpCount, j) + bcolors.ENDC) for j in range(N): g = Greenlet(_deliver, j) g.start_later(random.random()*maxdelay) msgThreads.append(g) # Keep reference return _broadcast ts = [] #cid = 1 for i in range(N): bc = makeBroadcast(i) recv = buffers[i].get vi = inputs[i] th = Greenlet(mv84consensus, i, N, t, vi, bc, recv) th.start_later(random.random() * maxdelay) ts.append(th) try: gevent.joinall(ts) except gevent.hub.LoopExit: # Manual fix for early stop agreed = "" for key, value in globalState.items(): if globalState[key] != "": agreed = globalState[key] for key, value in globalState.items(): if globalState[key] == "": globalState[key] = agreed if globalState[key] != agreed: print "Consensus Error" print globalState
def time_produce(self, time): goods = [] for good in self.owner.goods: if good.type == self.type.to_good: goods.append(good) if goods: good = min(goods, key=lambda g: g.quantity) good.quantity += self.quantitylevel return True else: quantity = self.quantitylevel * time gl = Greenlet(self.create_product, session, quantity) gl.start_later(quantity * time) self.producing = True return True
class Periodic(object): def __init__(self, interval, f, *args, **kwargs): self.interval = interval self.f = f self.args = args self.kwargs = kwargs self._greenlet = None def _run(self): while True: spawn_raw(self.f, *self.args, **self.kwargs) sleep(self.interval) def _discard_greenlet(self, val): self._greenlet = None @property def started(self): return bool(self._greenlet) def start(self, right_away=True): if self._greenlet: raise RuntimeError("Periodic already started.") self._greenlet = Greenlet(self._run) self._greenlet.link(self._discard_greenlet) if right_away: self._greenlet.start() else: self._greenlet.start_later(self.interval) def stop(self, block=True, timeout=None): if not self._greenlet: raise RuntimeError("Periodic is not started") self._greenlet.kill(block=block, timeout=timeout) self._greenlet = None def __repr__(self): return "<Periodic[%.2f seconds, %s] %r(*%r, **%r)>" % ( self.interval, "running" if self.started else "stopped", self.f, self.args, self.kwargs, )
def random_delay_binary_consensus(N, t): maxdelay = 0.01 buffers = map(lambda _: Queue(1), range(N)) # Instantiate the "broadcast" instruction def makeBroadcast(i): def _broadcast(v): def _deliver(j): mylog(bcolors.OKGREEN + "MSG: [%d] -> [%d]: %s" % (i, j, repr(v)) + bcolors.ENDC) buffers[j].put((i, v)) mylog(bcolors.OKGREEN + " [%d] -> [%d]: Finish" % (i, j) + bcolors.ENDC) for j in range(N): Greenlet(_deliver, j).start_later(random.random() * maxdelay) return _broadcast ts = [] for i in range(N): bc = makeBroadcast(i) recv = buffers[i].get vi = random.randint(0, 1) decideChannel = Queue(1) th = Greenlet(binary_consensus, i, N, t, vi, decideChannel, bc, recv) th.start_later(random.random() * maxdelay) ts.append(th) try: gevent.joinall(ts) except gevent.hub.LoopExit: # Manual fix for early stop agreed = "" for key, value in globalState.items(): if globalState[key] != "": agreed = globalState[key] for key, value in globalState.items(): if globalState[key] == "": globalState[key] = agreed if globalState[key] != agreed: print "Consensus Error" print globalState
class Periodic(object): def __init__(self, interval, f, *args, **kwargs): self.interval = interval self.f = f self.args = args self.kwargs = kwargs self._greenlet = None def _run(self): while True: spawn_raw(self.f, *self.args, **self.kwargs) sleep(self.interval) def _discard_greenlet(self, val): self._greenlet = None @property def started(self): return bool(self._greenlet) def start(self, right_away=True): if self._greenlet: raise RuntimeError("Periodic already started.") self._greenlet = Greenlet(self._run) self._greenlet.link(self._discard_greenlet) if right_away: self._greenlet.start() else: self._greenlet.start_later(self.interval) def stop(self, block=True, timeout=None): if not self._greenlet: raise RuntimeError("Periodic is not started") self._greenlet.kill(block=block, timeout=timeout) self._greenlet = None def __repr__(self): return "<Periodic[%.2f seconds, %s] %r(*%r, **%r)>" % ( self.interval, 'running' if self.started else 'stopped', self.f, self.args, self.kwargs)
def _schedule_new_greenlet(self, func: Callable, *args: Any, in_seconds_from_now: int = None, **kwargs: Any) -> Greenlet: """ Spawn a sub-task and ensures an error on it crashes self/main greenlet """ def on_success(greenlet: Greenlet) -> None: if greenlet in self.greenlets: self.greenlets.remove(greenlet) greenlet = Greenlet(func, *args, **kwargs) greenlet.link_exception(self.on_error) greenlet.link_value(on_success) self.greenlets.append(greenlet) if in_seconds_from_now: greenlet.start_later(in_seconds_from_now) else: greenlet.start() return greenlet
def random_delay_binary_consensus(N, t, inputs): maxdelay = 0.01 buffers = map(lambda _: Queue(1), range(N)) random_delay_binary_consensus.msgCount = 0 # Instantiate the "broadcast" instruction def makeBroadcast(i): def _broadcast(v): def _deliver(j): random_delay_binary_consensus.msgCount += 1 tmpCount = random_delay_binary_consensus.msgCount mylog(bcolors.OKGREEN + "MSG: [%d] -[%d]-> [%d]: %s" % (i, tmpCount, j, repr(v)) + bcolors.ENDC) buffers[j].put((i, v)) mylog(bcolors.OKGREEN + " [%d] -[%d]-> [%d]: Finish" % (i, tmpCount, j) + bcolors.ENDC) for j in range(N): Greenlet(_deliver, j).start_later(random.random() * maxdelay) return _broadcast ts = [] for i in range(N): bc = makeBroadcast(i) recv = buffers[i].get vi = inputs[i] #random.randint(0, 1) decideChannel = Queue(1) th = Greenlet(binary_consensus, i, N, t, vi, decideChannel, bc, recv) th.start_later(random.random() * maxdelay) ts.append(th) gevent.joinall(ts) for key, item in globalState.items(): if item != globalState[0]: mylog(bcolors.FAIL + 'Bad Concensus!' + bcolors.ENDC) print globalState
def random_delay_sharedcoin_dummy(N, t): maxdelay = 0.01 buffers = map(lambda _: Queue(1), range(N)) # Instantiate the "broadcast" instruction def makeBroadcast(i): def _broadcast(v): def _deliver(j): buffers[j].put((i, v)) for j in range(N): Greenlet(_deliver, j).start_later(random.random() * maxdelay) return _broadcast def _run(i, coin): # Party i, continue to run the shared coin r = 0 while r < 5: gevent.sleep(random.random() * maxdelay) print '[', i, '] at round ', r b = next(coin) print '[', i, '] bit[%d]:' % r, b r += 1 print '[', i, '] done' ts = [] for i in range(N): bc = makeBroadcast(i) recv = buffers[i].get coin = shared_coin_dummy(i, N, t, bc, recv) th = Greenlet(_run, i, coin) th.start_later(random.random() * maxdelay) ts.append(th) try: gevent.joinall(ts) except gevent.hub.LoopExit: pass
def random_delay_binary_consensus(N, t): maxdelay = 0.01 buffers = map(lambda _: Queue(1), range(N)) # Instantiate the "broadcast" instruction def makeBroadcast(i): def _broadcast(v): def _deliver(j): mylog(bcolors.OKGREEN + "MSG: [%d] -> [%d]: %s" % (i, j, repr(v)) + bcolors.ENDC) buffers[j].put((i,v)) mylog(bcolors.OKGREEN + " [%d] -> [%d]: Finish" % (i, j) + bcolors.ENDC) for j in range(N): Greenlet(_deliver, j).start_later(random.random()*maxdelay) return _broadcast ts = [] for i in range(N): bc = makeBroadcast(i) recv = buffers[i].get vi = random.randint(0, 1) decideChannel = Queue(1) th = Greenlet(binary_consensus, i, N, t, vi, decideChannel, bc, recv) th.start_later(random.random() * maxdelay) ts.append(th) try: gevent.joinall(ts) except gevent.hub.LoopExit: # Manual fix for early stop agreed = "" for key, value in globalState.items(): if globalState[key] != "": agreed = globalState[key] for key, value in globalState.items(): if globalState[key] == "": globalState[key] = agreed if globalState[key] != agreed: print "Consensus Error" print globalState
def check_users_online(redis_server): """check for expired users and send a message they left the room""" before_timestamp = int((time.time()) - (ENVIRONMENT['USER_TIMEOUT'])) logging.info("checking users online, purging before %s" % before_timestamp) logging.info("checking for users before: %s" % before_timestamp) expired_users_count = redis_server.zcount( ENVIRONMENT['REDIS_PREFIX'] + "users_timestamp", 0, before_timestamp) logging.info("found %d users to expire" % expired_users_count) if expired_users_count > 0: expired_users = redis_server.zrange( ENVIRONMENT['REDIS_PREFIX'] + "users_timestamp", 0, expired_users_count) if expired_users != None: for key in expired_users: channel_name = key.split(':')[0] username = key.split(':')[1] key = ENVIRONMENT['REDIS_PREFIX'] + "users:%s" % username data = redis_server.get(key) if data != None: user = User(**json.loads(data)) msg = ChatMessage( nickname='system', username='******', message="%s can not been found in the room" % user.nickname, channel_name=channel_name) chat_channel = get_chat_channel(redis_server, channel_name) chat_channel.add_chat_message(msg) chat_channel.remove_user(user) else: logging.info("unable to find expired user: %s" % (key)) ## setup our next check g = Greenlet(check_users_online, redis_server) g.start_later(ENVIRONMENT['USER_TIMEOUT_INTERVAL'])
def random_delay_binary_consensus(N, t, inputs): maxdelay = 0.01 buffers = map(lambda _: Queue(1), range(N)) random_delay_binary_consensus.msgCount = 0 # Instantiate the "broadcast" instruction def makeBroadcast(i): def _broadcast(v): def _deliver(j): random_delay_binary_consensus.msgCount += 1 tmpCount = random_delay_binary_consensus.msgCount mylog(bcolors.OKGREEN + "MSG: [%d] -[%d]-> [%d]: %s" % (i, tmpCount, j, repr(v)) + bcolors.ENDC) buffers[j].put((i, v)) mylog(bcolors.OKGREEN + " [%d] -[%d]-> [%d]: Finish" % (i, tmpCount, j) + bcolors.ENDC) for j in range(N): Greenlet(_deliver, j).start_later(random.random()*maxdelay) return _broadcast ts = [] for i in range(N): bc = makeBroadcast(i) recv = buffers[i].get vi = inputs[i] #random.randint(0, 1) decideChannel = Queue(1) th = Greenlet(binary_consensus, i, N, t, vi, decideChannel, bc, recv) th.start_later(random.random() * maxdelay) ts.append(th) gevent.joinall(ts) for key, item in globalState.items(): if item != globalState[0]: mylog(bcolors.FAIL + 'Bad Concensus!' + bcolors.ENDC) print globalState
def random_delay_sharedcoin_dummy(N, t): maxdelay = 0.01 buffers = map(lambda _: Queue(1), range(N)) # Instantiate the "broadcast" instruction def makeBroadcast(i): def _broadcast(v): def _deliver(j): buffers[j].put((i,v)) for j in range(N): Greenlet(_deliver, j).start_later(random.random()*maxdelay) return _broadcast def _run(i, coin): # Party i, continue to run the shared coin r = 0 while r < 5: gevent.sleep(random.random() * maxdelay) print '[',i,'] at round ', r b = next(coin) print '[',i,'] bit[%d]:'%r, b r += 1 print '[',i,'] done' ts = [] for i in range(N): bc = makeBroadcast(i) recv = buffers[i].get coin = shared_coin_dummy(i, N, t, bc, recv) th = Greenlet(_run, i, coin) th.start_later(random.random() * maxdelay) ts.append(th) try: gevent.joinall(ts) except gevent.hub.LoopExit: pass
def random_delay_broadcast1(inputs, t): maxdelay = 0.01 N = len(inputs) buffers = map(lambda _: Queue(1), inputs) # Instantiate the "broadcast" instruction def makeBroadcast(i): def _broadcast(v): def _deliver(j): buffers[j].put((i, v)) for j in range(N): Greenlet(_deliver, j).start_later(random.random() * maxdelay) return _broadcast def makeOutput(i): def _output(v): print '[%d]' % i, 'output:', v return _output ts = [] for i in range(N): bc = makeBroadcast(i) recv = buffers[i].get outp = makeOutput(i) inp = bv_broadcast(i, N, t, bc, recv, outp) th = Greenlet(inp, inputs[i]) th.start_later(random.random() * maxdelay) ts.append(th) try: gevent.joinall(ts) except gevent.hub.LoopExit: pass
def check_users_online(redis_server): """check for expired users and send a message they left the room""" before_timestamp = int((time.time()) - (ENVIRONMENT["USER_TIMEOUT"])) logging.info("checking users online, purging before %s" % before_timestamp) logging.info("checking for users before: %s" % before_timestamp) expired_users_count = redis_server.zcount(ENVIRONMENT["REDIS_PREFIX"] + "users_timestamp", 0, before_timestamp) logging.info("found %d users to expire" % expired_users_count) if expired_users_count > 0: expired_users = redis_server.zrange(ENVIRONMENT["REDIS_PREFIX"] + "users_timestamp", 0, expired_users_count) if expired_users != None: for key in expired_users: channel_name = key.split(":")[0] username = key.split(":")[1] key = ENVIRONMENT["REDIS_PREFIX"] + "users:%s" % username data = redis_server.get(key) if data != None: user = User(**json.loads(data)) msg = ChatMessage( nickname="system", username="******", message="%s can not been found in the room" % user.nickname, channel_name=channel_name, ) chat_channel = get_chat_channel(redis_server, channel_name) chat_channel.add_chat_message(msg) chat_channel.remove_user(user) else: logging.info("unable to find expired user: %s" % (key)) ## setup our next check g = Greenlet(check_users_online, redis_server) g.start_later(ENVIRONMENT["USER_TIMEOUT_INTERVAL"])
class Controller(object): """Manages a program's main loop in a Greenlet.""" def __init__(self, program_id=DEFAULT_PROGRAM): """Creates a controller to control the specified program. The program doesn't start executing until the start method is called.""" self.messages = Queue() self.program_id = program_id self.program = PROGRAMS[program_id]() self.green = None self.can_reset = False def start(self): """Starts (or resumes) the execution of the program.""" self.green = Greenlet(self.main_loop) self.green.start_later(START_DELAY) self.program.start() self.can_reset = True def stop(self): """Stops the execution of the program.""" self.program.stop() if self.green: self.green.kill() def reset(self): """Stops the program and resets it to its initial state.""" self.stop() self.program.reset() self.can_reset = False def switch_program(self, program_id): """Stops execution and switches to a new program.""" self.stop() self.program_id = program_id self.program = PROGRAMS[program_id]() self.can_reset = False def main_loop(self): """Runs the program's loop method continously, collecting any returned messages into the messages queue.""" while True: msg = self.program.loop() if msg: self.messages.put(msg) sleep(LOOP_DELAY) def __call__(self, command): """Accepts a command and either performs the desired action or passes the message on to the program. Returns a status message.""" if command == 'short:sync': pid = self.program_id running = bool(self.green) can_reset = self.can_reset return "{} {} {}".format(pid, running, can_reset) if command == 'short:param-help': return json.dumps(self.program.codes) if command == 'long:status': try: msg = self.messages.get(timeout=STATUS_POLL_TIMEOUT) except Empty: return None return msg if command.startswith(PROGRAM_PREFIX): prog = command[len(PROGRAM_PREFIX):] self.switch_program(prog) return "switched to {}".format(prog) if command == 'control:start': reason = self.program.no_start() if reason: return reason if self.green: return "already running" self.start() return "program resumed" if command == 'control:stop': if not self.green: return "not running" self.stop() return "program paused" if command == 'control:reset': self.reset() return "program reset" return self.program(command)
def client_test_freenet(N, t, options): ''' Test for the client with random delay channels command list i [target]: send a transaction to include for some particular party h [target]: stop some particular party m [target]: manually make particular party send some message help: show the help screen :param N: the number of parties :param t: the number of malicious parties :return None: ''' maxdelay = 0.01 initiateThresholdSig(open(options.threshold_keys, 'r').read()) initiateECDSAKeys(open(options.ecdsa, 'r').read()) initiateThresholdEnc(open(options.threshold_encs, 'r').read()) initializeGIPC(getKeys()[0]) buffers = map(lambda _: Queue(1), range(N)) '''global logGreenlet logGreenlet = Greenlet(logWriter, open('msglog.TorMultiple', 'w')) logGreenlet.parent_args = (N, t) logGreenlet.name = 'client_test_freenet.logWriter' logGreenlet.start()''' # Instantiate the "broadcast" instruction def makeBroadcast(i): def _broadcast(v): def _deliver(j): buffers[j].put(encode((j, i, v))) for j in range(N): Greenlet(_deliver, j).start() return _broadcast def recvWithDecode(buf): def recv(): s = buf.get() return decode(s)[1:] return recv def makeSend(i): # point to point message delivery def _send(j, v): buffers[j].put(encode((j, i, v))) return _send while True: #if True: initBeforeBinaryConsensus() ts = [] controlChannels = [Queue() for _ in range(N)] transactionSet = set([ encodeTransaction(randomTransaction()) for trC in range(int(options.tx)) ]) # we are using the same one for i in range(N): bc = makeBroadcast(i) recv = recvWithDecode(buffers[i]) th = Greenlet(honestParty, i, N, t, controlChannels[i], bc, recv, makeSend(i), options.B) controlChannels[i].put(('IncludeTransaction', transactionSet)) th.start_later(random.random() * maxdelay) ts.append(th) try: gevent.joinall(ts) except ACSException: print 'ACSException' gevent.killall(ts) except finishTransactionLeap: ### Manually jump to this level print 'msgCounter', msgCounter print 'msgTypeCounter', msgTypeCounter # message id 0 (duplicated) for signatureCost logChannel.put(StopIteration) mylog("=====", verboseLevel=-1) for item in logChannel: mylog(item, verboseLevel=-1) mylog("=====", verboseLevel=-1) continue except gevent.hub.LoopExit: # Manual fix for early stop print 'LoopExit' while True: gevent.sleep(1) checkExceptionPerGreenlet() finally: print "Concensus Finished" print 'End?!'
def client_test_freenet(N, t, options): ''' Test for the client with random delay channels command list i [target]: send a transaction to include for some particular party h [target]: stop some particular party m [target]: manually make particular party send some message help: show the help screen :param N: the number of parties :param t: the number of malicious parties :return None: ''' maxdelay = 0.01 initiateThresholdSig(open(options.threshold_keys, 'r').read()) initiateECDSAKeys(open(options.ecdsa, 'r').read()) initiateThresholdEnc(open(options.threshold_encs, 'r').read()) initializeGIPC(getKeys()[0]) buffers = map(lambda _: Queue(1), range(N)) global logGreenlet logGreenlet = Greenlet(logWriter, open('msglog.TorMultiple', 'w')) logGreenlet.parent_args = (N, t) logGreenlet.name = 'client_test_freenet.logWriter' logGreenlet.start() # Instantiate the "broadcast" instruction def makeBroadcast(i): def _broadcast(v): def _deliver(j): buffers[j].put(encode((j, i, v))) for j in range(N): Greenlet(_deliver, j).start() return _broadcast def recvWithDecode(buf): def recv(): s = buf.get() return decode(s)[1:] return recv def makeSend(i): # point to point message delivery def _send(j, v): buffers[j].put(encode((j, i, v))) return _send while True: #if True: initBeforeBinaryConsensus() ts = [] controlChannels = [Queue() for _ in range(N)] transactionSet = set([encodeTransaction(randomTransaction()) for trC in range(int(options.tx))]) # we are using the same one for i in range(N): bc = makeBroadcast(i) recv = recvWithDecode(buffers[i]) th = Greenlet(honestParty, i, N, t, controlChannels[i], bc, recv, makeSend(i), options.B) controlChannels[i].put(('IncludeTransaction', transactionSet)) th.start_later(random.random() * maxdelay) ts.append(th) try: gevent.joinall(ts) except ACSException: gevent.killall(ts) except finishTransactionLeap: ### Manually jump to this level print 'msgCounter', msgCounter print 'msgTypeCounter', msgTypeCounter # message id 0 (duplicated) for signatureCost logChannel.put(StopIteration) mylog("=====", verboseLevel=-1) for item in logChannel: mylog(item, verboseLevel=-1) mylog("=====", verboseLevel=-1) continue except gevent.hub.LoopExit: # Manual fix for early stop while True: gevent.sleep(1) checkExceptionPerGreenlet() finally: print "Concensus Finished"
def _response_handler (self, env, start_response): """handle HTTP request/response""" uri_path = env['PATH_INFO'] body = Queue() ## NB: these handler cases can be collapsed into a common pattern ## except for config/stop -- later ########################################## # shard lifecycle endpoints if uri_path == '/shard/config': # configure the service to run a shard payload = loads(env['wsgi.input'].read()) gl = Greenlet(self.shard_config, payload, body, start_response) gl.start() elif uri_path == '/shard/persist': # checkpoint the service state to durable storage payload = loads(env['wsgi.input'].read()) print "POST", payload ## TODO start_response('200 OK', [('Content-Type', 'text/plain')]) body.put("Bokay\r\n") body.put(StopIteration) elif uri_path == '/shard/recover': # restart the service, recovering from the most recent checkpoint payload = loads(env['wsgi.input'].read()) print "POST", payload ## TODO start_response('200 OK', [('Content-Type', 'text/plain')]) body.put("Bokay\r\n") body.put(StopIteration) ########################################## # HashRing endpoints elif uri_path == '/ring/init': # initialize the HashRing payload = loads(env['wsgi.input'].read()) gl = Greenlet(self.ring_init, payload, body, start_response) gl.start() elif uri_path == '/ring/add': # add a node to the HashRing payload = loads(env['wsgi.input'].read()) print "POST", payload ## TODO start_response('200 OK', [('Content-Type', 'text/plain')]) body.put("Bokay\r\n") body.put(StopIteration) elif uri_path == '/ring/del': # delete a node from the HashRing payload = loads(env['wsgi.input'].read()) print "POST", payload ## TODO start_response('200 OK', [('Content-Type', 'text/plain')]) body.put("Bokay\r\n") body.put(StopIteration) ########################################## # evolution endpoints elif uri_path == '/pop/init': # initialize the Population subset on this shard payload = loads(env['wsgi.input'].read()) gl = Greenlet(self.pop_init, payload, body, start_response) gl.start() elif uri_path == '/pop/gen': # create generation 0 of Individuals in this shard of the Population payload = loads(env['wsgi.input'].read()) gl = Greenlet(self.pop_gen, payload, body, start_response) gl.start() elif uri_path == '/pop/wait': # wait until all shards have finished sending reify requests payload = loads(env['wsgi.input'].read()) gl = Greenlet(self.pop_wait, payload, body, start_response) gl.start() elif uri_path == '/pop/join': # join on the reify queue, to wait until it empties payload = loads(env['wsgi.input'].read()) gl = Greenlet(self.pop_join, payload, body, start_response) gl.start() elif uri_path == '/pop/hist': # calculate a partial histogram for the fitness distribution payload = loads(env['wsgi.input'].read()) gl = Greenlet(self.pop_hist, payload, body, start_response) gl.start() elif uri_path == '/pop/next': # attempt to run another generation payload = loads(env['wsgi.input'].read()) gl = Greenlet(self.pop_next, payload, body, start_response) gl.start() elif uri_path == '/pop/enum': # enumerate the Individuals in this shard of the Population payload = loads(env['wsgi.input'].read()) gl = Greenlet(self.pop_enum, payload, body, start_response) gl.start() elif uri_path == '/pop/reify': # test/add a newly generated Individual into the Population (birth) payload = loads(env['wsgi.input'].read()) gl = Greenlet(self.pop_reify, payload, body, start_response) gl.start() elif uri_path == '/pop/evict': # remove an Individual from the Population (death) payload = loads(env['wsgi.input'].read()) print "POST", payload ## TODO start_response('200 OK', [('Content-Type', 'text/plain')]) body.put("Bokay\r\n") body.put(StopIteration) ########################################## # utility endpoints elif uri_path == '/': # dump info about the service in general start_response('200 OK', [('Content-Type', 'text/plain')]) body.put(str(env) + "\r\n") body.put(StopIteration) elif uri_path == '/stop': # shutdown the service payload = loads(env['wsgi.input'].read()) gl = Greenlet(self.stop, payload, body) gl.start_later(1) # HTTP response must start here, to avoid failure when server stops start_response('200 OK', [('Content-Type', 'text/plain')]) body.put("Goodbye\r\n") body.put(StopIteration) else: # ne znayu start_response('404 Not Found', [('Content-Type', 'text/plain')]) body.put('Not Found\r\n') body.put(StopIteration) return body