def _test_if_pipe_blocks(self, buffer_class):
r, w = self.pipe()
# set nbytes such that for sure it is > maximum pipe buffer
nbytes = 1000000
block = b'x' * 4096
buf = buffer_class(block)
# Lack of "nonlocal" keyword in Python 2.x:
bytesread = [0]
byteswritten = [0]
def produce():
while byteswritten[0] != nbytes:
bytesleft = nbytes - byteswritten[0]
byteswritten[0] += self.write(w, buf[:min(bytesleft, 4096)])
def consume():
while bytesread[0] != nbytes:
bytesleft = nbytes - bytesread[0]
bytesread[0] += len(self.read(r, min(bytesleft, 4096)))
producer = Greenlet(produce)
producer.start()
consumer = Greenlet(consume)
consumer.start_later(1)
# If patching was not succesful, the producer will have filled
# the pipe before the consumer starts, and would block the entire
# process. Therefore the next line would never finish.
joinall([producer, consumer])
self.assertEqual(bytesread[0], nbytes)
self.assertEqual(bytesread[0], byteswritten[0])
def run(self):
dispatcher = downloader.init()
stopper = Greenlet(downloader.stop)
stopper.start_later(5)
dispatcher.join()
def test_if_pipe_blocks(self):
r, w = self.pipe()
# set nbytes such that for sure it is > maximum pipe buffer
nbytes = 1000000
block = six.b('x') * 4096
buf = buffer(block)
# Lack of "nonlocal" keyword in Python 2.x:
bytesread = [0]
byteswritten = [0]
def produce():
while byteswritten[0] != nbytes:
bytesleft = nbytes - byteswritten[0]
byteswritten[0] += self.write(w, buf[:min(bytesleft, 4096)])
def consume():
while bytesread[0] != nbytes:
bytesleft = nbytes - bytesread[0]
bytesread[0] += len(self.read(r, min(bytesleft, 4096)))
producer = Greenlet(produce)
producer.start()
consumer = Greenlet(consume)
consumer.start_later(1)
# If patching was not succesful, the producer will have filled
# the pipe before the consumer starts, and would block the entire
# process. Therefore the next line would never finish.
joinall([producer, consumer])
assert bytesread[0] == nbytes
assert bytesread[0] == byteswritten[0]
def random_delay_broadcast1(inputs, t):
maxdelay = 0.01
N = len(inputs)
buffers = map(lambda _: Queue(1), inputs)
# Instantiate the "broadcast" instruction
def makeBroadcast(i):
def _broadcast(v):
def _deliver(j):
buffers[j].put((i,v))
for j in range(N):
Greenlet(_deliver, j).start_later(random.random()*maxdelay)
return _broadcast
def makeOutput(i):
def _output(v):
print '[%d]' % i, 'output:', v
return _output
ts = []
for i in range(N):
bc = makeBroadcast(i)
recv = buffers[i].get
outp = makeOutput(i)
inp = bv_broadcast(i, N, t, bc, recv, outp)
th = Greenlet(inp, inputs[i])
th.start_later(random.random()*maxdelay)
ts.append(th)
try:
gevent.joinall(ts)
except gevent.hub.LoopExit: pass
def produce(self, session, time):
goods = []
if not self.type.from_good:
return self.time_produce(time)
for type in self.type.from_good:
for good in self.owner.goods:
if type == good.type:
goods.append(good)
break
if not goods:
return False
quantity = min(goods, key=lambda g: g.quantity).quantity
quantity += self.quantitylevel
for good in goods:
session.delete(good)
session.commit()
self.producing = True
gl = Greenlet(self.create_product, session, quantity)
gl.start_later(quantity * time)
return True
def CompleteIn(n):
"""Returns an AsyncResult that completes in <n> seconds
Args:
n - The number of seconds to wait before completing.
"""
ar = AsyncResult()
def helper():
ar.set()
g = Greenlet(helper)
g.start_later(float(n))
return ar
def _broadcast(v):
def _deliver(j):
random_delay_multivalue_consensus.msgCount += 1
tmpCount = random_delay_multivalue_consensus.msgCount
mylog(bcolors.OKGREEN + "MSG: [%d] -[%d]-> [%d]: %s" % (i, tmpCount, j, repr(v)) + bcolors.ENDC)
buffers[j].put((i,v))
mylog(bcolors.OKGREEN + " [%d] -[%d]-> [%d]: Finish" % (i, tmpCount, j) + bcolors.ENDC)
for j in range(N):
g = Greenlet(_deliver, j)
g.start_later(random.random()*maxdelay)
msgThreads.append(g) # Keep reference
def random_delay_multivalue_consensus(N, t, inputs):
maxdelay = 0.01
msgThreads = []
buffers = map(lambda _: Queue(1), range(N))
random_delay_multivalue_consensus.msgCount = 0
# Instantiate the "broadcast" instruction
def makeBroadcast(i):
def _broadcast(v):
def _deliver(j):
random_delay_multivalue_consensus.msgCount += 1
tmpCount = random_delay_multivalue_consensus.msgCount
mylog(bcolors.OKGREEN + "MSG: [%d] -[%d]-> [%d]: %s" %
(i, tmpCount, j, repr(v)) + bcolors.ENDC)
buffers[j].put((i, v))
mylog(bcolors.OKGREEN + " [%d] -[%d]-> [%d]: Finish" %
(i, tmpCount, j) + bcolors.ENDC)
for j in range(N):
g = Greenlet(_deliver, j)
g.start_later(random.random() * maxdelay)
msgThreads.append(g) # Keep reference
return _broadcast
ts = []
#cid = 1
for i in range(N):
bc = makeBroadcast(i)
recv = buffers[i].get
vi = inputs[i]
th = Greenlet(mv84consensus, i, N, t, vi, bc, recv)
th.start_later(random.random() * maxdelay)
ts.append(th)
try:
gevent.joinall(ts)
except gevent.hub.LoopExit: # Manual fix for early stop
agreed = ""
for key, value in globalState.items():
if globalState[key] != "":
agreed = globalState[key]
for key, value in globalState.items():
if globalState[key] == "":
globalState[key] = agreed
if globalState[key] != agreed:
print "Consensus Error"
print globalState
def _broadcast(v):
def _deliver(j):
random_delay_multivalue_consensus.msgCount += 1
tmpCount = random_delay_multivalue_consensus.msgCount
mylog(bcolors.OKGREEN + "MSG: [%d] -[%d]-> [%d]: %s" %
(i, tmpCount, j, repr(v)) + bcolors.ENDC)
buffers[j].put((i, v))
mylog(bcolors.OKGREEN + " [%d] -[%d]-> [%d]: Finish" %
(i, tmpCount, j) + bcolors.ENDC)
for j in range(N):
g = Greenlet(_deliver, j)
g.start_later(random.random() * maxdelay)
msgThreads.append(g) # Keep reference
def _queue_keepalive(self, persona, timeout=900):
"""
Queue keepalive for persona in @param timeout seconds (default 15 minutes)
"""
buf = 10 # seconds
remaining = (self._get_session(persona)['timeout'] - datetime.datetime.utcnow()).seconds - buf
if (remaining - buf) < 0:
remaining = 2
self.logger.debug("Next keepalive for {} queued in {} seconds".format(persona, remaining))
ping = Greenlet(self._keepalive, persona)
ping.start_later(remaining)
def check_users_online():
"""check for expired users and send a message they left the room"""
before_timestamp = int((time.time()) - (USER_TIMEOUT))
logging.info("checking users online, purging before %s" % before_timestamp)
if using_redis:
redis_check_users_online(before_timestamp)
else:
list_check_users_online(before_timestamp)
## setup our next check
g = Greenlet(check_users_online)
g.start_later(USER_TIMEOUT_INTERVAL)
def queue_keepalive(self, persona):
""" Send keep-alive some time before the session times out """
if persona.id not in self.sessions:
send_in_seconds = 2
else:
buf = 30 # seconds
timeout = dateutil_parse(self.sessions[persona.id]['timeout'])
send_in_seconds = (timeout - datetime.datetime.now()).seconds - buf
if send_in_seconds < 0:
send_in_seconds = 2
ping = Greenlet(self.keep_alive, persona)
ping.start_later(send_in_seconds)
def check_users_online():
"""check for expired users and send a message they left the room"""
before_timestamp = int((time.time()) - (USER_TIMEOUT))
logging.info("checking users online, purging before %s" % before_timestamp)
if using_redis:
redis_check_users_online(before_timestamp)
else:
list_check_users_online(before_timestamp)
## setup our next check
g = Greenlet(check_users_online)
g.start_later(USER_TIMEOUT_INTERVAL)
def CompleteIn(n):
"""Returns an AsyncResult that completes in <n> seconds
Args:
n - The number of seconds to wait before completing.
"""
ar = AsyncResult()
def helper():
ar.set()
g = Greenlet(helper)
g.start_later(float(n))
return ar
def myelin_receive(self, recipient_id, interval=None):
"""
Request Vesicles directed at recipient from Myelin and pass them on to Synapse for handling.
Parameters:
recipient_id (String) The ID of the Persona for which to listen
interval (int) If set to an amount of seconds, the function will repeatedly be called again in this interval
"""
recipient = Persona.query.get(recipient_id)
if not recipient:
self.logger.error("Could not find Persona {}".format(recipient_id))
return
self.logger.debug("Updating Myelin of {} at {} second intervals".format(recipient, interval))
params = dict()
# Determine offset
offset = recipient.myelin_offset
if offset is not None:
self.logger.debug("Last Vesicle for {} received {} ({})".format(
recipient, naturaltime(datetime.datetime.utcnow() - offset), offset))
params["offset"] = str(recipient.myelin_offset)
resp, errors = self._request_resource("GET", ["myelin", "recipient", recipient.id], params, None)
if errors:
self._log_errors("Error receiving from Myelin", errors)
else:
for v in resp["vesicles"]:
vesicle = self.synapse.handle_vesicle(v)
if vesicle is not None:
myelin_modified = iso8601.parse_date(
resp["meta"]["myelin_modified"][vesicle.id]).replace(tzinfo=None)
if offset is None or myelin_modified > offset:
offset = myelin_modified
# Update recipient's offset if a more recent Vesicle has been received
if offset is not None:
if recipient.myelin_offset is None or offset > recipient.myelin_offset:
recipient.myelin_offset = offset
session = create_session()
session.add(recipient)
session.commit()
# session.close()
# Schedule this method to be called in again in interval seconds
if interval is not None:
update = Greenlet(self.myelin_receive, recipient_id, interval)
update.start_later(interval)
def random_delay_multivalue_consensus(N, t, inputs):
maxdelay = 0.01
msgThreads = []
buffers = map(lambda _: Queue(1), range(N))
random_delay_multivalue_consensus.msgCount = 0
# Instantiate the "broadcast" instruction
def makeBroadcast(i):
def _broadcast(v):
def _deliver(j):
random_delay_multivalue_consensus.msgCount += 1
tmpCount = random_delay_multivalue_consensus.msgCount
mylog(bcolors.OKGREEN + "MSG: [%d] -[%d]-> [%d]: %s" % (i, tmpCount, j, repr(v)) + bcolors.ENDC)
buffers[j].put((i,v))
mylog(bcolors.OKGREEN + " [%d] -[%d]-> [%d]: Finish" % (i, tmpCount, j) + bcolors.ENDC)
for j in range(N):
g = Greenlet(_deliver, j)
g.start_later(random.random()*maxdelay)
msgThreads.append(g) # Keep reference
return _broadcast
ts = []
#cid = 1
for i in range(N):
bc = makeBroadcast(i)
recv = buffers[i].get
vi = inputs[i]
th = Greenlet(mv84consensus, i, N, t, vi, bc, recv)
th.start_later(random.random() * maxdelay)
ts.append(th)
try:
gevent.joinall(ts)
except gevent.hub.LoopExit: # Manual fix for early stop
agreed = ""
for key, value in globalState.items():
if globalState[key] != "":
agreed = globalState[key]
for key, value in globalState.items():
if globalState[key] == "":
globalState[key] = agreed
if globalState[key] != agreed:
print "Consensus Error"
print globalState
def time_produce(self, time):
goods = []
for good in self.owner.goods:
if good.type == self.type.to_good:
goods.append(good)
if goods:
good = min(goods, key=lambda g: g.quantity)
good.quantity += self.quantitylevel
return True
else:
quantity = self.quantitylevel * time
gl = Greenlet(self.create_product, session, quantity)
gl.start_later(quantity * time)
self.producing = True
return True
class Periodic(object):
def __init__(self, interval, f, *args, **kwargs):
self.interval = interval
self.f = f
self.args = args
self.kwargs = kwargs
self._greenlet = None
def _run(self):
while True:
spawn_raw(self.f, *self.args, **self.kwargs)
sleep(self.interval)
def _discard_greenlet(self, val):
self._greenlet = None
@property
def started(self):
return bool(self._greenlet)
def start(self, right_away=True):
if self._greenlet:
raise RuntimeError("Periodic already started.")
self._greenlet = Greenlet(self._run)
self._greenlet.link(self._discard_greenlet)
if right_away:
self._greenlet.start()
else:
self._greenlet.start_later(self.interval)
def stop(self, block=True, timeout=None):
if not self._greenlet:
raise RuntimeError("Periodic is not started")
self._greenlet.kill(block=block, timeout=timeout)
self._greenlet = None
def __repr__(self):
return "<Periodic[%.2f seconds, %s] %r(*%r, **%r)>" % (
self.interval,
"running" if self.started else "stopped",
self.f,
self.args,
self.kwargs,
)
def random_delay_binary_consensus(N, t):
maxdelay = 0.01
buffers = map(lambda _: Queue(1), range(N))
# Instantiate the "broadcast" instruction
def makeBroadcast(i):
def _broadcast(v):
def _deliver(j):
mylog(bcolors.OKGREEN + "MSG: [%d] -> [%d]: %s" %
(i, j, repr(v)) + bcolors.ENDC)
buffers[j].put((i, v))
mylog(bcolors.OKGREEN + " [%d] -> [%d]: Finish" % (i, j) +
bcolors.ENDC)
for j in range(N):
Greenlet(_deliver, j).start_later(random.random() * maxdelay)
return _broadcast
ts = []
for i in range(N):
bc = makeBroadcast(i)
recv = buffers[i].get
vi = random.randint(0, 1)
decideChannel = Queue(1)
th = Greenlet(binary_consensus, i, N, t, vi, decideChannel, bc, recv)
th.start_later(random.random() * maxdelay)
ts.append(th)
try:
gevent.joinall(ts)
except gevent.hub.LoopExit: # Manual fix for early stop
agreed = ""
for key, value in globalState.items():
if globalState[key] != "":
agreed = globalState[key]
for key, value in globalState.items():
if globalState[key] == "":
globalState[key] = agreed
if globalState[key] != agreed:
print "Consensus Error"
print globalState
class Periodic(object):
def __init__(self, interval, f, *args, **kwargs):
self.interval = interval
self.f = f
self.args = args
self.kwargs = kwargs
self._greenlet = None
def _run(self):
while True:
spawn_raw(self.f, *self.args, **self.kwargs)
sleep(self.interval)
def _discard_greenlet(self, val):
self._greenlet = None
@property
def started(self):
return bool(self._greenlet)
def start(self, right_away=True):
if self._greenlet:
raise RuntimeError("Periodic already started.")
self._greenlet = Greenlet(self._run)
self._greenlet.link(self._discard_greenlet)
if right_away:
self._greenlet.start()
else:
self._greenlet.start_later(self.interval)
def stop(self, block=True, timeout=None):
if not self._greenlet:
raise RuntimeError("Periodic is not started")
self._greenlet.kill(block=block, timeout=timeout)
self._greenlet = None
def __repr__(self):
return "<Periodic[%.2f seconds, %s] %r(*%r, **%r)>" % (
self.interval, 'running' if self.started else 'stopped', self.f,
self.args, self.kwargs)
def _schedule_new_greenlet(self,
func: Callable,
*args: Any,
in_seconds_from_now: int = None,
**kwargs: Any) -> Greenlet:
""" Spawn a sub-task and ensures an error on it crashes self/main greenlet """
def on_success(greenlet: Greenlet) -> None:
if greenlet in self.greenlets:
self.greenlets.remove(greenlet)
greenlet = Greenlet(func, *args, **kwargs)
greenlet.link_exception(self.on_error)
greenlet.link_value(on_success)
self.greenlets.append(greenlet)
if in_seconds_from_now:
greenlet.start_later(in_seconds_from_now)
else:
greenlet.start()
return greenlet
def random_delay_binary_consensus(N, t, inputs):
maxdelay = 0.01
buffers = map(lambda _: Queue(1), range(N))
random_delay_binary_consensus.msgCount = 0
# Instantiate the "broadcast" instruction
def makeBroadcast(i):
def _broadcast(v):
def _deliver(j):
random_delay_binary_consensus.msgCount += 1
tmpCount = random_delay_binary_consensus.msgCount
mylog(bcolors.OKGREEN + "MSG: [%d] -[%d]-> [%d]: %s" %
(i, tmpCount, j, repr(v)) + bcolors.ENDC)
buffers[j].put((i, v))
mylog(bcolors.OKGREEN + " [%d] -[%d]-> [%d]: Finish" %
(i, tmpCount, j) + bcolors.ENDC)
for j in range(N):
Greenlet(_deliver, j).start_later(random.random() * maxdelay)
return _broadcast
ts = []
for i in range(N):
bc = makeBroadcast(i)
recv = buffers[i].get
vi = inputs[i] #random.randint(0, 1)
decideChannel = Queue(1)
th = Greenlet(binary_consensus, i, N, t, vi, decideChannel, bc, recv)
th.start_later(random.random() * maxdelay)
ts.append(th)
gevent.joinall(ts)
for key, item in globalState.items():
if item != globalState[0]:
mylog(bcolors.FAIL + 'Bad Concensus!' + bcolors.ENDC)
print globalState
def random_delay_sharedcoin_dummy(N, t):
maxdelay = 0.01
buffers = map(lambda _: Queue(1), range(N))
# Instantiate the "broadcast" instruction
def makeBroadcast(i):
def _broadcast(v):
def _deliver(j):
buffers[j].put((i, v))
for j in range(N):
Greenlet(_deliver, j).start_later(random.random() * maxdelay)
return _broadcast
def _run(i, coin):
# Party i, continue to run the shared coin
r = 0
while r < 5:
gevent.sleep(random.random() * maxdelay)
print '[', i, '] at round ', r
b = next(coin)
print '[', i, '] bit[%d]:' % r, b
r += 1
print '[', i, '] done'
ts = []
for i in range(N):
bc = makeBroadcast(i)
recv = buffers[i].get
coin = shared_coin_dummy(i, N, t, bc, recv)
th = Greenlet(_run, i, coin)
th.start_later(random.random() * maxdelay)
ts.append(th)
try:
gevent.joinall(ts)
except gevent.hub.LoopExit:
pass
def random_delay_binary_consensus(N, t):
maxdelay = 0.01
buffers = map(lambda _: Queue(1), range(N))
# Instantiate the "broadcast" instruction
def makeBroadcast(i):
def _broadcast(v):
def _deliver(j):
mylog(bcolors.OKGREEN + "MSG: [%d] -> [%d]: %s" % (i, j, repr(v)) + bcolors.ENDC)
buffers[j].put((i,v))
mylog(bcolors.OKGREEN + " [%d] -> [%d]: Finish" % (i, j) + bcolors.ENDC)
for j in range(N):
Greenlet(_deliver, j).start_later(random.random()*maxdelay)
return _broadcast
ts = []
for i in range(N):
bc = makeBroadcast(i)
recv = buffers[i].get
vi = random.randint(0, 1)
decideChannel = Queue(1)
th = Greenlet(binary_consensus, i, N, t, vi, decideChannel, bc, recv)
th.start_later(random.random() * maxdelay)
ts.append(th)
try:
gevent.joinall(ts)
except gevent.hub.LoopExit: # Manual fix for early stop
agreed = ""
for key, value in globalState.items():
if globalState[key] != "":
agreed = globalState[key]
for key, value in globalState.items():
if globalState[key] == "":
globalState[key] = agreed
if globalState[key] != agreed:
print "Consensus Error"
print globalState
def check_users_online(redis_server):
"""check for expired users and send a message they left the room"""
before_timestamp = int((time.time()) - (ENVIRONMENT['USER_TIMEOUT']))
logging.info("checking users online, purging before %s" % before_timestamp)
logging.info("checking for users before: %s" % before_timestamp)
expired_users_count = redis_server.zcount(
ENVIRONMENT['REDIS_PREFIX'] + "users_timestamp", 0, before_timestamp)
logging.info("found %d users to expire" % expired_users_count)
if expired_users_count > 0:
expired_users = redis_server.zrange(
ENVIRONMENT['REDIS_PREFIX'] + "users_timestamp", 0,
expired_users_count)
if expired_users != None:
for key in expired_users:
channel_name = key.split(':')[0]
username = key.split(':')[1]
key = ENVIRONMENT['REDIS_PREFIX'] + "users:%s" % username
data = redis_server.get(key)
if data != None:
user = User(**json.loads(data))
msg = ChatMessage(
nickname='system',
username='******',
message="%s can not been found in the room" %
user.nickname,
channel_name=channel_name)
chat_channel = get_chat_channel(redis_server, channel_name)
chat_channel.add_chat_message(msg)
chat_channel.remove_user(user)
else:
logging.info("unable to find expired user: %s" % (key))
## setup our next check
g = Greenlet(check_users_online, redis_server)
g.start_later(ENVIRONMENT['USER_TIMEOUT_INTERVAL'])
def random_delay_binary_consensus(N, t, inputs):
maxdelay = 0.01
buffers = map(lambda _: Queue(1), range(N))
random_delay_binary_consensus.msgCount = 0
# Instantiate the "broadcast" instruction
def makeBroadcast(i):
def _broadcast(v):
def _deliver(j):
random_delay_binary_consensus.msgCount += 1
tmpCount = random_delay_binary_consensus.msgCount
mylog(bcolors.OKGREEN + "MSG: [%d] -[%d]-> [%d]: %s" % (i, tmpCount, j, repr(v)) + bcolors.ENDC)
buffers[j].put((i, v))
mylog(bcolors.OKGREEN + " [%d] -[%d]-> [%d]: Finish" % (i, tmpCount, j) + bcolors.ENDC)
for j in range(N):
Greenlet(_deliver, j).start_later(random.random()*maxdelay)
return _broadcast
ts = []
for i in range(N):
bc = makeBroadcast(i)
recv = buffers[i].get
vi = inputs[i] #random.randint(0, 1)
decideChannel = Queue(1)
th = Greenlet(binary_consensus, i, N, t, vi, decideChannel, bc, recv)
th.start_later(random.random() * maxdelay)
ts.append(th)
gevent.joinall(ts)
for key, item in globalState.items():
if item != globalState[0]:
mylog(bcolors.FAIL + 'Bad Concensus!' + bcolors.ENDC)
print globalState
def random_delay_sharedcoin_dummy(N, t):
maxdelay = 0.01
buffers = map(lambda _: Queue(1), range(N))
# Instantiate the "broadcast" instruction
def makeBroadcast(i):
def _broadcast(v):
def _deliver(j):
buffers[j].put((i,v))
for j in range(N):
Greenlet(_deliver, j).start_later(random.random()*maxdelay)
return _broadcast
def _run(i, coin):
# Party i, continue to run the shared coin
r = 0
while r < 5:
gevent.sleep(random.random() * maxdelay)
print '[',i,'] at round ', r
b = next(coin)
print '[',i,'] bit[%d]:'%r, b
r += 1
print '[',i,'] done'
ts = []
for i in range(N):
bc = makeBroadcast(i)
recv = buffers[i].get
coin = shared_coin_dummy(i, N, t, bc, recv)
th = Greenlet(_run, i, coin)
th.start_later(random.random() * maxdelay)
ts.append(th)
try:
gevent.joinall(ts)
except gevent.hub.LoopExit: pass
def random_delay_broadcast1(inputs, t):
maxdelay = 0.01
N = len(inputs)
buffers = map(lambda _: Queue(1), inputs)
# Instantiate the "broadcast" instruction
def makeBroadcast(i):
def _broadcast(v):
def _deliver(j):
buffers[j].put((i, v))
for j in range(N):
Greenlet(_deliver, j).start_later(random.random() * maxdelay)
return _broadcast
def makeOutput(i):
def _output(v):
print '[%d]' % i, 'output:', v
return _output
ts = []
for i in range(N):
bc = makeBroadcast(i)
recv = buffers[i].get
outp = makeOutput(i)
inp = bv_broadcast(i, N, t, bc, recv, outp)
th = Greenlet(inp, inputs[i])
th.start_later(random.random() * maxdelay)
ts.append(th)
try:
gevent.joinall(ts)
except gevent.hub.LoopExit:
pass
def check_users_online(redis_server):
"""check for expired users and send a message they left the room"""
before_timestamp = int((time.time()) - (ENVIRONMENT["USER_TIMEOUT"]))
logging.info("checking users online, purging before %s" % before_timestamp)
logging.info("checking for users before: %s" % before_timestamp)
expired_users_count = redis_server.zcount(ENVIRONMENT["REDIS_PREFIX"] + "users_timestamp", 0, before_timestamp)
logging.info("found %d users to expire" % expired_users_count)
if expired_users_count > 0:
expired_users = redis_server.zrange(ENVIRONMENT["REDIS_PREFIX"] + "users_timestamp", 0, expired_users_count)
if expired_users != None:
for key in expired_users:
channel_name = key.split(":")[0]
username = key.split(":")[1]
key = ENVIRONMENT["REDIS_PREFIX"] + "users:%s" % username
data = redis_server.get(key)
if data != None:
user = User(**json.loads(data))
msg = ChatMessage(
nickname="system",
username="******",
message="%s can not been found in the room" % user.nickname,
channel_name=channel_name,
)
chat_channel = get_chat_channel(redis_server, channel_name)
chat_channel.add_chat_message(msg)
chat_channel.remove_user(user)
else:
logging.info("unable to find expired user: %s" % (key))
## setup our next check
g = Greenlet(check_users_online, redis_server)
g.start_later(ENVIRONMENT["USER_TIMEOUT_INTERVAL"])
class Controller(object):
"""Manages a program's main loop in a Greenlet."""
def __init__(self, program_id=DEFAULT_PROGRAM):
"""Creates a controller to control the specified program. The program
doesn't start executing until the start method is called."""
self.messages = Queue()
self.program_id = program_id
self.program = PROGRAMS[program_id]()
self.green = None
self.can_reset = False
def start(self):
"""Starts (or resumes) the execution of the program."""
self.green = Greenlet(self.main_loop)
self.green.start_later(START_DELAY)
self.program.start()
self.can_reset = True
def stop(self):
"""Stops the execution of the program."""
self.program.stop()
if self.green:
self.green.kill()
def reset(self):
"""Stops the program and resets it to its initial state."""
self.stop()
self.program.reset()
self.can_reset = False
def switch_program(self, program_id):
"""Stops execution and switches to a new program."""
self.stop()
self.program_id = program_id
self.program = PROGRAMS[program_id]()
self.can_reset = False
def main_loop(self):
"""Runs the program's loop method continously, collecting any returned
messages into the messages queue."""
while True:
msg = self.program.loop()
if msg:
self.messages.put(msg)
sleep(LOOP_DELAY)
def __call__(self, command):
"""Accepts a command and either performs the desired action or passes
the message on to the program. Returns a status message."""
if command == 'short:sync':
pid = self.program_id
running = bool(self.green)
can_reset = self.can_reset
return "{} {} {}".format(pid, running, can_reset)
if command == 'short:param-help':
return json.dumps(self.program.codes)
if command == 'long:status':
try:
msg = self.messages.get(timeout=STATUS_POLL_TIMEOUT)
except Empty:
return None
return msg
if command.startswith(PROGRAM_PREFIX):
prog = command[len(PROGRAM_PREFIX):]
self.switch_program(prog)
return "switched to {}".format(prog)
if command == 'control:start':
reason = self.program.no_start()
if reason:
return reason
if self.green:
return "already running"
self.start()
return "program resumed"
if command == 'control:stop':
if not self.green:
return "not running"
self.stop()
return "program paused"
if command == 'control:reset':
self.reset()
return "program reset"
return self.program(command)
def client_test_freenet(N, t, options):
'''
Test for the client with random delay channels
command list
i [target]: send a transaction to include for some particular party
h [target]: stop some particular party
m [target]: manually make particular party send some message
help: show the help screen
:param N: the number of parties
:param t: the number of malicious parties
:return None:
'''
maxdelay = 0.01
initiateThresholdSig(open(options.threshold_keys, 'r').read())
initiateECDSAKeys(open(options.ecdsa, 'r').read())
initiateThresholdEnc(open(options.threshold_encs, 'r').read())
initializeGIPC(getKeys()[0])
buffers = map(lambda _: Queue(1), range(N))
'''global logGreenlet
logGreenlet = Greenlet(logWriter, open('msglog.TorMultiple', 'w'))
logGreenlet.parent_args = (N, t)
logGreenlet.name = 'client_test_freenet.logWriter'
logGreenlet.start()'''
# Instantiate the "broadcast" instruction
def makeBroadcast(i):
def _broadcast(v):
def _deliver(j):
buffers[j].put(encode((j, i, v)))
for j in range(N):
Greenlet(_deliver, j).start()
return _broadcast
def recvWithDecode(buf):
def recv():
s = buf.get()
return decode(s)[1:]
return recv
def makeSend(i): # point to point message delivery
def _send(j, v):
buffers[j].put(encode((j, i, v)))
return _send
while True:
#if True:
initBeforeBinaryConsensus()
ts = []
controlChannels = [Queue() for _ in range(N)]
transactionSet = set([
encodeTransaction(randomTransaction())
for trC in range(int(options.tx))
]) # we are using the same one
for i in range(N):
bc = makeBroadcast(i)
recv = recvWithDecode(buffers[i])
th = Greenlet(honestParty, i, N, t, controlChannels[i], bc, recv,
makeSend(i), options.B)
controlChannels[i].put(('IncludeTransaction', transactionSet))
th.start_later(random.random() * maxdelay)
ts.append(th)
try:
gevent.joinall(ts)
except ACSException:
print 'ACSException'
gevent.killall(ts)
except finishTransactionLeap: ### Manually jump to this level
print 'msgCounter', msgCounter
print 'msgTypeCounter', msgTypeCounter
# message id 0 (duplicated) for signatureCost
logChannel.put(StopIteration)
mylog("=====", verboseLevel=-1)
for item in logChannel:
mylog(item, verboseLevel=-1)
mylog("=====", verboseLevel=-1)
continue
except gevent.hub.LoopExit: # Manual fix for early stop
print 'LoopExit'
while True:
gevent.sleep(1)
checkExceptionPerGreenlet()
finally:
print "Concensus Finished"
print 'End?!'
class Controller(object):
"""Manages a program's main loop in a Greenlet."""
def __init__(self, program_id=DEFAULT_PROGRAM):
"""Creates a controller to control the specified program. The program
doesn't start executing until the start method is called."""
self.messages = Queue()
self.program_id = program_id
self.program = PROGRAMS[program_id]()
self.green = None
self.can_reset = False
def start(self):
"""Starts (or resumes) the execution of the program."""
self.green = Greenlet(self.main_loop)
self.green.start_later(START_DELAY)
self.program.start()
self.can_reset = True
def stop(self):
"""Stops the execution of the program."""
self.program.stop()
if self.green:
self.green.kill()
def reset(self):
"""Stops the program and resets it to its initial state."""
self.stop()
self.program.reset()
self.can_reset = False
def switch_program(self, program_id):
"""Stops execution and switches to a new program."""
self.stop()
self.program_id = program_id
self.program = PROGRAMS[program_id]()
self.can_reset = False
def main_loop(self):
"""Runs the program's loop method continously, collecting any returned
messages into the messages queue."""
while True:
msg = self.program.loop()
if msg:
self.messages.put(msg)
sleep(LOOP_DELAY)
def __call__(self, command):
"""Accepts a command and either performs the desired action or passes
the message on to the program. Returns a status message."""
if command == 'short:sync':
pid = self.program_id
running = bool(self.green)
can_reset = self.can_reset
return "{} {} {}".format(pid, running, can_reset)
if command == 'short:param-help':
return json.dumps(self.program.codes)
if command == 'long:status':
try:
msg = self.messages.get(timeout=STATUS_POLL_TIMEOUT)
except Empty:
return None
return msg
if command.startswith(PROGRAM_PREFIX):
prog = command[len(PROGRAM_PREFIX):]
self.switch_program(prog)
return "switched to {}".format(prog)
if command == 'control:start':
reason = self.program.no_start()
if reason:
return reason
if self.green:
return "already running"
self.start()
return "program resumed"
if command == 'control:stop':
if not self.green:
return "not running"
self.stop()
return "program paused"
if command == 'control:reset':
self.reset()
return "program reset"
return self.program(command)
def client_test_freenet(N, t, options):
'''
Test for the client with random delay channels
command list
i [target]: send a transaction to include for some particular party
h [target]: stop some particular party
m [target]: manually make particular party send some message
help: show the help screen
:param N: the number of parties
:param t: the number of malicious parties
:return None:
'''
maxdelay = 0.01
initiateThresholdSig(open(options.threshold_keys, 'r').read())
initiateECDSAKeys(open(options.ecdsa, 'r').read())
initiateThresholdEnc(open(options.threshold_encs, 'r').read())
initializeGIPC(getKeys()[0])
buffers = map(lambda _: Queue(1), range(N))
global logGreenlet
logGreenlet = Greenlet(logWriter, open('msglog.TorMultiple', 'w'))
logGreenlet.parent_args = (N, t)
logGreenlet.name = 'client_test_freenet.logWriter'
logGreenlet.start()
# Instantiate the "broadcast" instruction
def makeBroadcast(i):
def _broadcast(v):
def _deliver(j):
buffers[j].put(encode((j, i, v)))
for j in range(N):
Greenlet(_deliver, j).start()
return _broadcast
def recvWithDecode(buf):
def recv():
s = buf.get()
return decode(s)[1:]
return recv
def makeSend(i): # point to point message delivery
def _send(j, v):
buffers[j].put(encode((j, i, v)))
return _send
while True:
#if True:
initBeforeBinaryConsensus()
ts = []
controlChannels = [Queue() for _ in range(N)]
transactionSet = set([encodeTransaction(randomTransaction()) for trC in range(int(options.tx))]) # we are using the same one
for i in range(N):
bc = makeBroadcast(i)
recv = recvWithDecode(buffers[i])
th = Greenlet(honestParty, i, N, t, controlChannels[i], bc, recv, makeSend(i), options.B)
controlChannels[i].put(('IncludeTransaction', transactionSet))
th.start_later(random.random() * maxdelay)
ts.append(th)
try:
gevent.joinall(ts)
except ACSException:
gevent.killall(ts)
except finishTransactionLeap: ### Manually jump to this level
print 'msgCounter', msgCounter
print 'msgTypeCounter', msgTypeCounter
# message id 0 (duplicated) for signatureCost
logChannel.put(StopIteration)
mylog("=====", verboseLevel=-1)
for item in logChannel:
mylog(item, verboseLevel=-1)
mylog("=====", verboseLevel=-1)
continue
except gevent.hub.LoopExit: # Manual fix for early stop
while True:
gevent.sleep(1)
checkExceptionPerGreenlet()
finally:
print "Concensus Finished"
def _response_handler (self, env, start_response):
"""handle HTTP request/response"""
uri_path = env['PATH_INFO']
body = Queue()
## NB: these handler cases can be collapsed into a common pattern
## except for config/stop -- later
##########################################
# shard lifecycle endpoints
if uri_path == '/shard/config':
# configure the service to run a shard
payload = loads(env['wsgi.input'].read())
gl = Greenlet(self.shard_config, payload, body, start_response)
gl.start()
elif uri_path == '/shard/persist':
# checkpoint the service state to durable storage
payload = loads(env['wsgi.input'].read())
print "POST", payload
## TODO
start_response('200 OK', [('Content-Type', 'text/plain')])
body.put("Bokay\r\n")
body.put(StopIteration)
elif uri_path == '/shard/recover':
# restart the service, recovering from the most recent checkpoint
payload = loads(env['wsgi.input'].read())
print "POST", payload
## TODO
start_response('200 OK', [('Content-Type', 'text/plain')])
body.put("Bokay\r\n")
body.put(StopIteration)
##########################################
# HashRing endpoints
elif uri_path == '/ring/init':
# initialize the HashRing
payload = loads(env['wsgi.input'].read())
gl = Greenlet(self.ring_init, payload, body, start_response)
gl.start()
elif uri_path == '/ring/add':
# add a node to the HashRing
payload = loads(env['wsgi.input'].read())
print "POST", payload
## TODO
start_response('200 OK', [('Content-Type', 'text/plain')])
body.put("Bokay\r\n")
body.put(StopIteration)
elif uri_path == '/ring/del':
# delete a node from the HashRing
payload = loads(env['wsgi.input'].read())
print "POST", payload
## TODO
start_response('200 OK', [('Content-Type', 'text/plain')])
body.put("Bokay\r\n")
body.put(StopIteration)
##########################################
# evolution endpoints
elif uri_path == '/pop/init':
# initialize the Population subset on this shard
payload = loads(env['wsgi.input'].read())
gl = Greenlet(self.pop_init, payload, body, start_response)
gl.start()
elif uri_path == '/pop/gen':
# create generation 0 of Individuals in this shard of the Population
payload = loads(env['wsgi.input'].read())
gl = Greenlet(self.pop_gen, payload, body, start_response)
gl.start()
elif uri_path == '/pop/wait':
# wait until all shards have finished sending reify requests
payload = loads(env['wsgi.input'].read())
gl = Greenlet(self.pop_wait, payload, body, start_response)
gl.start()
elif uri_path == '/pop/join':
# join on the reify queue, to wait until it empties
payload = loads(env['wsgi.input'].read())
gl = Greenlet(self.pop_join, payload, body, start_response)
gl.start()
elif uri_path == '/pop/hist':
# calculate a partial histogram for the fitness distribution
payload = loads(env['wsgi.input'].read())
gl = Greenlet(self.pop_hist, payload, body, start_response)
gl.start()
elif uri_path == '/pop/next':
# attempt to run another generation
payload = loads(env['wsgi.input'].read())
gl = Greenlet(self.pop_next, payload, body, start_response)
gl.start()
elif uri_path == '/pop/enum':
# enumerate the Individuals in this shard of the Population
payload = loads(env['wsgi.input'].read())
gl = Greenlet(self.pop_enum, payload, body, start_response)
gl.start()
elif uri_path == '/pop/reify':
# test/add a newly generated Individual into the Population (birth)
payload = loads(env['wsgi.input'].read())
gl = Greenlet(self.pop_reify, payload, body, start_response)
gl.start()
elif uri_path == '/pop/evict':
# remove an Individual from the Population (death)
payload = loads(env['wsgi.input'].read())
print "POST", payload
## TODO
start_response('200 OK', [('Content-Type', 'text/plain')])
body.put("Bokay\r\n")
body.put(StopIteration)
##########################################
# utility endpoints
elif uri_path == '/':
# dump info about the service in general
start_response('200 OK', [('Content-Type', 'text/plain')])
body.put(str(env) + "\r\n")
body.put(StopIteration)
elif uri_path == '/stop':
# shutdown the service
payload = loads(env['wsgi.input'].read())
gl = Greenlet(self.stop, payload, body)
gl.start_later(1)
# HTTP response must start here, to avoid failure when server stops
start_response('200 OK', [('Content-Type', 'text/plain')])
body.put("Goodbye\r\n")
body.put(StopIteration)
else:
# ne znayu
start_response('404 Not Found', [('Content-Type', 'text/plain')])
body.put('Not Found\r\n')
body.put(StopIteration)
return body
