def __init__(self, *args: Any, **kwargs: Any) -> None:
self.args = args
self.kwargs = kwargs
self.greenlet = Greenlet(self._run, *self.args, **self.kwargs)
self.greenlet.name = f"{self.__class__.__name__}|{self.greenlet.name}"
def spawn(self):
self.greenlet = Greenlet(self.func)
self.greenlet.link_exception(self)
self.greenlet.start()
return self.greenlet
import asyncore import cx_Oracle c = cx_Oracle.connect() c.rollback() import fabric from django.db import transaction transaction.atomic() from threading import Thread import gevent from gevent import Greenlet import paramiko a = Greenlet() # cx_Oracle.makedsn() # str.replace() # import traceback # traceback.format_exc() from flask import url_for # # import traceback # traceback.format_exc() # # from datetime import datetime, date # print(date.__eq__()) # print(datetime.today().date()) # # import subprocess
def __make(self):
g = Greenlet(self._fn, self, *self._fn_arg, **self._fn_kw)
g.link(self.__callback)
return g
def client_test_freenet(N, t, options):
'''
Test for the client with random delay channels
command list
i [target]: send a transaction to include for some particular party
h [target]: stop some particular party
m [target]: manually make particular party send some message
help: show the help screen
:param N: the number of parties
:param t: the number of malicious parties
:return None:
'''
maxdelay = 0.01
initiateThresholdSig(open(options.threshold_keys, 'r').read())
initiateECDSAKeys(open(options.ecdsa, 'r').read())
initiateThresholdEnc(open(options.threshold_encs, 'r').read())
initializeGIPC(getKeys()[0])
buffers = map(lambda _: Queue(1), range(N))
global logGreenlet
logGreenlet = Greenlet(logWriter, open('msglog.TorMultiple', 'w'))
logGreenlet.parent_args = (N, t)
logGreenlet.name = 'client_test_freenet.logWriter'
logGreenlet.start()
# Instantiate the "broadcast" instruction
def makeBroadcast(i):
def _broadcast(v):
def _deliver(j):
buffers[j].put(encode((j, i, v)))
for j in range(N):
Greenlet(_deliver, j).start()
return _broadcast
def recvWithDecode(buf):
def recv():
s = buf.get()
return decode(s)[1:]
return recv
def makeSend(i): # point to point message delivery
def _send(j, v):
buffers[j].put(encode((j, i, v)))
return _send
while True:
#if True:
initBeforeBinaryConsensus()
ts = []
controlChannels = [Queue() for _ in range(N)]
transactionSet = set([
encodeTransaction(randomTransaction())
for trC in range(int(options.tx))
]) # we are using the same one
for i in range(N):
bc = makeBroadcast(i)
recv = recvWithDecode(buffers[i])
th = Greenlet(honestParty, i, N, t, controlChannels[i], bc, recv,
makeSend(i), options.B)
controlChannels[i].put(('IncludeTransaction', transactionSet))
th.start_later(random.random() * maxdelay)
ts.append(th)
try:
gevent.joinall(ts)
except ACSException:
gevent.killall(ts)
except finishTransactionLeap: ### Manually jump to this level
print 'msgCounter', msgCounter
print 'msgTypeCounter', msgTypeCounter
# message id 0 (duplicated) for signatureCost
logChannel.put(StopIteration)
mylog("=====", verboseLevel=-1)
for item in logChannel:
mylog(item, verboseLevel=-1)
mylog("=====", verboseLevel=-1)
continue
except gevent.hub.LoopExit: # Manual fix for early stop
while True:
gevent.sleep(1)
checkExceptionPerGreenlet()
finally:
print "Concensus Finished"
def _test_pool(self, use_request):
"""
Test that the connection pool prevents both threads and greenlets from
using a socket at the same time.
Sequence:
gr0: start a slow find()
gr1: start a fast find()
gr1: get results
gr0: get results
"""
cx = get_connection(use_greenlets=self.use_greenlets,
auto_start_request=False)
db = cx.pymongo_test
db.test.remove(safe=True)
db.test.insert({'_id': 1}, safe=True)
history = []
def find_fast():
if use_request:
cx.start_request()
history.append('find_fast start')
# With greenlets and the old connection._Pool, this would throw
# AssertionError: "This event is already used by another
# greenlet"
self.assertEqual({'_id': 1}, db.test.find_one())
history.append('find_fast done')
if use_request:
cx.end_request()
def find_slow():
if use_request:
cx.start_request()
history.append('find_slow start')
# Javascript function that pauses N seconds per document
fn = delay(10)
if (is_mongos(db.connection)
or not version.at_least(db.connection, (1, 7, 2))):
# mongos doesn't support eval so we have to use $where
# which is less reliable in this context.
self.assertEqual(1, db.test.find({"$where": fn}).count())
else:
# 'nolock' allows find_fast to start and finish while we're
# waiting for this to complete.
self.assertEqual({
'ok': 1.0,
'retval': True
}, db.command('eval', fn, nolock=True))
history.append('find_slow done')
if use_request:
cx.end_request()
if self.use_greenlets:
gr0, gr1 = Greenlet(find_slow), Greenlet(find_fast)
gr0.start()
gr1.start_later(.1)
else:
gr0 = threading.Thread(target=find_slow)
gr0.setDaemon(True)
gr1 = threading.Thread(target=find_fast)
gr1.setDaemon(True)
gr0.start()
time.sleep(.1)
gr1.start()
gr0.join()
gr1.join()
self.assertEqual([
'find_slow start',
'find_fast start',
'find_fast done',
'find_slow done',
], history)
def run_badger_node(myID, N, f, sPK, sSK, ePK, eSK, sendPath, receivePath):
'''
Test for the client with random delay channels
:param i: the current node index
:param N: the number of parties
:param t: the number of malicious parties toleranted
:return None:
'''
assert type(sPK) is boldyreva.TBLSPublicKey
assert type(sSK) is boldyreva.TBLSPrivateKey
assert type(ePK) is tpke.TPKEPublicKey
assert type(eSK) is tpke.TPKEPrivateKey
# Create the listening channel
recv_queue = listen_to_channel(BASE_PORT + myID)
recv = recv_queue.get
print 'server started'
# Create the sending channels
send_queues = []
for i in range(N):
port = BASE_PORT + i
send_queues.append(connect_to_channel('127.0.0.1', port, myID))
def send(j, obj):
send_queues[j].put(obj)
# Start the honeybadger instance
tx_submit = Queue()
def send_to_hyperledger(transactions):
global sendConnection
for tx in transactions:
if os.path.exists(sendPath):
if sendConnection is None:
print "Opening sending socket at path " + sendPath
sendConnection = socket.socket(socket.AF_UNIX,
socket.SOCK_STREAM)
sendConnection.connect(sendPath)
print "sending length " + str(len(tx))
sendConnection.send(struct.pack('!Q', len(tx)))
print "sending tx " + tx
sendConnection.send(tx)
hbbft = HoneyBadgerBFT("sid",
myID,
8,
N,
f,
sPK,
sSK,
ePK,
eSK,
send,
recv,
tx_submit.get,
send_to_hyperledger,
encode=repr,
decode=ast.literal_eval)
th = Greenlet(hbbft.run)
th.parent_args = (N, f)
th.name = __file__ + '.honestParty(%d)' % i
th.start()
if os.path.exists(receivePath):
os.remove(receivePath)
print "Opening listening socket at path " + receivePath
server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
server.bind(receivePath)
# Listen for incoming connections
server.listen(1)
while True:
# Wait for a connection
connection, client_address = server.accept()
try:
while True:
message = connection.recv(8)
if message:
print "Message " + message
length, = struct.unpack('!Q', message)
print length
message = connection.recv(length)
print message
tx_submit.put([message])
else:
print >> sys.stderr, 'no more data from', client_address
break
finally:
# Clean up the connection
connection.close()
os.remove(receivePath)
th.join()
def create_process(self, runnable, name):
def run_wrapper():
gevent.sleep()
runnable()
gevent.sleep()
return Greenlet(run=run_wrapper)
def main():
offset = 69
t = 4
n = 13
seed = None
symmetric = False
if not symmetric:
#group = PairingGroup('BN256')
group = PairingGroup('MNT159')
alpha = group.random(ZR, seed=seed)
alpha2 = group.random(ZR, seed=seed)
pkg = group.random(G1, seed=seed)
pkghat = group.random(G2, seed=seed)
pkh = group.random(G1, seed=seed)
pk2g = group.random(G1, seed=seed)
pk2ghat = group.random(G2, seed=seed)
pk2h = group.random(G1, seed=seed)
pk = []
pk2 = []
for i in range(t + 1):
pk.append(pkg**(alpha**i))
for i in range(2):
pk.append(pkghat**(alpha**i))
for i in range(t + 1):
pk.append(pkh**(alpha**i))
for i in range(n + 1):
pk2.append(pk2g**(alpha2**i))
for i in range(2):
pk2.append(pk2ghat**(alpha2**i))
for i in range(n + 1):
pk2.append(pk2h**(alpha2**i))
else:
#group = PairingGroup('SS1536')
group = PairingGroup('SS512')
alpha = group.random(ZR, seed=seed)
alpha2 = group.random(ZR, seed=seed)
pkg = group.random(G1, seed=seed)
pkg2 = group.random(G1, seed=seed)
pkh = group.random(G1, seed=seed)
pkh2 = group.random(G1, seed=seed)
pk = []
pk2 = []
for i in range(t + 1):
pk.append(pkg**(alpha**i))
for i in range(t + 1):
pk.append(pkh**(alpha**i))
for i in range(n + 1):
pk2.append(pkg2**(alpha2**i))
for i in range(n + 1):
pk2.append(pkh2**(alpha2**i))
#Preprocessing to speed up exponentiation
for item in pk:
item.initPP()
for item in pk2:
item.initPP()
participantids = []
for i in range(n):
participantids.append(i + 1 + offset)
dealerid = 4242
sends, recvs = simple_router(participantids + [dealerid], seed=seed)
threads = []
#Initialize Players
thread = Greenlet(VssDealer,
k=n,
t=t,
secret=[42, 69, 420, 11111, 1717],
pk=pk,
pk2=pk2,
participantids=participantids,
group=group,
symflag=symmetric,
send_function=sends[dealerid])
thread.start()
threads.append(thread)
for i in range(n):
thread = Greenlet(VssRecipient,
k=n,
t=t,
nodeid=i + 1 + offset,
pk=pk,
pk2=pk2,
participantids=participantids,
group=group,
symflag=symmetric,
send_function=sends[i + 1 + offset],
recv_function=recvs[i + 1 + offset])
thread.start()
threads.append(thread)
gevent.joinall(threads)
def _broadcast(v):
def _deliver(j):
buffers[j].put((i, v))
for j in range(N):
Greenlet(_deliver, j).start_later(random.random() * maxdelay)
def main():
"""module main"""
module = FabricAnsibleModule(
argument_spec=dict(
fabric_uuid=dict(required=True),
job_ctx=dict(type='dict', required=True),
credentials=dict(required=True, type='list'),
hosts=dict(type='list'),
subnets=dict(type='list'),
version=dict(required=True, choices=['v2', 'v2c', 'v3']),
community=dict(required=True),
device_family_info=dict(required=True, type='list'),
vendor_mapping=dict(required=True, type='list'),
pool_size=dict(default=500, type='int')
),
supports_check_mode=True,
required_one_of=[['hosts', 'subnets']]
)
concurrent = module.params['pool_size']
fabric_uuid = module.params['fabric_uuid']
all_hosts = []
job_ctx = module.params['job_ctx']
object_log = ObjectLogUtil(job_ctx)
if module.params['subnets']:
for subnet in module.params['subnets']:
try:
ip_net = IPNetwork(subnet)
all_hosts.extend(list(ip_net))
except Exception as ex:
_result['failed'] = True
_result['msg'] = "ERROR: Invalid subnet \"%s\" (%s)" % \
(subnet, str(ex))
_send_sandesh_log(object_log, _result.get('msg'),
module.logger, True)
module.exit_json(**_result)
if module.params['hosts']:
for host in module.params['hosts']:
try:
ipaddr = socket.gethostbyname(host)
all_hosts.append(ipaddr)
except Exception as ex:
_result['failed'] = True
_result['msg'] = "ERROR: Invalid ip address \"%s\" (%s)" % \
(host, str(ex))
_send_sandesh_log(object_log, _result.get('msg'),
module.logger, True)
module.exit_json(**_result)
# Verify that we receive a community when using snmp v2
if module.params['version'] == "v2" or module.params['version'] == "v2c":
if module.params['community'] is None:
_result['failed'] = True
_result['msg'] = "ERROR: Community not set when using \
snmp version 2"
_send_sandesh_log(object_log, _result.get('msg'), module.logger,
True)
module.exit_json(**_result)
if module.params['version'] == "v3":
_result['failed'] = True
_result['msg'] = "ERROR: Donot support snmp version 3"
_send_sandesh_log(object_log, _result.get('msg'), module.logger,
True)
module.exit_json(**_result)
_result['msg'] = "Prefix(es) to be discovered: " + \
','.join(module.params['subnets'])
_send_sandesh_log(object_log, _result.get('msg'), module.logger, False)
if len(all_hosts) < concurrent:
concurrent = len(all_hosts)
threadpool = pool.Pool(concurrent)
try:
vncapi = VncApi(auth_type=VncApi._KEYSTONE_AUTHN_STRATEGY,
auth_token=job_ctx.get('auth_token'))
for host in all_hosts:
threadpool.start(
Greenlet(
_device_info_processing,
str(host),
vncapi,
module,
object_log,
fabric_uuid))
threadpool.join()
except Exception as ex:
_result['failed'] = True
_result['msg'] = "Failed to connect to API server due to error: %s"\
% str(ex)
module.exit_json(**_result)
_result['msg'] = "Device discovery complete"
_send_sandesh_log(object_log, _result.get('msg'), module.logger, True)
module.exit_json(**_result)
def client_test_freenet(N, t, options):
'''
Test for the client with random delay channels
command list
i [target]: send a transaction to include for some particular party
h [target]: stop some particular party
m [target]: manually make particular party send some message
help: show the help screen
:param N: the number of parties
:param t: the number of malicious parties
:return None:
'''
# Initialize keys required for operation
initiateThresholdSig(open(options.threshold_keys, 'r').read())
initiateECDSAKeys(open(options.ecdsa, 'r').read())
initiateThresholdEnc(open(options.threshold_encs, 'r').read())
initializeGIPC(PK=getKeys()[0])
global logGreenlet
logGreenlet = Greenlet(logWriter, open('msglog.TorMultiple', 'w'))
logGreenlet.parent_args = (N, t)
logGreenlet.name = 'client_test_freenet.logWriter'
logGreenlet.start()
# query amazon meta-data
localIP = check_output(
['curl', 'http://169.254.169.254/latest/meta-data/public-ipv4'])
myID = IP_LIST.index(localIP)
N = len(IP_LIST)
initiateRND(options.tx)
def makeBroadcast(i):
chans = []
# First establish N connections (including a self connection)
for j in range(N):
host, port = IP_MAPPINGS[j] # TOR_MAPPINGS[j]
chans.append(connect_to_channel(host, port, i))
def _broadcast(v):
for j in range(N):
chans[j].put((j, i, v)) # from i to j
def _send(j, v):
chans[j].put((j, i, v))
return _broadcast, _send
iterList = [myID]
servers = []
for i in iterList:
_, port = IP_MAPPINGS[i]
servers.append(listen_to_channel(port))
print 'servers started'
gevent.sleep(WAITING_SETUP_TIME_IN_SEC) # wait for set-up to be ready
print 'sleep over'
if True: # We only test for once
initBeforeBinaryConsensus()
ts = []
controlChannels = [Queue() for _ in range(N)]
bcList = dict()
sdList = dict()
tList = []
def _makeBroadcast(x):
bc, sd = makeBroadcast(x)
bcList[x] = bc
sdList[x] = sd
for i in iterList:
tmp_t = Greenlet(_makeBroadcast, i)
tmp_t.parent_args = (N, t)
tmp_t.name = 'client_test_freenet._makeBroadcast(%d)' % i
tmp_t.start()
tList.append(tmp_t)
gevent.joinall(tList)
rnd = Random()
rnd.seed(123123)
# This makes sure that all the EC2 instances have the same transaction pool
transactionSet = set([
encodeTransaction(randomTransaction(rnd), randomGenerator=rnd)
for trC in range(int(options.tx))
]) # we are using the same one
def toBeScheduled():
for i in iterList:
bc = bcList[i] # makeBroadcast(i)
sd = sdList[i]
recv = servers[0].get
th = Greenlet(honestParty, i, N, t, controlChannels[i], bc,
recv, sd, options.B)
th.parent_args = (N, t)
th.name = 'client_test_freenet.honestParty(%d)' % i
controlChannels[i].put(('IncludeTransaction', transactionSet))
th.start()
mylog('Summoned party %i at time %f' % (i, time.time()),
verboseLevel=-1)
ts.append(th)
try:
gevent.joinall(ts)
except ACSException:
gevent.killall(ts)
except finishTransactionLeap: ### Manually jump to this level
print 'msgCounter', msgCounter
print 'msgTypeCounter', msgTypeCounter
# message id 0 (duplicated) for signatureCost
logChannel.put(StopIteration)
mylog("=====", verboseLevel=-1)
for item in logChannel:
mylog(item, verboseLevel=-1)
mylog("=====", verboseLevel=-1)
except gevent.hub.LoopExit: # Manual fix for early stop
while True:
gevent.sleep(1)
checkExceptionPerGreenlet()
finally:
print "Consensus Finished"
s = sched.scheduler(time.time, time.sleep)
time_now = time.time()
delay = options.delaytime - time_now
s.enter(delay, 1, toBeScheduled, ())
print myID, "waits for", time_now + delay, 'now is', time_now
s.run()
s.close()
mylog('retrying (%s, %d) caused by %s...' %
(hostname, port, str(e)),
verboseLevel=-1)
q = Queue()
def _handle():
while True:
obj = q.get()
content = encode(obj)
try:
s.sendall(struct.pack('<I', len(content)) + content)
except SocketError:
print '!! [to %d] sending %d bytes' % (party, len(content))
gtemp = Greenlet(_handle)
gtemp.parent_args = (hostname, port, party)
gtemp.name = 'connect_to_channel._handle'
gtemp.start()
return q
BASE_PORT = 49500
def getAddrFromEC2Summary(s):
return [
x.split('ec2.')[-1]
for x in s.replace('.compute.amazonaws.com', '').replace(
'.us-west-1',
'' # Later we need to add more such lines
def acs(pid, N, t, Q, broadcast, receive):
assert (isinstance(Q, list))
assert (len(Q) == N)
decideChannel = [Queue(1) for _ in range(N)]
receivedChannelsFlags = []
def callbackFactory(i):
def _callback(val): # Get notified for i
# Greenlet(callBackWrap(binary_consensus, callbackFactory(i)), pid,
# N, t, 1, make_bc(i), reliableBroadcastReceiveQueue[i].get).start()
if not i in receivedChannelsFlags:
receivedChannelsFlags.append(i)
# mylog('B[%d]binary consensus_%d_starts with 1 at %f' % (pid, i, time.time()), verboseLevel=-1)
greenletPacker(
Greenlet(binary_consensus, i, pid, N, t, 1,
decideChannel[i], make_bc(i),
reliableBroadcastReceiveQueue[i].get),
'acs.callbackFactory.binary_consensus',
(pid, N, t, Q, broadcast, receive)).start()
return _callback
for i, q in enumerate(Q):
assert (isinstance(q, MonitoredInt))
q.registerSetCallBack(callbackFactory(i))
def make_bc(i):
def _bc(m):
broadcast((i, m))
return _bc
reliableBroadcastReceiveQueue = [Queue() for x in range(N)]
def _listener():
while True:
sender, (instance, m) = receive()
reliableBroadcastReceiveQueue[instance].put((sender, m))
greenletPacker(Greenlet(_listener), 'acs._listener',
(pid, N, t, Q, broadcast, receive)).start()
BA = [0] * N
locker = Queue(1)
locker2 = Queue(1)
callbackCounter = [0]
def listenerFactory(i, channel):
def _listener():
BA[i] = channel.get()
if callbackCounter[0] >= 2 * t and (not locker2.full()):
locker2.put("Key") # Now we've got 2t+1 1's
callbackCounter[0] += 1
if callbackCounter[0] == N and (
not locker.full()): # if we have all of them responded
locker.put("Key")
return _listener
for i in range(N):
greenletPacker(Greenlet(listenerFactory(i, decideChannel[i])),
'acs.listenerFactory(i, decideChannel[i])',
(pid, N, t, Q, broadcast, receive)).start()
locker2.get()
# Now we feed 0 to all the other binary consensus protocols
for i in range(N):
if not i in receivedChannelsFlags:
receivedChannelsFlags.append(i)
greenletPacker(
Greenlet(binary_consensus, i, pid, N, t, 0, decideChannel[i],
make_bc(i), reliableBroadcastReceiveQueue[i].get),
'acs.binary_consensus',
(pid, N, t, Q, broadcast, receive)).start()
locker.get() # Now we can check'''
BA = checkBA(BA, N, t)
return BA
def run(self):
'''
Runs the message processor, no lightweight threads are started until this point
'''
Greenlet(self._run()).start()
def honestParty(pid, N, t, controlChannel, broadcast, receive, send, socket, B = -1):
# RequestChannel is called by the client and it is the client's duty to broadcast the tx it wants to include
if B < 0:
B = int(math.ceil(N * math.log(N)))
transactionCache = []
finishedTx = set()
sentTx = set()
proposals = []
receivedProposals = False
commonSet = []
locks = defaultdict(lambda : Queue(1))
doneCombination = defaultdict(lambda : False)
ENC_THRESHOLD = N - 2 * t
global finishcount
encPK, encSKs = getEncKeys()
encCounter = defaultdict(lambda : {})
includeTransactionChannel = Queue()
def probe(i):
if len(encCounter[i]) >= ENC_THRESHOLD and receivedProposals and not locks[i].full() and not doneCombination[i]: # by == this part only executes once.
oriM = encPK.combine_shares(deserializeEnc(proposals[i][:ENC_SERIALIZED_LENGTH]),
dict(itertools.islice(encCounter[i].iteritems(), ENC_THRESHOLD))
)
doneCombination[i] = True
locks[i].put(oriM)
def listener():
while True:
sender, msgBundle = receive()
if msgBundle[0] == 'O':
encCounter[msgBundle[1]][sender] = msgBundle[2]
probe(msgBundle[1])
else:
includeTransactionChannel.put((sender, msgBundle)) # redirect to includeTransaction
Greenlet(listener).start()
while True:
op, msg = controlChannel.get()
if op == "IncludeTransaction":
if isinstance(msg, Transaction):
# transactionCache.add(msg)
transactionCache.append(msg)
elif isinstance(msg, set):
for tx in msg:
transactionCache.append(tx)
elif isinstance(msg, list):
transactionCache.extend(msg)
elif op == "Halt":
break
elif op == "Msg":
broadcast(eval(msg)) # now the msg is something we mannually send
mylog("timestampB (%d, %lf)" % (pid, time.time()), verboseLevel=-2)
if len(transactionCache) < B: # Let's wait for many transactions. : )
time.sleep(0.5)
print "Not enough transactions", len(transactionCache)
continue
oldest_B = transactionCache[:B]
selected_B = random.sample(oldest_B, min(B/N, len(oldest_B)))
print "[%d] proposing %d transactions" % (pid, min(B/N, len(oldest_B)))
aesKey = random._urandom(32) #
encrypted_B = encrypt(aesKey, ''.join(selected_B))
encryptedAESKey = encPK.encrypt(aesKey)
proposal = serializeEnc(encryptedAESKey) + encrypted_B
mylog("timestampIB (%d, %lf)" % (pid, time.time()), verboseLevel=-2)
commonSet, proposals = includeTransaction(pid, N, t, proposal, broadcast, includeTransactionChannel.get, send)
mylog("timestampIE (%d, %lf)" % (pid, time.time()), verboseLevel=-2)
receivedProposals = True
for i in range(N):
probe(i)
for i, c in enumerate(commonSet): # stx is the same for every party
if c:
share = encSKs[pid].decrypt_share(deserializeEnc(proposals[i][:ENC_SERIALIZED_LENGTH]))
broadcast(('O', i, share))
mylog("timestampIE2 (%d, %lf)" % (pid, time.time()), verboseLevel=-2)
recoveredSyncedTxList = []
def prepareTx(i):
rec = locks[i].get()
encodedTxSet = decrypt(rec, proposals[i][ENC_SERIALIZED_LENGTH:])
assert len(encodedTxSet) % TR_SIZE == 0
recoveredSyncedTx = [encodedTxSet[i:i+TR_SIZE] for i in range(0, len(encodedTxSet), TR_SIZE)]
recoveredSyncedTxList.append(recoveredSyncedTx)
thList = []
for i, c in enumerate(commonSet): # stx is the same for every party
if c:
s = Greenlet(prepareTx, i)
thList.append(s)
s.start()
gevent.joinall(thList)
mylog("timestampE (%d, %lf)" % (pid, time.time()), verboseLevel=-2)
for rtx in recoveredSyncedTxList:
finishedTx.update(set(rtx))
for transaction in rtx:
if transaction not in sentTx:
sentTx.add(transaction)
# send_envelope(socket, transaction.envelope)
mylog("Sent back envelope %d to BFTProxy" % transaction.id, verboseLevel=-2)
mylog("[%d] %d distinct tx synced and %d tx left in the pool." % (pid, len(finishedTx), len(transactionCache) - len(finishedTx)), verboseLevel=-2)
lock.get()
finishcount += 1
lock.put(1)
# if finishcount >= N - t: # convenient for local experiments
# sys.exit()
mylog("[%d] Now halting..." % (pid))
def test_socket_reclamation(self):
if sys.platform.startswith('java'):
raise SkipTest("Jython can't do socket reclamation")
# Check that if a thread starts a request and dies without ending
# the request, that the socket is reclaimed into the pool.
cx_pool = self.get_pool(
pair=(host, port),
max_size=10,
net_timeout=1000,
conn_timeout=1000,
use_ssl=False,
)
self.assertEqual(0, len(cx_pool.sockets))
lock = None
the_sock = [None]
def leak_request():
self.assertEqual(NO_REQUEST, cx_pool._get_request_state())
cx_pool.start_request()
self.assertEqual(NO_SOCKET_YET, cx_pool._get_request_state())
sock_info = cx_pool.get_socket()
self.assertEqual(sock_info, cx_pool._get_request_state())
the_sock[0] = id(sock_info.sock)
cx_pool.maybe_return_socket(sock_info)
if not self.use_greenlets:
lock.release()
if self.use_greenlets:
g = Greenlet(leak_request)
g.start()
g.join(1)
self.assertTrue(g.ready(), "Greenlet is hung")
else:
lock = thread.allocate_lock()
lock.acquire()
# Start a thread WITHOUT a threading.Thread - important to test that
# Pool can deal with primitive threads.
thread.start_new_thread(leak_request, ())
# Join thread
acquired = lock.acquire()
self.assertTrue(acquired, "Thread is hung")
# Make sure thread is really gone
time.sleep(1)
if 'PyPy' in sys.version:
gc.collect()
# Access the thread local from the main thread to trigger the
# ThreadVigil's delete callback, returning the request socket to
# the pool.
# In Python 2.6 and lesser, a dead thread's locals are deleted
# and those locals' weakref callbacks are fired only when another
# thread accesses the locals and finds the thread state is stale.
# This is more or less a bug in Python <= 2.6. Accessing the thread
# local from the main thread is a necessary part of this test, and
# realistic: in a multithreaded web server a new thread will access
# Pool._local soon after an old thread has died.
getattr(cx_pool._local, 'whatever', None)
# Pool reclaimed the socket
self.assertEqual(1, len(cx_pool.sockets))
self.assertEqual(the_sock[0], id(one(cx_pool.sockets).sock))
def multiSigBr(pid, N, t, msg, broadcast, receive, outputs, send):
# Since all the parties we have are symmetric, so I implement this function for N instances of A-cast as a whole
# Here msg is a set of transactions
assert(isinstance(outputs, list))
for i in outputs:
assert(isinstance(i, Queue))
keys = getECDSAKeys()
Threshold = N - 2 * t
Threshold2 = N - t
zfecEncoder = zfec.Encoder(Threshold, N)
zfecDecoder = zfec.Decoder(Threshold, N)
def merkleTree(strList, someHash = coolSHA256Hash):
# someHash is a mapping from a int to a int
treeLength = 2 ** ceil(math.log(len(strList)) / math.log(2))
mt = [0] * (treeLength * 2) # find a place to put our leaves
for i in range(len(strList)):
mt[i + treeLength] = someHash(strList[i]) # TODO: need to change strList[i] from a string to an integer here.
for i in range(treeLength - 1, 0, -1): # 1, 2, 3, ..., treeLength - 1
# mt[i] = someHash(''.join([chr(ord(a) ^ ord(b)) for a, b in zip(mt[i*2], mt[i*2+1])])) # XOR is commutative
mt[i] = someHash(mt[i*2] + mt[i*2+1]) # concat is not commutative
return mt
def getMerkleBranch(index, mt):
res = []
t = index + (len(mt) >> 1)
while t > 1:
res.append(mt[t ^ 1]) # we are picking up the sibling
t /= 2
return res
def merkleVerify(val, rootHash, branch, someHash, index):
# index has information on whether we are facing a left sibling or a right sibling
tmp = someHash(val)
tindex = index
for br in branch:
tmp = someHash((tindex & 1) and br + tmp or tmp + br)
tindex >>= 1
if tmp != rootHash:
print "Verification failed with", someHash(val), rootHash, branch, tmp == rootHash
return tmp == rootHash
def Listener():
opinions = [defaultdict(lambda: 0) for _ in range(N)]
rootHashes = dict()
readyCounter = [defaultdict(lambda: 0) for _ in range(N)]
signed = [False]*N
readySent = [False] * N
reconstDone = [False] * N
while True: # main loop
sender, msgBundle = receive()
if msgBundle[0] == 'i' and not signed[sender]:
if keys[sender].verify(sha1hash(''.join([msgBundle[1][0], msgBundle[1][1], ''.join(msgBundle[1][2])])), msgBundle[2]):
assert isinstance(msgBundle[1], tuple)
if not merkleVerify(msgBundle[1][0], msgBundle[1][1], msgBundle[1][2], coolSHA256Hash, pid):
continue
if sender in rootHashes:
if rootHashes[sender]!= msgBundle[1][1]:
print "Cheating caught, exiting"
sys.exit(0)
else:
rootHashes[sender] = msgBundle[1][1]
newBundle = (sender, msgBundle[1][0], msgBundle[1][1], msgBundle[1][2]) # assert each frag has a length of step
broadcast(('e', newBundle, keys[pid].sign(
sha1hash(''.join([str(newBundle[0]), newBundle[1], newBundle[2], ''.join(newBundle[3])]))
)))
signed[sender] = True
else:
raise ECDSASignatureError()
elif msgBundle[0] == 'e':
if keys[sender].verify(sha1hash(''.join([str(msgBundle[1][0]), msgBundle[1][1], msgBundle[1][2], ''.join(msgBundle[1][3])])), msgBundle[2]):
originBundle = msgBundle[1]
if not merkleVerify(originBundle[1], originBundle[2], originBundle[3], coolSHA256Hash, sender):
continue
if originBundle[0] in rootHashes:
if rootHashes[originBundle[0]]!= originBundle[2]:
print "Cheating caught, exiting"
sys.exit(0)
else:
rootHashes[originBundle[0]] = originBundle[2]
opinions[originBundle[0]][sender] = originBundle[1] # We are going to move this part to kekeketktktktk
if len(opinions[originBundle[0]]) >= Threshold2 and not readySent[originBundle[0]]:
readySent[originBundle[0]] = True
broadcast(('r', originBundle[0], originBundle[2])) # We are broadcasting its hash
else:
raise ECDSASignatureError()
elif msgBundle[0] == 'r':
readyCounter[msgBundle[1]][msgBundle[2]] += 1
tmp = readyCounter[msgBundle[1]][msgBundle[2]]
if tmp >= t+1 and not readySent[msgBundle[1]]:
readySent[msgBundle[1]] = True
broadcast(('r', msgBundle[1], msgBundle[2]))
if tmp >= Threshold2 and not outputs[msgBundle[1]].full() and \
not reconstDone[msgBundle[1]] and len(opinions[msgBundle[1]]) >= Threshold:
reconstDone[msgBundle[1]] = True
if msgBundle[1] in rootHashes:
if rootHashes[msgBundle[1]]!= msgBundle[2]:
print "Cheating caught, exiting"
sys.exit(0)
else:
rootHashes[msgBundle[1]] = msgBundle[2]
if opinions[msgBundle[1]].values()[0] == '':
reconstruction = ['']
else:
reconstruction = zfecDecoder.decode(opinions[msgBundle[1]].values()[:Threshold],
opinions[msgBundle[1]].keys()[:Threshold]) # We only take the first [Threshold] fragments
rawbuf = ''.join(reconstruction)
buf = rawbuf[:-ord(rawbuf[-1])]
# Check root hash
step = len(buf) / Threshold + 1 # len(buf) % Threshold == 0 and len(buf) / Threshold or (len(buf) / Threshold + 1)
assert step * Threshold - len(buf) < 256 # assumption
buf_ = buf.ljust(step * Threshold - 1, '\xFF') + chr(step * Threshold - len(buf))
fragList = [buf_[i*step : (i+1)*step] for i in range(Threshold)]
encodedFragList = zfecEncoder.encode(fragList)
mt = merkleTree(encodedFragList, coolSHA256Hash)
assert rootHashes[msgBundle[1]] == mt[1] # full binary tree
if outputs[msgBundle[1]].empty():
outputs[msgBundle[1]].put(buf)
greenletPacker(Greenlet(Listener), 'multiSigBr.Listener', (pid, N, t, msg, broadcast, receive, outputs)).start()
buf = msg # We already assumed the proposals are byte strings
step = len(buf) / Threshold + 1 # len(buf) % Threshold == 0 and len(buf) / Threshold or (len(buf) / Threshold + 1)
assert step * Threshold - len(buf) < 256 # assumption
buf = buf.ljust(step * Threshold - 1, '\xFF') + chr(step * Threshold - len(buf))
fragList = [buf[i*step : (i+1)*step] for i in range(Threshold)]
encodedFragList = zfecEncoder.encode(fragList)
mt = merkleTree(encodedFragList, coolSHA256Hash)
rootHash = mt[1] # full binary tree
for i in range(N):
mb = getMerkleBranch(i, mt) # notice that index starts from 1 and pid starts from 0
newBundle = (encodedFragList[i], rootHash, mb)
send(i, ('i', newBundle, keys[pid].sign(sha1hash(''.join([newBundle[0], newBundle[1], ''.join(newBundle[2])])))))
print 'Connection established (%s, %d)' % (hostname, port)
while True:
obj = q.get()
try:
content = encode((myID, obj))
except TypeError:
print obj
raise
try:
s.sendall(struct.pack('<I', len(content)) + content)
except SocketError:
print '!! [to %d] sending %d bytes' % (myID, len(content))
break
print 'closed channel'
gtemp = Greenlet(_run)
gtemp.parent_args = (hostname, port, myID)
gtemp.name = 'connect_to_channel._handle'
gtemp.start()
return q
def exception(msg):
print "Exception: %s" % msg
os.exit(1)
def encode(m):
return pickle.dumps(m)
def _test_pool(self, use_request):
"""
Test that the connection pool prevents both threads and greenlets from
using a socket at the same time.
Sequence:
gr0: start a slow find()
gr1: start a fast find()
gr1: get results
gr0: get results
"""
cx = get_connection(use_greenlets=self.use_greenlets,
auto_start_request=False)
db = cx.pymongo_test
if not version.at_least(db.connection, (1, 7, 2)):
raise SkipTest("Need at least MongoDB version 1.7.2 to use"
" db.eval(nolock=True)")
db.test.remove(safe=True)
db.test.insert({'_id': 1}, safe=True)
history = []
def find_fast():
if use_request:
cx.start_request()
history.append('find_fast start')
# With greenlets and the old connection._Pool, this would throw
# AssertionError: "This event is already used by another
# greenlet"
self.assertEqual({'_id': 1}, db.test.find_one())
history.append('find_fast done')
if use_request:
cx.end_request()
def find_slow():
if use_request:
cx.start_request()
history.append('find_slow start')
# Javascript function that pauses 5 sec. 'nolock' allows find_fast
# to start and finish while we're waiting for this.
fn = delay(5)
self.assertEqual({
'ok': 1.0,
'retval': True
}, db.command('eval', fn, nolock=True))
history.append('find_slow done')
if use_request:
cx.end_request()
if self.use_greenlets:
gr0, gr1 = Greenlet(find_slow), Greenlet(find_fast)
gr0.start()
gr1.start_later(.1)
else:
gr0 = threading.Thread(target=find_slow)
gr0.setDaemon(True)
gr1 = threading.Thread(target=find_fast)
gr1.setDaemon(True)
gr0.start()
time.sleep(.1)
gr1.start()
gr0.join()
gr1.join()
self.assertEqual([
'find_slow start',
'find_fast start',
'find_fast done',
'find_slow done',
], history)
def wrapper(*args, **kwargs):
thr = Greenlet(f, *args, **kwargs)
thr.start()
def handle_handler(self, handler):
"""Add the Greenlet to the pool with ``handler.run()``, the function to run.
:param handler: The RawRequestHandler provided in the constructor
:type handler: RawRequestHandler"""
self.pool.start(Greenlet(handler.run))
def _broadcast(v):
def _deliver(j):
buffers[j].put(encode((j, i, v)))
for j in range(N):
Greenlet(_deliver, j).start()
def handle_handler(self, handler):
"""Add the Greenlet to the pool with ``self.callback(handler, self)``: the function to run.
:param handler: The RawRequestHandler provided in the constructor
:type handler: RawRequestHandler"""
self.pool.start(Greenlet(self.callback, handler, self))
def _start():
for runner in self._node_runners:
pool.start(Greenlet(runner.start))
pool.join(raise_error=True)
log.info("All nodes started")
def main_loop(config):
"""
Основной цикл приложения.
:param config: конфигурация
:type config: Config
Алгоритм:
* Открываем соединение с tarantool.queue, использую config.QUEUE_* настройки.
* Создаем пул обработчиков.
* Создаем очередь куда обработчики будут помещать выполненные задачи.
* Пока количество обработчиков <= config.WORKER_POOL_SIZE, берем задачу из tarantool.queue
и запускаем greenlet для ее обработки.
* Посылаем уведомления о том, что задачи завершены в tarantool.queue.
* Спим config.SLEEP секунд.
"""
logger.info(
'Connect to queue server on {host}:{port} space #{space}.'.format(
host=config.QUEUE_HOST,
port=config.QUEUE_PORT,
space=config.QUEUE_SPACE))
queue = tarantool_queue.Queue(host=config.QUEUE_HOST,
port=config.QUEUE_PORT,
space=config.QUEUE_SPACE)
logger.info('Use tube [{tube}], take timeout={take_timeout}.'.format(
tube=config.QUEUE_TUBE, take_timeout=config.QUEUE_TAKE_TIMEOUT))
tube = queue.tube(config.QUEUE_TUBE)
logger.info(
'Create worker pool[{size}].'.format(size=config.WORKER_POOL_SIZE))
worker_pool = Pool(config.WORKER_POOL_SIZE)
processed_task_queue = gevent_queue.Queue()
logger.info(
'Run main loop. Worker pool size={count}. Sleep time is {sleep}.'.
format(count=config.WORKER_POOL_SIZE, sleep=config.SLEEP))
while run_application:
free_workers_count = worker_pool.free_count()
logger.debug(
'Pool has {count} free workers.'.format(count=free_workers_count))
for number in xrange(free_workers_count):
logger.debug('Get task from tube for worker#{number}.'.format(
number=number))
task = tube.take(config.QUEUE_TAKE_TIMEOUT)
if task:
logger.info(
'Start worker#{number} for task id={task_id}.'.format(
task_id=task.task_id, number=number))
worker = Greenlet(notification_worker,
task,
processed_task_queue,
timeout=config.HTTP_CONNECTION_TIMEOUT,
verify=False)
worker_pool.add(worker)
worker.start()
done_with_processed_tasks(processed_task_queue)
sleep(config.SLEEP)
else:
logger.info('Stop application loop.')
def start_later(self, seconds):
Greenlet(self.run).start_later(seconds)
def start(self):
"""Starts (or resumes) the execution of the program."""
self.green = Greenlet(self.main_loop)
self.green.start_later(START_DELAY)
self.program.start()
self.can_reset = True
