def __init__(self, *args: Any, **kwargs: Any) -> None: self.args = args self.kwargs = kwargs self.greenlet = Greenlet(self._run, *self.args, **self.kwargs) self.greenlet.name = f"{self.__class__.__name__}|{self.greenlet.name}"
def spawn(self): self.greenlet = Greenlet(self.func) self.greenlet.link_exception(self) self.greenlet.start() return self.greenlet
import asyncore import cx_Oracle c = cx_Oracle.connect() c.rollback() import fabric from django.db import transaction transaction.atomic() from threading import Thread import gevent from gevent import Greenlet import paramiko a = Greenlet() # cx_Oracle.makedsn() # str.replace() # import traceback # traceback.format_exc() from flask import url_for # # import traceback # traceback.format_exc() # # from datetime import datetime, date # print(date.__eq__()) # print(datetime.today().date()) # # import subprocess
def __make(self): g = Greenlet(self._fn, self, *self._fn_arg, **self._fn_kw) g.link(self.__callback) return g
def client_test_freenet(N, t, options): ''' Test for the client with random delay channels command list i [target]: send a transaction to include for some particular party h [target]: stop some particular party m [target]: manually make particular party send some message help: show the help screen :param N: the number of parties :param t: the number of malicious parties :return None: ''' maxdelay = 0.01 initiateThresholdSig(open(options.threshold_keys, 'r').read()) initiateECDSAKeys(open(options.ecdsa, 'r').read()) initiateThresholdEnc(open(options.threshold_encs, 'r').read()) initializeGIPC(getKeys()[0]) buffers = map(lambda _: Queue(1), range(N)) global logGreenlet logGreenlet = Greenlet(logWriter, open('msglog.TorMultiple', 'w')) logGreenlet.parent_args = (N, t) logGreenlet.name = 'client_test_freenet.logWriter' logGreenlet.start() # Instantiate the "broadcast" instruction def makeBroadcast(i): def _broadcast(v): def _deliver(j): buffers[j].put(encode((j, i, v))) for j in range(N): Greenlet(_deliver, j).start() return _broadcast def recvWithDecode(buf): def recv(): s = buf.get() return decode(s)[1:] return recv def makeSend(i): # point to point message delivery def _send(j, v): buffers[j].put(encode((j, i, v))) return _send while True: #if True: initBeforeBinaryConsensus() ts = [] controlChannels = [Queue() for _ in range(N)] transactionSet = set([ encodeTransaction(randomTransaction()) for trC in range(int(options.tx)) ]) # we are using the same one for i in range(N): bc = makeBroadcast(i) recv = recvWithDecode(buffers[i]) th = Greenlet(honestParty, i, N, t, controlChannels[i], bc, recv, makeSend(i), options.B) controlChannels[i].put(('IncludeTransaction', transactionSet)) th.start_later(random.random() * maxdelay) ts.append(th) try: gevent.joinall(ts) except ACSException: gevent.killall(ts) except finishTransactionLeap: ### Manually jump to this level print 'msgCounter', msgCounter print 'msgTypeCounter', msgTypeCounter # message id 0 (duplicated) for signatureCost logChannel.put(StopIteration) mylog("=====", verboseLevel=-1) for item in logChannel: mylog(item, verboseLevel=-1) mylog("=====", verboseLevel=-1) continue except gevent.hub.LoopExit: # Manual fix for early stop while True: gevent.sleep(1) checkExceptionPerGreenlet() finally: print "Concensus Finished"
def _test_pool(self, use_request): """ Test that the connection pool prevents both threads and greenlets from using a socket at the same time. Sequence: gr0: start a slow find() gr1: start a fast find() gr1: get results gr0: get results """ cx = get_connection(use_greenlets=self.use_greenlets, auto_start_request=False) db = cx.pymongo_test db.test.remove(safe=True) db.test.insert({'_id': 1}, safe=True) history = [] def find_fast(): if use_request: cx.start_request() history.append('find_fast start') # With greenlets and the old connection._Pool, this would throw # AssertionError: "This event is already used by another # greenlet" self.assertEqual({'_id': 1}, db.test.find_one()) history.append('find_fast done') if use_request: cx.end_request() def find_slow(): if use_request: cx.start_request() history.append('find_slow start') # Javascript function that pauses N seconds per document fn = delay(10) if (is_mongos(db.connection) or not version.at_least(db.connection, (1, 7, 2))): # mongos doesn't support eval so we have to use $where # which is less reliable in this context. self.assertEqual(1, db.test.find({"$where": fn}).count()) else: # 'nolock' allows find_fast to start and finish while we're # waiting for this to complete. self.assertEqual({ 'ok': 1.0, 'retval': True }, db.command('eval', fn, nolock=True)) history.append('find_slow done') if use_request: cx.end_request() if self.use_greenlets: gr0, gr1 = Greenlet(find_slow), Greenlet(find_fast) gr0.start() gr1.start_later(.1) else: gr0 = threading.Thread(target=find_slow) gr0.setDaemon(True) gr1 = threading.Thread(target=find_fast) gr1.setDaemon(True) gr0.start() time.sleep(.1) gr1.start() gr0.join() gr1.join() self.assertEqual([ 'find_slow start', 'find_fast start', 'find_fast done', 'find_slow done', ], history)
def run_badger_node(myID, N, f, sPK, sSK, ePK, eSK, sendPath, receivePath): ''' Test for the client with random delay channels :param i: the current node index :param N: the number of parties :param t: the number of malicious parties toleranted :return None: ''' assert type(sPK) is boldyreva.TBLSPublicKey assert type(sSK) is boldyreva.TBLSPrivateKey assert type(ePK) is tpke.TPKEPublicKey assert type(eSK) is tpke.TPKEPrivateKey # Create the listening channel recv_queue = listen_to_channel(BASE_PORT + myID) recv = recv_queue.get print 'server started' # Create the sending channels send_queues = [] for i in range(N): port = BASE_PORT + i send_queues.append(connect_to_channel('127.0.0.1', port, myID)) def send(j, obj): send_queues[j].put(obj) # Start the honeybadger instance tx_submit = Queue() def send_to_hyperledger(transactions): global sendConnection for tx in transactions: if os.path.exists(sendPath): if sendConnection is None: print "Opening sending socket at path " + sendPath sendConnection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) sendConnection.connect(sendPath) print "sending length " + str(len(tx)) sendConnection.send(struct.pack('!Q', len(tx))) print "sending tx " + tx sendConnection.send(tx) hbbft = HoneyBadgerBFT("sid", myID, 8, N, f, sPK, sSK, ePK, eSK, send, recv, tx_submit.get, send_to_hyperledger, encode=repr, decode=ast.literal_eval) th = Greenlet(hbbft.run) th.parent_args = (N, f) th.name = __file__ + '.honestParty(%d)' % i th.start() if os.path.exists(receivePath): os.remove(receivePath) print "Opening listening socket at path " + receivePath server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) server.bind(receivePath) # Listen for incoming connections server.listen(1) while True: # Wait for a connection connection, client_address = server.accept() try: while True: message = connection.recv(8) if message: print "Message " + message length, = struct.unpack('!Q', message) print length message = connection.recv(length) print message tx_submit.put([message]) else: print >> sys.stderr, 'no more data from', client_address break finally: # Clean up the connection connection.close() os.remove(receivePath) th.join()
def create_process(self, runnable, name): def run_wrapper(): gevent.sleep() runnable() gevent.sleep() return Greenlet(run=run_wrapper)
def main(): offset = 69 t = 4 n = 13 seed = None symmetric = False if not symmetric: #group = PairingGroup('BN256') group = PairingGroup('MNT159') alpha = group.random(ZR, seed=seed) alpha2 = group.random(ZR, seed=seed) pkg = group.random(G1, seed=seed) pkghat = group.random(G2, seed=seed) pkh = group.random(G1, seed=seed) pk2g = group.random(G1, seed=seed) pk2ghat = group.random(G2, seed=seed) pk2h = group.random(G1, seed=seed) pk = [] pk2 = [] for i in range(t + 1): pk.append(pkg**(alpha**i)) for i in range(2): pk.append(pkghat**(alpha**i)) for i in range(t + 1): pk.append(pkh**(alpha**i)) for i in range(n + 1): pk2.append(pk2g**(alpha2**i)) for i in range(2): pk2.append(pk2ghat**(alpha2**i)) for i in range(n + 1): pk2.append(pk2h**(alpha2**i)) else: #group = PairingGroup('SS1536') group = PairingGroup('SS512') alpha = group.random(ZR, seed=seed) alpha2 = group.random(ZR, seed=seed) pkg = group.random(G1, seed=seed) pkg2 = group.random(G1, seed=seed) pkh = group.random(G1, seed=seed) pkh2 = group.random(G1, seed=seed) pk = [] pk2 = [] for i in range(t + 1): pk.append(pkg**(alpha**i)) for i in range(t + 1): pk.append(pkh**(alpha**i)) for i in range(n + 1): pk2.append(pkg2**(alpha2**i)) for i in range(n + 1): pk2.append(pkh2**(alpha2**i)) #Preprocessing to speed up exponentiation for item in pk: item.initPP() for item in pk2: item.initPP() participantids = [] for i in range(n): participantids.append(i + 1 + offset) dealerid = 4242 sends, recvs = simple_router(participantids + [dealerid], seed=seed) threads = [] #Initialize Players thread = Greenlet(VssDealer, k=n, t=t, secret=[42, 69, 420, 11111, 1717], pk=pk, pk2=pk2, participantids=participantids, group=group, symflag=symmetric, send_function=sends[dealerid]) thread.start() threads.append(thread) for i in range(n): thread = Greenlet(VssRecipient, k=n, t=t, nodeid=i + 1 + offset, pk=pk, pk2=pk2, participantids=participantids, group=group, symflag=symmetric, send_function=sends[i + 1 + offset], recv_function=recvs[i + 1 + offset]) thread.start() threads.append(thread) gevent.joinall(threads)
def _broadcast(v): def _deliver(j): buffers[j].put((i, v)) for j in range(N): Greenlet(_deliver, j).start_later(random.random() * maxdelay)
def main(): """module main""" module = FabricAnsibleModule( argument_spec=dict( fabric_uuid=dict(required=True), job_ctx=dict(type='dict', required=True), credentials=dict(required=True, type='list'), hosts=dict(type='list'), subnets=dict(type='list'), version=dict(required=True, choices=['v2', 'v2c', 'v3']), community=dict(required=True), device_family_info=dict(required=True, type='list'), vendor_mapping=dict(required=True, type='list'), pool_size=dict(default=500, type='int') ), supports_check_mode=True, required_one_of=[['hosts', 'subnets']] ) concurrent = module.params['pool_size'] fabric_uuid = module.params['fabric_uuid'] all_hosts = [] job_ctx = module.params['job_ctx'] object_log = ObjectLogUtil(job_ctx) if module.params['subnets']: for subnet in module.params['subnets']: try: ip_net = IPNetwork(subnet) all_hosts.extend(list(ip_net)) except Exception as ex: _result['failed'] = True _result['msg'] = "ERROR: Invalid subnet \"%s\" (%s)" % \ (subnet, str(ex)) _send_sandesh_log(object_log, _result.get('msg'), module.logger, True) module.exit_json(**_result) if module.params['hosts']: for host in module.params['hosts']: try: ipaddr = socket.gethostbyname(host) all_hosts.append(ipaddr) except Exception as ex: _result['failed'] = True _result['msg'] = "ERROR: Invalid ip address \"%s\" (%s)" % \ (host, str(ex)) _send_sandesh_log(object_log, _result.get('msg'), module.logger, True) module.exit_json(**_result) # Verify that we receive a community when using snmp v2 if module.params['version'] == "v2" or module.params['version'] == "v2c": if module.params['community'] is None: _result['failed'] = True _result['msg'] = "ERROR: Community not set when using \ snmp version 2" _send_sandesh_log(object_log, _result.get('msg'), module.logger, True) module.exit_json(**_result) if module.params['version'] == "v3": _result['failed'] = True _result['msg'] = "ERROR: Donot support snmp version 3" _send_sandesh_log(object_log, _result.get('msg'), module.logger, True) module.exit_json(**_result) _result['msg'] = "Prefix(es) to be discovered: " + \ ','.join(module.params['subnets']) _send_sandesh_log(object_log, _result.get('msg'), module.logger, False) if len(all_hosts) < concurrent: concurrent = len(all_hosts) threadpool = pool.Pool(concurrent) try: vncapi = VncApi(auth_type=VncApi._KEYSTONE_AUTHN_STRATEGY, auth_token=job_ctx.get('auth_token')) for host in all_hosts: threadpool.start( Greenlet( _device_info_processing, str(host), vncapi, module, object_log, fabric_uuid)) threadpool.join() except Exception as ex: _result['failed'] = True _result['msg'] = "Failed to connect to API server due to error: %s"\ % str(ex) module.exit_json(**_result) _result['msg'] = "Device discovery complete" _send_sandesh_log(object_log, _result.get('msg'), module.logger, True) module.exit_json(**_result)
def client_test_freenet(N, t, options): ''' Test for the client with random delay channels command list i [target]: send a transaction to include for some particular party h [target]: stop some particular party m [target]: manually make particular party send some message help: show the help screen :param N: the number of parties :param t: the number of malicious parties :return None: ''' # Initialize keys required for operation initiateThresholdSig(open(options.threshold_keys, 'r').read()) initiateECDSAKeys(open(options.ecdsa, 'r').read()) initiateThresholdEnc(open(options.threshold_encs, 'r').read()) initializeGIPC(PK=getKeys()[0]) global logGreenlet logGreenlet = Greenlet(logWriter, open('msglog.TorMultiple', 'w')) logGreenlet.parent_args = (N, t) logGreenlet.name = 'client_test_freenet.logWriter' logGreenlet.start() # query amazon meta-data localIP = check_output( ['curl', 'http://169.254.169.254/latest/meta-data/public-ipv4']) myID = IP_LIST.index(localIP) N = len(IP_LIST) initiateRND(options.tx) def makeBroadcast(i): chans = [] # First establish N connections (including a self connection) for j in range(N): host, port = IP_MAPPINGS[j] # TOR_MAPPINGS[j] chans.append(connect_to_channel(host, port, i)) def _broadcast(v): for j in range(N): chans[j].put((j, i, v)) # from i to j def _send(j, v): chans[j].put((j, i, v)) return _broadcast, _send iterList = [myID] servers = [] for i in iterList: _, port = IP_MAPPINGS[i] servers.append(listen_to_channel(port)) print 'servers started' gevent.sleep(WAITING_SETUP_TIME_IN_SEC) # wait for set-up to be ready print 'sleep over' if True: # We only test for once initBeforeBinaryConsensus() ts = [] controlChannels = [Queue() for _ in range(N)] bcList = dict() sdList = dict() tList = [] def _makeBroadcast(x): bc, sd = makeBroadcast(x) bcList[x] = bc sdList[x] = sd for i in iterList: tmp_t = Greenlet(_makeBroadcast, i) tmp_t.parent_args = (N, t) tmp_t.name = 'client_test_freenet._makeBroadcast(%d)' % i tmp_t.start() tList.append(tmp_t) gevent.joinall(tList) rnd = Random() rnd.seed(123123) # This makes sure that all the EC2 instances have the same transaction pool transactionSet = set([ encodeTransaction(randomTransaction(rnd), randomGenerator=rnd) for trC in range(int(options.tx)) ]) # we are using the same one def toBeScheduled(): for i in iterList: bc = bcList[i] # makeBroadcast(i) sd = sdList[i] recv = servers[0].get th = Greenlet(honestParty, i, N, t, controlChannels[i], bc, recv, sd, options.B) th.parent_args = (N, t) th.name = 'client_test_freenet.honestParty(%d)' % i controlChannels[i].put(('IncludeTransaction', transactionSet)) th.start() mylog('Summoned party %i at time %f' % (i, time.time()), verboseLevel=-1) ts.append(th) try: gevent.joinall(ts) except ACSException: gevent.killall(ts) except finishTransactionLeap: ### Manually jump to this level print 'msgCounter', msgCounter print 'msgTypeCounter', msgTypeCounter # message id 0 (duplicated) for signatureCost logChannel.put(StopIteration) mylog("=====", verboseLevel=-1) for item in logChannel: mylog(item, verboseLevel=-1) mylog("=====", verboseLevel=-1) except gevent.hub.LoopExit: # Manual fix for early stop while True: gevent.sleep(1) checkExceptionPerGreenlet() finally: print "Consensus Finished" s = sched.scheduler(time.time, time.sleep) time_now = time.time() delay = options.delaytime - time_now s.enter(delay, 1, toBeScheduled, ()) print myID, "waits for", time_now + delay, 'now is', time_now s.run()
s.close() mylog('retrying (%s, %d) caused by %s...' % (hostname, port, str(e)), verboseLevel=-1) q = Queue() def _handle(): while True: obj = q.get() content = encode(obj) try: s.sendall(struct.pack('<I', len(content)) + content) except SocketError: print '!! [to %d] sending %d bytes' % (party, len(content)) gtemp = Greenlet(_handle) gtemp.parent_args = (hostname, port, party) gtemp.name = 'connect_to_channel._handle' gtemp.start() return q BASE_PORT = 49500 def getAddrFromEC2Summary(s): return [ x.split('ec2.')[-1] for x in s.replace('.compute.amazonaws.com', '').replace( '.us-west-1', '' # Later we need to add more such lines
def acs(pid, N, t, Q, broadcast, receive): assert (isinstance(Q, list)) assert (len(Q) == N) decideChannel = [Queue(1) for _ in range(N)] receivedChannelsFlags = [] def callbackFactory(i): def _callback(val): # Get notified for i # Greenlet(callBackWrap(binary_consensus, callbackFactory(i)), pid, # N, t, 1, make_bc(i), reliableBroadcastReceiveQueue[i].get).start() if not i in receivedChannelsFlags: receivedChannelsFlags.append(i) # mylog('B[%d]binary consensus_%d_starts with 1 at %f' % (pid, i, time.time()), verboseLevel=-1) greenletPacker( Greenlet(binary_consensus, i, pid, N, t, 1, decideChannel[i], make_bc(i), reliableBroadcastReceiveQueue[i].get), 'acs.callbackFactory.binary_consensus', (pid, N, t, Q, broadcast, receive)).start() return _callback for i, q in enumerate(Q): assert (isinstance(q, MonitoredInt)) q.registerSetCallBack(callbackFactory(i)) def make_bc(i): def _bc(m): broadcast((i, m)) return _bc reliableBroadcastReceiveQueue = [Queue() for x in range(N)] def _listener(): while True: sender, (instance, m) = receive() reliableBroadcastReceiveQueue[instance].put((sender, m)) greenletPacker(Greenlet(_listener), 'acs._listener', (pid, N, t, Q, broadcast, receive)).start() BA = [0] * N locker = Queue(1) locker2 = Queue(1) callbackCounter = [0] def listenerFactory(i, channel): def _listener(): BA[i] = channel.get() if callbackCounter[0] >= 2 * t and (not locker2.full()): locker2.put("Key") # Now we've got 2t+1 1's callbackCounter[0] += 1 if callbackCounter[0] == N and ( not locker.full()): # if we have all of them responded locker.put("Key") return _listener for i in range(N): greenletPacker(Greenlet(listenerFactory(i, decideChannel[i])), 'acs.listenerFactory(i, decideChannel[i])', (pid, N, t, Q, broadcast, receive)).start() locker2.get() # Now we feed 0 to all the other binary consensus protocols for i in range(N): if not i in receivedChannelsFlags: receivedChannelsFlags.append(i) greenletPacker( Greenlet(binary_consensus, i, pid, N, t, 0, decideChannel[i], make_bc(i), reliableBroadcastReceiveQueue[i].get), 'acs.binary_consensus', (pid, N, t, Q, broadcast, receive)).start() locker.get() # Now we can check''' BA = checkBA(BA, N, t) return BA
def run(self): ''' Runs the message processor, no lightweight threads are started until this point ''' Greenlet(self._run()).start()
def honestParty(pid, N, t, controlChannel, broadcast, receive, send, socket, B = -1): # RequestChannel is called by the client and it is the client's duty to broadcast the tx it wants to include if B < 0: B = int(math.ceil(N * math.log(N))) transactionCache = [] finishedTx = set() sentTx = set() proposals = [] receivedProposals = False commonSet = [] locks = defaultdict(lambda : Queue(1)) doneCombination = defaultdict(lambda : False) ENC_THRESHOLD = N - 2 * t global finishcount encPK, encSKs = getEncKeys() encCounter = defaultdict(lambda : {}) includeTransactionChannel = Queue() def probe(i): if len(encCounter[i]) >= ENC_THRESHOLD and receivedProposals and not locks[i].full() and not doneCombination[i]: # by == this part only executes once. oriM = encPK.combine_shares(deserializeEnc(proposals[i][:ENC_SERIALIZED_LENGTH]), dict(itertools.islice(encCounter[i].iteritems(), ENC_THRESHOLD)) ) doneCombination[i] = True locks[i].put(oriM) def listener(): while True: sender, msgBundle = receive() if msgBundle[0] == 'O': encCounter[msgBundle[1]][sender] = msgBundle[2] probe(msgBundle[1]) else: includeTransactionChannel.put((sender, msgBundle)) # redirect to includeTransaction Greenlet(listener).start() while True: op, msg = controlChannel.get() if op == "IncludeTransaction": if isinstance(msg, Transaction): # transactionCache.add(msg) transactionCache.append(msg) elif isinstance(msg, set): for tx in msg: transactionCache.append(tx) elif isinstance(msg, list): transactionCache.extend(msg) elif op == "Halt": break elif op == "Msg": broadcast(eval(msg)) # now the msg is something we mannually send mylog("timestampB (%d, %lf)" % (pid, time.time()), verboseLevel=-2) if len(transactionCache) < B: # Let's wait for many transactions. : ) time.sleep(0.5) print "Not enough transactions", len(transactionCache) continue oldest_B = transactionCache[:B] selected_B = random.sample(oldest_B, min(B/N, len(oldest_B))) print "[%d] proposing %d transactions" % (pid, min(B/N, len(oldest_B))) aesKey = random._urandom(32) # encrypted_B = encrypt(aesKey, ''.join(selected_B)) encryptedAESKey = encPK.encrypt(aesKey) proposal = serializeEnc(encryptedAESKey) + encrypted_B mylog("timestampIB (%d, %lf)" % (pid, time.time()), verboseLevel=-2) commonSet, proposals = includeTransaction(pid, N, t, proposal, broadcast, includeTransactionChannel.get, send) mylog("timestampIE (%d, %lf)" % (pid, time.time()), verboseLevel=-2) receivedProposals = True for i in range(N): probe(i) for i, c in enumerate(commonSet): # stx is the same for every party if c: share = encSKs[pid].decrypt_share(deserializeEnc(proposals[i][:ENC_SERIALIZED_LENGTH])) broadcast(('O', i, share)) mylog("timestampIE2 (%d, %lf)" % (pid, time.time()), verboseLevel=-2) recoveredSyncedTxList = [] def prepareTx(i): rec = locks[i].get() encodedTxSet = decrypt(rec, proposals[i][ENC_SERIALIZED_LENGTH:]) assert len(encodedTxSet) % TR_SIZE == 0 recoveredSyncedTx = [encodedTxSet[i:i+TR_SIZE] for i in range(0, len(encodedTxSet), TR_SIZE)] recoveredSyncedTxList.append(recoveredSyncedTx) thList = [] for i, c in enumerate(commonSet): # stx is the same for every party if c: s = Greenlet(prepareTx, i) thList.append(s) s.start() gevent.joinall(thList) mylog("timestampE (%d, %lf)" % (pid, time.time()), verboseLevel=-2) for rtx in recoveredSyncedTxList: finishedTx.update(set(rtx)) for transaction in rtx: if transaction not in sentTx: sentTx.add(transaction) # send_envelope(socket, transaction.envelope) mylog("Sent back envelope %d to BFTProxy" % transaction.id, verboseLevel=-2) mylog("[%d] %d distinct tx synced and %d tx left in the pool." % (pid, len(finishedTx), len(transactionCache) - len(finishedTx)), verboseLevel=-2) lock.get() finishcount += 1 lock.put(1) # if finishcount >= N - t: # convenient for local experiments # sys.exit() mylog("[%d] Now halting..." % (pid))
def test_socket_reclamation(self): if sys.platform.startswith('java'): raise SkipTest("Jython can't do socket reclamation") # Check that if a thread starts a request and dies without ending # the request, that the socket is reclaimed into the pool. cx_pool = self.get_pool( pair=(host, port), max_size=10, net_timeout=1000, conn_timeout=1000, use_ssl=False, ) self.assertEqual(0, len(cx_pool.sockets)) lock = None the_sock = [None] def leak_request(): self.assertEqual(NO_REQUEST, cx_pool._get_request_state()) cx_pool.start_request() self.assertEqual(NO_SOCKET_YET, cx_pool._get_request_state()) sock_info = cx_pool.get_socket() self.assertEqual(sock_info, cx_pool._get_request_state()) the_sock[0] = id(sock_info.sock) cx_pool.maybe_return_socket(sock_info) if not self.use_greenlets: lock.release() if self.use_greenlets: g = Greenlet(leak_request) g.start() g.join(1) self.assertTrue(g.ready(), "Greenlet is hung") else: lock = thread.allocate_lock() lock.acquire() # Start a thread WITHOUT a threading.Thread - important to test that # Pool can deal with primitive threads. thread.start_new_thread(leak_request, ()) # Join thread acquired = lock.acquire() self.assertTrue(acquired, "Thread is hung") # Make sure thread is really gone time.sleep(1) if 'PyPy' in sys.version: gc.collect() # Access the thread local from the main thread to trigger the # ThreadVigil's delete callback, returning the request socket to # the pool. # In Python 2.6 and lesser, a dead thread's locals are deleted # and those locals' weakref callbacks are fired only when another # thread accesses the locals and finds the thread state is stale. # This is more or less a bug in Python <= 2.6. Accessing the thread # local from the main thread is a necessary part of this test, and # realistic: in a multithreaded web server a new thread will access # Pool._local soon after an old thread has died. getattr(cx_pool._local, 'whatever', None) # Pool reclaimed the socket self.assertEqual(1, len(cx_pool.sockets)) self.assertEqual(the_sock[0], id(one(cx_pool.sockets).sock))
def multiSigBr(pid, N, t, msg, broadcast, receive, outputs, send): # Since all the parties we have are symmetric, so I implement this function for N instances of A-cast as a whole # Here msg is a set of transactions assert(isinstance(outputs, list)) for i in outputs: assert(isinstance(i, Queue)) keys = getECDSAKeys() Threshold = N - 2 * t Threshold2 = N - t zfecEncoder = zfec.Encoder(Threshold, N) zfecDecoder = zfec.Decoder(Threshold, N) def merkleTree(strList, someHash = coolSHA256Hash): # someHash is a mapping from a int to a int treeLength = 2 ** ceil(math.log(len(strList)) / math.log(2)) mt = [0] * (treeLength * 2) # find a place to put our leaves for i in range(len(strList)): mt[i + treeLength] = someHash(strList[i]) # TODO: need to change strList[i] from a string to an integer here. for i in range(treeLength - 1, 0, -1): # 1, 2, 3, ..., treeLength - 1 # mt[i] = someHash(''.join([chr(ord(a) ^ ord(b)) for a, b in zip(mt[i*2], mt[i*2+1])])) # XOR is commutative mt[i] = someHash(mt[i*2] + mt[i*2+1]) # concat is not commutative return mt def getMerkleBranch(index, mt): res = [] t = index + (len(mt) >> 1) while t > 1: res.append(mt[t ^ 1]) # we are picking up the sibling t /= 2 return res def merkleVerify(val, rootHash, branch, someHash, index): # index has information on whether we are facing a left sibling or a right sibling tmp = someHash(val) tindex = index for br in branch: tmp = someHash((tindex & 1) and br + tmp or tmp + br) tindex >>= 1 if tmp != rootHash: print "Verification failed with", someHash(val), rootHash, branch, tmp == rootHash return tmp == rootHash def Listener(): opinions = [defaultdict(lambda: 0) for _ in range(N)] rootHashes = dict() readyCounter = [defaultdict(lambda: 0) for _ in range(N)] signed = [False]*N readySent = [False] * N reconstDone = [False] * N while True: # main loop sender, msgBundle = receive() if msgBundle[0] == 'i' and not signed[sender]: if keys[sender].verify(sha1hash(''.join([msgBundle[1][0], msgBundle[1][1], ''.join(msgBundle[1][2])])), msgBundle[2]): assert isinstance(msgBundle[1], tuple) if not merkleVerify(msgBundle[1][0], msgBundle[1][1], msgBundle[1][2], coolSHA256Hash, pid): continue if sender in rootHashes: if rootHashes[sender]!= msgBundle[1][1]: print "Cheating caught, exiting" sys.exit(0) else: rootHashes[sender] = msgBundle[1][1] newBundle = (sender, msgBundle[1][0], msgBundle[1][1], msgBundle[1][2]) # assert each frag has a length of step broadcast(('e', newBundle, keys[pid].sign( sha1hash(''.join([str(newBundle[0]), newBundle[1], newBundle[2], ''.join(newBundle[3])])) ))) signed[sender] = True else: raise ECDSASignatureError() elif msgBundle[0] == 'e': if keys[sender].verify(sha1hash(''.join([str(msgBundle[1][0]), msgBundle[1][1], msgBundle[1][2], ''.join(msgBundle[1][3])])), msgBundle[2]): originBundle = msgBundle[1] if not merkleVerify(originBundle[1], originBundle[2], originBundle[3], coolSHA256Hash, sender): continue if originBundle[0] in rootHashes: if rootHashes[originBundle[0]]!= originBundle[2]: print "Cheating caught, exiting" sys.exit(0) else: rootHashes[originBundle[0]] = originBundle[2] opinions[originBundle[0]][sender] = originBundle[1] # We are going to move this part to kekeketktktktk if len(opinions[originBundle[0]]) >= Threshold2 and not readySent[originBundle[0]]: readySent[originBundle[0]] = True broadcast(('r', originBundle[0], originBundle[2])) # We are broadcasting its hash else: raise ECDSASignatureError() elif msgBundle[0] == 'r': readyCounter[msgBundle[1]][msgBundle[2]] += 1 tmp = readyCounter[msgBundle[1]][msgBundle[2]] if tmp >= t+1 and not readySent[msgBundle[1]]: readySent[msgBundle[1]] = True broadcast(('r', msgBundle[1], msgBundle[2])) if tmp >= Threshold2 and not outputs[msgBundle[1]].full() and \ not reconstDone[msgBundle[1]] and len(opinions[msgBundle[1]]) >= Threshold: reconstDone[msgBundle[1]] = True if msgBundle[1] in rootHashes: if rootHashes[msgBundle[1]]!= msgBundle[2]: print "Cheating caught, exiting" sys.exit(0) else: rootHashes[msgBundle[1]] = msgBundle[2] if opinions[msgBundle[1]].values()[0] == '': reconstruction = [''] else: reconstruction = zfecDecoder.decode(opinions[msgBundle[1]].values()[:Threshold], opinions[msgBundle[1]].keys()[:Threshold]) # We only take the first [Threshold] fragments rawbuf = ''.join(reconstruction) buf = rawbuf[:-ord(rawbuf[-1])] # Check root hash step = len(buf) / Threshold + 1 # len(buf) % Threshold == 0 and len(buf) / Threshold or (len(buf) / Threshold + 1) assert step * Threshold - len(buf) < 256 # assumption buf_ = buf.ljust(step * Threshold - 1, '\xFF') + chr(step * Threshold - len(buf)) fragList = [buf_[i*step : (i+1)*step] for i in range(Threshold)] encodedFragList = zfecEncoder.encode(fragList) mt = merkleTree(encodedFragList, coolSHA256Hash) assert rootHashes[msgBundle[1]] == mt[1] # full binary tree if outputs[msgBundle[1]].empty(): outputs[msgBundle[1]].put(buf) greenletPacker(Greenlet(Listener), 'multiSigBr.Listener', (pid, N, t, msg, broadcast, receive, outputs)).start() buf = msg # We already assumed the proposals are byte strings step = len(buf) / Threshold + 1 # len(buf) % Threshold == 0 and len(buf) / Threshold or (len(buf) / Threshold + 1) assert step * Threshold - len(buf) < 256 # assumption buf = buf.ljust(step * Threshold - 1, '\xFF') + chr(step * Threshold - len(buf)) fragList = [buf[i*step : (i+1)*step] for i in range(Threshold)] encodedFragList = zfecEncoder.encode(fragList) mt = merkleTree(encodedFragList, coolSHA256Hash) rootHash = mt[1] # full binary tree for i in range(N): mb = getMerkleBranch(i, mt) # notice that index starts from 1 and pid starts from 0 newBundle = (encodedFragList[i], rootHash, mb) send(i, ('i', newBundle, keys[pid].sign(sha1hash(''.join([newBundle[0], newBundle[1], ''.join(newBundle[2])])))))
print 'Connection established (%s, %d)' % (hostname, port) while True: obj = q.get() try: content = encode((myID, obj)) except TypeError: print obj raise try: s.sendall(struct.pack('<I', len(content)) + content) except SocketError: print '!! [to %d] sending %d bytes' % (myID, len(content)) break print 'closed channel' gtemp = Greenlet(_run) gtemp.parent_args = (hostname, port, myID) gtemp.name = 'connect_to_channel._handle' gtemp.start() return q def exception(msg): print "Exception: %s" % msg os.exit(1) def encode(m): return pickle.dumps(m)
def _test_pool(self, use_request): """ Test that the connection pool prevents both threads and greenlets from using a socket at the same time. Sequence: gr0: start a slow find() gr1: start a fast find() gr1: get results gr0: get results """ cx = get_connection(use_greenlets=self.use_greenlets, auto_start_request=False) db = cx.pymongo_test if not version.at_least(db.connection, (1, 7, 2)): raise SkipTest("Need at least MongoDB version 1.7.2 to use" " db.eval(nolock=True)") db.test.remove(safe=True) db.test.insert({'_id': 1}, safe=True) history = [] def find_fast(): if use_request: cx.start_request() history.append('find_fast start') # With greenlets and the old connection._Pool, this would throw # AssertionError: "This event is already used by another # greenlet" self.assertEqual({'_id': 1}, db.test.find_one()) history.append('find_fast done') if use_request: cx.end_request() def find_slow(): if use_request: cx.start_request() history.append('find_slow start') # Javascript function that pauses 5 sec. 'nolock' allows find_fast # to start and finish while we're waiting for this. fn = delay(5) self.assertEqual({ 'ok': 1.0, 'retval': True }, db.command('eval', fn, nolock=True)) history.append('find_slow done') if use_request: cx.end_request() if self.use_greenlets: gr0, gr1 = Greenlet(find_slow), Greenlet(find_fast) gr0.start() gr1.start_later(.1) else: gr0 = threading.Thread(target=find_slow) gr0.setDaemon(True) gr1 = threading.Thread(target=find_fast) gr1.setDaemon(True) gr0.start() time.sleep(.1) gr1.start() gr0.join() gr1.join() self.assertEqual([ 'find_slow start', 'find_fast start', 'find_fast done', 'find_slow done', ], history)
def wrapper(*args, **kwargs): thr = Greenlet(f, *args, **kwargs) thr.start()
def handle_handler(self, handler): """Add the Greenlet to the pool with ``handler.run()``, the function to run. :param handler: The RawRequestHandler provided in the constructor :type handler: RawRequestHandler""" self.pool.start(Greenlet(handler.run))
def _broadcast(v): def _deliver(j): buffers[j].put(encode((j, i, v))) for j in range(N): Greenlet(_deliver, j).start()
def handle_handler(self, handler): """Add the Greenlet to the pool with ``self.callback(handler, self)``: the function to run. :param handler: The RawRequestHandler provided in the constructor :type handler: RawRequestHandler""" self.pool.start(Greenlet(self.callback, handler, self))
def _start(): for runner in self._node_runners: pool.start(Greenlet(runner.start)) pool.join(raise_error=True) log.info("All nodes started")
def main_loop(config): """ Основной цикл приложения. :param config: конфигурация :type config: Config Алгоритм: * Открываем соединение с tarantool.queue, использую config.QUEUE_* настройки. * Создаем пул обработчиков. * Создаем очередь куда обработчики будут помещать выполненные задачи. * Пока количество обработчиков <= config.WORKER_POOL_SIZE, берем задачу из tarantool.queue и запускаем greenlet для ее обработки. * Посылаем уведомления о том, что задачи завершены в tarantool.queue. * Спим config.SLEEP секунд. """ logger.info( 'Connect to queue server on {host}:{port} space #{space}.'.format( host=config.QUEUE_HOST, port=config.QUEUE_PORT, space=config.QUEUE_SPACE)) queue = tarantool_queue.Queue(host=config.QUEUE_HOST, port=config.QUEUE_PORT, space=config.QUEUE_SPACE) logger.info('Use tube [{tube}], take timeout={take_timeout}.'.format( tube=config.QUEUE_TUBE, take_timeout=config.QUEUE_TAKE_TIMEOUT)) tube = queue.tube(config.QUEUE_TUBE) logger.info( 'Create worker pool[{size}].'.format(size=config.WORKER_POOL_SIZE)) worker_pool = Pool(config.WORKER_POOL_SIZE) processed_task_queue = gevent_queue.Queue() logger.info( 'Run main loop. Worker pool size={count}. Sleep time is {sleep}.'. format(count=config.WORKER_POOL_SIZE, sleep=config.SLEEP)) while run_application: free_workers_count = worker_pool.free_count() logger.debug( 'Pool has {count} free workers.'.format(count=free_workers_count)) for number in xrange(free_workers_count): logger.debug('Get task from tube for worker#{number}.'.format( number=number)) task = tube.take(config.QUEUE_TAKE_TIMEOUT) if task: logger.info( 'Start worker#{number} for task id={task_id}.'.format( task_id=task.task_id, number=number)) worker = Greenlet(notification_worker, task, processed_task_queue, timeout=config.HTTP_CONNECTION_TIMEOUT, verify=False) worker_pool.add(worker) worker.start() done_with_processed_tasks(processed_task_queue) sleep(config.SLEEP) else: logger.info('Stop application loop.')
def start_later(self, seconds): Greenlet(self.run).start_later(seconds)
def start(self): """Starts (or resumes) the execution of the program.""" self.green = Greenlet(self.main_loop) self.green.start_later(START_DELAY) self.program.start() self.can_reset = True