def DevDemo(args): args = make_argument_parser().parse_args() if not args.srv_host or not args.uuidfile: print make_argument_parser().parse_args(['-h']) exit(1) errlog = ErrLog('DevDemo') statlog = StatLog('DevDemo') errworker = threading.Thread(target=logger_worker, args=(errqueue, errlog)) #errworker.daemon = True errworker.start() statworker = threading.Thread(target=logger_worker, args=(statqueue, statlog)) #statworker.daemon = True statworker.start() host = () try: d = args.srv_host.index(':') host = (args.srv_host[:d], int(args.srv_host[d:])) except: host = (args.srv_host, 3478) uulist = read_uuid_file(args.uuidfile) pool = eventlet.GreenPool(len(uulist)) for uid in uulist: #pt = threading.Thread(target=DevicesFunc,args=(host,uid)) pool.spawn_n(DevicesFunc, host, uid) eventlet.sleep(0.3)
def AppDemo(args): args = make_argument_parser().parse_args() if not args.srv_host or not args.uuidfile: print make_argument_parser().parse_args(['-h']) exit(1) errlog = ErrLog('AppDemo') statlog = StatLog('AppDemo') errworker = threading.Thread(target=logger_worker, args=(errqueue, errlog)) #errworker.daemon = True errworker.start() statworker = threading.Thread(target=logger_worker, args=(statqueue, statlog)) #statworker.daemon = True statworker.start() host = () try: d = args.srv_host.index(':') host = (args.srv_host[:d], int(args.srv_host[d:])) except: host = (args.srv_host, 3478) ulist = read_uuid_file(args.uuidfile) bind = args.b_count if args.b_count < len(ulist) else len(ulist) tbuf = ulist bind = 1 ucount = len(ulist) pool = eventlet.GreenPool(len(ulist)) for uid in ulist: pool.spawn_n(APPfunc, host, uid) eventlet.sleep(0.3)
def test_derived(self): # Issue 3088: if there is a threads switch inside the __init__ # of a threading.local derived class, the per-thread dictionary # is created but not correctly set on the object. # The first member set may be bogus. from eventlet.green import threading from eventlet.green import time class Local(threading.local): def __init__(self): time.sleep(0.01) local = Local() def f(i): local.x = i # Simply check that the variable is correctly set self.assertEqual(local.x, i) threads = [] for i in range(10): t = threading.Thread(target=f, args=(i, )) t.start() threads.append(t) for t in threads: t.join()
def __init__(self, bitHopper): self.curs = None self.bitHopper = bitHopper self.pool = bitHopper.pool self.check_database() self.shares = {} self.rejects = {} self.payout = {} self.lock = threading.RLock() thread = threading.Thread(target=self.write_database) thread.start()
class Tunnel(object): # pylint: disable=R0902 """Create a TCP server which will use TunnelHandler.""" def __init__(self, target_host, target_port, sshclient, tunnel_host='localhost', tunnel_port=0): """Constructor.""" if not isinstance(sshclient, paramiko.SSHClient): raise TypeError("'sshclient' must be an instance of " "paramiko.SSHClient.") self.target_host = target_host self.target_port = target_port self.target_address = (target_host, target_port) self.address = (tunnel_host, tunnel_port) self._tunnel = None self._tunnel_thread = None self.sshclient = sshclient self._ssh_transport = self.get_sshclient_transport(self.sshclient) TunnelHandler.target_address = self.target_address TunnelHandler.ssh_transport = self._ssh_transport self._tunnel = TunnelServer(self.address, TunnelHandler) # reset attribute to the port it has actually been set to self.address = self._tunnel.server_address tunnel_host, self.tunnel_port = self.address def get_sshclient_transport(self, sshclient): """Get the sshclient's transport. Connect the sshclient, that has been passed in and return its transport. """ sshclient.connect() return sshclient.get_transport() def serve_forever(self, async=True): """Serve the tunnel forever. if async is True, this will be done in a background thread """ if not async: self._tunnel.serve_forever() else: self._tunnel_thread = threading.Thread( target=self._tunnel.serve_forever) self._tunnel_thread.start() # cooperative yield time.sleep(0)
def ping_message(message): @copy_current_request_context def background_thread(addr): cmd = 'ping -c 6 {}'.format(addr) r = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) for line in iter(r.stdout.readline, b''): line = str(line).strip('\r\n') print(line) emit('ping-reply', {'data': line}, namespace='/test') addr = message['addr'] thread = threading.Thread(target=background_thread, args=(addr, )) thread.daemon = True thread.start()
def test_enumerate_after_join(self): # Try hard to trigger #1703448: a thread is still returned in # threading.enumerate() after it has been join()ed. enum = threading.enumerate old_interval = sys.getcheckinterval() sys.setcheckinterval(1) try: for i in xrange(1, 1000): t = threading.Thread(target=lambda: None) t.start() t.join() l = enum() self.assertFalse( t in l, "#1703448 triggered after %d trials: %s" % (i, l)) finally: sys.setcheckinterval(old_interval)
def main(): global _config parser = argparse.ArgumentParser(description="") parser.add_argument( 'config_path', type=str, help="path to the config file", ) args = parser.parse_args() _config = Config.from_file(args.config_path) t = threading.Thread(target=run_server) t.start() t.join()
def test_rude_shutdown__write(): if test_support.verbose: print "test_rude_shutdown__variant ..." from eventlet.green import threading # Some random port to connect to. PORT = [9934] listener_ready = threading.Event() listener_gone = threading.Event() # `listener` runs in a thread. It opens a socket listening on PORT, and # sits in an accept() until the main thread connects. Then it rudely # closes the socket, and sets Event `listener_gone` to let the main thread # know the socket is gone. def listener(): s = socket.socket() PORT[0] = test_support.bind_port(s, '', PORT[0]) s.listen(5) listener_ready.set() s.accept() s = None # reclaim the socket object, which also closes it listener_gone.set() def connector(): listener_ready.wait() s = socket.socket() s.connect(('localhost', PORT[0])) listener_gone.wait() try: ssl_sock = socket.ssl(s) ssl_sock.write("hello") except socket.sslerror: pass else: raise test_support.TestFailed( 'connecting to closed SSL socket should have failed') t = threading.Thread(target=listener) t.start() connector() t.join()
def __init__(self, num_threads=DEFAULT_NUM_THREADS, daemon_threads=True, max_q_in_depth=None, full_sleep_time=None): self.num_threads = num_threads self.max_q_in_depth = max_q_in_depth self.full_sleep_time = full_sleep_time or DEFAULT_QUEUE_FULL_SLEEP_TIME if self.max_q_in_depth: self._q_in = Queue.Queue(self.max_q_in_depth) else: self._q_in = Queue.Queue() self._stopped = False self._threads = {} for i in xrange(self.num_threads): name = 'thread-%d' % i self._threads[name] = threading.Thread(target=self._process_q, name=name) self._threads[name].start()
def __init__(self, cms_id, **kwargs): super(VsdClientImpl, self).__init__() self.restproxy = restproxy.RESTProxyServer(**kwargs) response = self.restproxy.generate_nuage_auth() if self.__renew_auth_key: threading.Thread(target=self._auth_key_renewal, args=[response]).start() self.verify_cms(cms_id) cms_id_helper.CMS_ID = cms_id self.net_part = netpartition.NuageNetPartition(self.restproxy) self.policygroups = policygroups.NuagePolicyGroups(self.restproxy) self.redirecttargets = policygroups.NuageRedirectTargets( self.restproxy) self.l2domain = l2domain.NuageL2Domain(self.restproxy, self.policygroups) self.domain = domain.NuageDomain(self.restproxy, self.policygroups) self.vm = vm.NuageVM(self.restproxy, self) self.dhcp_options = dhcpoptions.NuageDhcpOptions(self.restproxy) self.nuagegw = gateway.NuageGateway(self.restproxy, self.policygroups) self.trunk = trunk.NuageTrunk(self.restproxy)
if not stable.exists(engine): engine.connect().execute(''' CREATE TABLE account_status ( uname character varying(255) NOT NULL , is_login boolean NOT NULL DEFAULT false, last_login_time timestamp with time zone DEFAULT now(), chost bigint[] NOT NULL DEFAULT '{0,0}'::bigint[], CONSTRAINT account_status_uname_fkey FOREIGN KEY (uname) REFERENCES account (uname) MATCH SIMPLE ON UPDATE NO ACTION ON DELETE NO ACTION ) ''') errqueue = Queue() statqueue = Queue() errlog = ErrLog('srv_log') statlog = StatLog('srv_stat') errworker = threading.Thread(target=logger_worker,args=(errqueue,errlog)) errworker.daemon = True errworker.start() statworker = threading.Thread(target=logger_worker,args=(statqueue,statlog)) statworker.daemon = True statworker.start() srv =EpollServer(port,errqueue,statqueue) srv.run()
def spawn(self, func, *args, **kwargs): t = green_threading.Thread(target=func, args=args, kwargs=kwargs) t.daemon = True t.start() return t
def run(): threads = [threading.Thread(target=t) for i in range(300)] for th in threads: th.start() for th in threads: th.join()
def process_loop(self, hbuf): gc.collect() if check_packet_vaild(hbuf): # 校验包头 errqueue.put(','.join( ['sock', '%d' % self.fileno, 'check_packet_vaild', hbuf])) errqueue.put(hbuf) return False hattr = get_packet_head_class(hbuf[:STUN_HEADER_LENGTH]) if not hattr: errqueue.put('sock %d,recv wrong head' % self.fileno) return False if stun_get_type(hattr.method) == STUN_METHOD_SEND: if hattr.srcsock == 0xFFFFFFFF: errqueue.put('sock %d,recv forward packet not srcsock,buf %s' % (self.fileno, hbuf)) return False dstsock = hattr.srcsock self.dstsock = hattr.srcsock if hattr.sequence[:2] == '03': eventlet.sleep(0.01) self.sbuf = self.send_data_to_app('02%s' % hattr.sequence[2:]) statqueue.put( "%s,sock %d,recv from app number of hex(%s), buf: %s" % (str(self.sock.getsockname()), self.fileno, hattr.sequence[2:], hbuf)) #下面是我方主动发数据 elif hattr.sequence[:2] == '02': rnum = int(hattr.sequence[2:], 16) if self.mynum > 0xFFFFFF: self.mynum = 0 errqueue.put( 'socket %d,packet counter is over 0xFFFFFF once' % self.fileno) elif self.mynum == rnum: self.mynum += 1 statqueue.put( "%s,sock %d,recv my confirm num %d is ok,data: %s" % (str( self.sock.getsockname()), self.fileno, rnum, hbuf)) else: errqueue.put( 'sock %d,losing packet,recv number %d, my counter %d' % (self.fileno, rnum, self.mynum)) self.sbuf = self.send_data_to_app('03%06x' % self.mynum) self.timer_queue.put(0) return self.write_sock() p = parser_stun_package(hbuf[STUN_HEADER_LENGTH:-8]) if not p: statqueue.put(','.join([ 'sock', '%d' % self.fileno, 'server packet is wrong,rdict is empty' ])) return False # 出错了 if not stun_is_success_response_str(hattr.method): errqueue.put(','.join(['sock','%d' % self.fileno,'server error sbuf',\ 'method',hattr.method])) return False hattr.method = stun_get_type(hattr.method) rdict = p[0] if hattr.method == STUN_METHOD_ALLOCATE: #statqueue.put('sock %d, login' % self.fileno) """ 登录成功 """ statqueue.put('sock %d,uuid %s login' % (self.fileno, self.uid)) try: stat = rdict[STUN_ATTRIBUTE_STATE] self.srcsock = int(stat[:8], 16) except KeyError: pass elif hattr.method == STUN_METHOD_INFO: try: stat = rdict[STUN_ATTRIBUTE_STATE] except KeyError: errqueue.put("sock %d,recv not state,%s,rdict %s" % (self.fileno, str(self.sock.getsockname()), hbuf)) else: self.dstsock = int(stat[:8], 16) self.sbuf = self.send_data_to_app('03%06x' % self.mynum) pt = threading.Thread(target=self.retransmit_packet) pt.start() self.timer_queue.put(0) return self.write_sock() return False
def process_loop(self, rbuf): gc.collect() if check_packet_vaild(rbuf): # 校验包头 errqueue.put(','.join( ['sock', '%d' % self.fileno, 'check_packet_vaild', rbuf])) errqueue.put(rbuf) return False hattr = get_packet_head_class(rbuf[:STUN_HEADER_LENGTH]) if not hattr: errqueue.put('sock %d,recv wrong head' % self.fileno) return False if stun_get_type(hattr.method) == STUN_METHOD_DATA: # 小机回应 if hattr.srcsock == 0xFFFFFFFF: errqueue.put( 'sock %d, recv forward packet not srcsock,buf %s' % (self.fileno, rbuf)) return False self.dstsock = hattr.srcsock if hattr.sequence[:2] == '03': statqueue.put( "%s,sock %d,recv from dev number of hex(%s), buf: %s" % (str(self.sock.getsockname()), self.fileno, hattr.sequence[2:], rbuf)) self.sbuf = self.stun_send_data_to_devid('02%s' % hattr.sequence[2:]) elif hattr.sequence[:2] == '02': n = int(hattr.sequence[2:], 16) if n > 0xFFFFFF: self.mynum = 0 errqueue.put('packet counter over 0xFFFFFF once') elif n == self.mynum: self.mynum += 1 statqueue.put( "%s,sock %d,recv dev confirm num %d ok,data: %s" % (str(self.sock.getsockname()), self.fileno, n, rbuf)) else: errqueue.put( 'sock %d,lost packet,recv num %d,my counter %d' % (self.fileno, n, self.mynum)) self.sbuf = self.stun_send_data_to_devid('03%06x' % self.mynum) self.timer_queue.put(0) statqueue.put("sock %d,send packet of %d to dev,data %s" % (self.fileno, n, self.sbuf)) return self.write_sock() if not stun_is_success_response_str(hattr.method): if cmp(hattr.method[-2:], STUN_METHOD_REGISTER[-2:]): errqueue.put(','.join(['sock','%d'% self.fileno,'recv server error',\ 'method',hattr.method,rbuf])) return False else: self.sbuf = self.stun_login_request() return self.write_sock() hattr.method = stun_get_type(hattr.method) p = parser_stun_package(rbuf[STUN_HEADER_LENGTH:-8]) # 去头去尾 if p is None: return False rdict = p[0] if not cmp(hattr.method, STUN_METHOD_BINDING): stat = rdict[STUN_ATTRIBUTE_STATE] self.srcsock = int(stat[:8], 16) # 下面绑定一些UUID #if len(self.ulist) > 1: # self.sbuf = stun_bind_uuids(''.join(self.ulist)) #else: statqueue.put('sock %d,uname %s login' % (self.fileno, self.user)) self.sbuf = self.stun_bind_single_uuid() elif hattr.method == STUN_METHOD_REGISTER: self.sbuf = self.stun_login_request() elif hattr.method == STUN_METHOD_REFRESH: return False elif hattr.method == STUN_METHOD_CHANNEL_BIND: # 绑定小机命令o # 开启重传线程 p = threading.Thread(target=self.retransmit_packet) p.start() try: self.dstsock = int(rdict[STUN_ATTRIBUTE_RUUID][-8:], 16) if self.dstsock != 0xFFFFFFFF: self.sbuf = self.stun_send_data_to_devid('03%06x' % self.mynum) statqueue.put( 'sock %d,start send packet to dev %d,buf %s' % (self.fileno, self.dstsock, self.sbuf)) self.timer_queue.put(0) else: return False except KeyError: errqueue.put('sock %d,recv server packet not RUUID ,buf %s' % (self.fileno, rbuf)) # elif rdict.has_key(STUN_ATTRIBUTE_MRUUID): # mlist = split_mruuid(rdict[STUN_ATTRIBUTE_MRUUID]) # for n in mlist: # eventlet.sleep(0.2) # dstsock = int(n[-8:],16) # if dstsock != 0xFFFFFFFF: # pass # #send_forward_buf(sock,srcsock,dstsock) # return False elif hattr.method == STUN_METHOD_INFO: try: self.dstsock = int(rdict[STUN_ATTRIBUTE_RUUID][-8:], 16) self.sbuf = self.stun_send_data_to_devid('03%06x' % self.mynum) statqueue.put('sock %d,start send packet to dev %d,buf: %s' % (self.fileno, self.dstsock, self.sbuf)) self.timer_queue.put(0) except KeyError: errqueue.put('sock %d,recv server packet not RUUID ,buf %s' % (self.fileno, rbuf)) # elif rdict.has_key(STUN_ATTRIBUTE_MRUUID): # mlist = split_mruuid(rdict[STUN_ATTRIBUTE_MRUUID]) # for n in mlist: # eventlet.sleep(0.2) # dstsock = int(n[-8:],16) # if dstsock != 0xFFFFFFFF: # pass # #send_forward_buf(sock,srcsock,dstsock) # return False elif hattr.method == STUN_METHOD_PULL: pass elif hattr.method == STUN_METHOD_MODIFY: pass elif hattr.method == STUN_METHOD_DELETE: pass else: pass return self.write_sock()
def spider(data): global current_sid global spider_async print('spider sid is 0x',request.sid) current_sid = request.sid GSym.set_value('current_sid', current_sid) # flask web container thread reenterable,here make sure at the same time only one index page could spider # since message queue is introduced,thread semaphore here become dangerous,should re-implement by database inquiry if not spider_semaphore.acquire(blocking=False): print("not get spider_semaphore") emit('progress', 80) return print(data) print("get spider_semaphore") # emit('progress', 0) # emit('progress', 1) # emit('progress', 1) # UpdateSpiderProgress(2) StartSeason = (data['date_start_month'] - 1) // 3 + 1 EndSeason = (data['date_end_month'] - 1) // 3 + 1 print("spider on pid and ppid", os.getpid(), os.getppid()) ''' spider_async = multiprocessing.Process( target=StockModal.Spider.Spider_main, kwargs={'StartYear':data['date_start_year'], 'EndYear':data['date_end_year'],'StartSeason':StartSeason,'EndSeason':EndSeason } ) spider_async.start() ''' # client_g[request.sid]['out_que']=Queue() # client_g[request.sid]['spider_progress_que'] = Queue() client_g[request.sid]['spider_thread'] = threading.Thread(target=StockModal.Spider.Spider_main, name='Spider_main', kwargs={'sid': current_sid, # no choice, sync weired reconnection of sid by ref 'StartYear': data['date_start_year'], 'EndYear': data['date_end_year'], 'StartSeason': StartSeason, 'EndSeason': EndSeason, #'progress_que':client_g[request.sid]['spider_progress_que'], #'out_que':client_g[request.sid]['out_que'], 'semaphore': spider_semaphore} ) client_g[request.sid]['spider_thread'].setDaemon(True) client_g[request.sid]['spider_thread'].start() ''' client_g[request.sid]['sp_progress_thread']= threading.Thread(target=sp_progress_thread, name='sp_progress_thread', args=(request.sid,client_g[request.sid]['spider_progress_que'],)) client_g[request.sid]['sp_progress_thread'].setDaemon(True) client_g[request.sid]['sp_progress_thread'].start() ''' # start_background_task also works,but parameter assign got bug deep inside, # args and kwargs will induce err in 3.4 lib,could only use serial assigning,and start could'nt be call manually '''client_g[request.sid]['spider_thread'] = socketio.start_background_task(StockModal.Spider.Spider_main, data['date_start_year'], data['date_end_year'], StartSeason, EndSeason, client_g[request.sid][ 'spider_progress_que'], client_g[request.sid]['out_que'], spider_semaphore) #name='Spider_main', #client_g[request.sid]['spider_thread'].setDaemon(True) #client_g[request.sid]['spider_thread'].start() client_g[request.sid]['sp_progress_thread']= socketio.start_background_task(sp_progress_thread, #name='sp_progress_thread', request.sid,client_g[request.sid]['spider_progress_que']) #client_g[request.sid]['sp_progress_thread'].setDaemon(True) #client_g[request.sid]['sp_progress_thread'].start() ''' print('handle spider finished')
def Save_DB(): print('Save_DB') Save_DB_Thread=threading.Thread(target=StockModal.GeneratorSINA.StartSaveDB, name='StartSaveDB') Save_DB_Thread.setDaemon(True) Save_DB_Thread.start() print('Save_DB_Thread started')
consumer_key = os.environ['TWITTER_CONSUMER_KEY'] consumer_secret = os.environ['TWITTER_CONSUMER_SECRET'] token = os.environ['TWITTER_TOKEN'] token_secret = os.environ['TWITTER_TOKEN_SECRET'] def consume_tweet(t): if 'place' in t and t['place'] is not None: print('Sending tweet') socketio.emit('tweet', t['place'], room='tweets') stream = TwitterStream(consumer_key, consumer_secret, token, token_secret, consume_tweet) threading.Thread(target=stream.statuses.sample, daemon=True).start() @app.route('/favicon.ico') def favicon(): return send_from_directory(os.path.join(app.root_path, 'static'), 'favicon.ico', mimetype='image/vnd.microsoft.icon') @app.route('/') def index(): return render_template('index.html') @socketio.on('connect')
def make_thread(*args, **kwargs): return threading.Thread(*args, **kwargs)
def run(): global _config global _dump global _traces parser = argparse.ArgumentParser(description="") parser.add_argument( 'config_path', type=str, help="path to the config file", ) parser.add_argument( '--dataset_size', type=str, help="config override", ) parser.add_argument( '--test', type=str2bool, help="confg override", ) args = parser.parse_args() _config = Config.from_file(args.config_path) if args.dataset_size is not None: _config.override( 'prooftrace_dataset_size', args.dataset_size, ) if args.test: dataset_dir = os.path.join( os.path.expanduser(_config.get('prooftrace_dataset_dir')), _config.get('prooftrace_dataset_size'), "test_traces", ) else: dataset_dir = os.path.join( os.path.expanduser(_config.get('prooftrace_dataset_dir')), _config.get('prooftrace_dataset_size'), "train_traces", ) ptre_path = os.path.join(dataset_dir, 'traces.embeds') Log.out("Loading ProofTraceEmbeds", { 'path': ptre_path, }) with gzip.open(ptre_path, 'rb') as f: embeds = pickle.load(f) _dump = { 'embeds': dict(embeds), } files = [os.path.join(dataset_dir, f) for f in os.listdir(dataset_dir)] for p in files: if re.search("\\.actions$", p) is None: continue Log.out("Loading ProofTraceActions", { 'path': p, }) with gzip.open(p, 'rb') as f: ptra = pickle.load(f) _traces[ptra.name()] = {'actions': []} for i in range(ptra.len()): action = dict(ptra.actions()[i]) argument = dict(ptra.arguments()[i]) if 'hyp' in argument: action['hyp'] = argument['hyp'] action['ccl'] = argument['ccl'] action['hash'] = argument['hash'] _traces[ptra.name()]['actions'].append(action) t = threading.Thread(target=run_server) t.start() t.join()