def DevDemo(args):
args = make_argument_parser().parse_args()
if not args.srv_host or not args.uuidfile:
print make_argument_parser().parse_args(['-h'])
exit(1)
errlog = ErrLog('DevDemo')
statlog = StatLog('DevDemo')
errworker = threading.Thread(target=logger_worker, args=(errqueue, errlog))
#errworker.daemon = True
errworker.start()
statworker = threading.Thread(target=logger_worker,
args=(statqueue, statlog))
#statworker.daemon = True
statworker.start()
host = ()
try:
d = args.srv_host.index(':')
host = (args.srv_host[:d], int(args.srv_host[d:]))
except:
host = (args.srv_host, 3478)
uulist = read_uuid_file(args.uuidfile)
pool = eventlet.GreenPool(len(uulist))
for uid in uulist:
#pt = threading.Thread(target=DevicesFunc,args=(host,uid))
pool.spawn_n(DevicesFunc, host, uid)
eventlet.sleep(0.3)
def AppDemo(args):
args = make_argument_parser().parse_args()
if not args.srv_host or not args.uuidfile:
print make_argument_parser().parse_args(['-h'])
exit(1)
errlog = ErrLog('AppDemo')
statlog = StatLog('AppDemo')
errworker = threading.Thread(target=logger_worker, args=(errqueue, errlog))
#errworker.daemon = True
errworker.start()
statworker = threading.Thread(target=logger_worker,
args=(statqueue, statlog))
#statworker.daemon = True
statworker.start()
host = ()
try:
d = args.srv_host.index(':')
host = (args.srv_host[:d], int(args.srv_host[d:]))
except:
host = (args.srv_host, 3478)
ulist = read_uuid_file(args.uuidfile)
bind = args.b_count if args.b_count < len(ulist) else len(ulist)
tbuf = ulist
bind = 1
ucount = len(ulist)
pool = eventlet.GreenPool(len(ulist))
for uid in ulist:
pool.spawn_n(APPfunc, host, uid)
eventlet.sleep(0.3)
def test_derived(self):
# Issue 3088: if there is a threads switch inside the __init__
# of a threading.local derived class, the per-thread dictionary
# is created but not correctly set on the object.
# The first member set may be bogus.
from eventlet.green import threading
from eventlet.green import time
class Local(threading.local):
def __init__(self):
time.sleep(0.01)
local = Local()
def f(i):
local.x = i
# Simply check that the variable is correctly set
self.assertEqual(local.x, i)
threads = []
for i in range(10):
t = threading.Thread(target=f, args=(i, ))
t.start()
threads.append(t)
for t in threads:
t.join()
def __init__(self, bitHopper):
self.curs = None
self.bitHopper = bitHopper
self.pool = bitHopper.pool
self.check_database()
self.shares = {}
self.rejects = {}
self.payout = {}
self.lock = threading.RLock()
thread = threading.Thread(target=self.write_database)
thread.start()
class Tunnel(object): # pylint: disable=R0902
"""Create a TCP server which will use TunnelHandler."""
def __init__(self,
target_host,
target_port,
sshclient,
tunnel_host='localhost',
tunnel_port=0):
"""Constructor."""
if not isinstance(sshclient, paramiko.SSHClient):
raise TypeError("'sshclient' must be an instance of "
"paramiko.SSHClient.")
self.target_host = target_host
self.target_port = target_port
self.target_address = (target_host, target_port)
self.address = (tunnel_host, tunnel_port)
self._tunnel = None
self._tunnel_thread = None
self.sshclient = sshclient
self._ssh_transport = self.get_sshclient_transport(self.sshclient)
TunnelHandler.target_address = self.target_address
TunnelHandler.ssh_transport = self._ssh_transport
self._tunnel = TunnelServer(self.address, TunnelHandler)
# reset attribute to the port it has actually been set to
self.address = self._tunnel.server_address
tunnel_host, self.tunnel_port = self.address
def get_sshclient_transport(self, sshclient):
"""Get the sshclient's transport.
Connect the sshclient, that has been passed in and return its
transport.
"""
sshclient.connect()
return sshclient.get_transport()
def serve_forever(self, async=True):
"""Serve the tunnel forever.
if async is True, this will be done in a background thread
"""
if not async:
self._tunnel.serve_forever()
else:
self._tunnel_thread = threading.Thread(
target=self._tunnel.serve_forever)
self._tunnel_thread.start()
# cooperative yield
time.sleep(0)
def ping_message(message):
@copy_current_request_context
def background_thread(addr):
cmd = 'ping -c 6 {}'.format(addr)
r = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
for line in iter(r.stdout.readline, b''):
line = str(line).strip('\r\n')
print(line)
emit('ping-reply', {'data': line}, namespace='/test')
addr = message['addr']
thread = threading.Thread(target=background_thread, args=(addr, ))
thread.daemon = True
thread.start()
def test_enumerate_after_join(self):
# Try hard to trigger #1703448: a thread is still returned in
# threading.enumerate() after it has been join()ed.
enum = threading.enumerate
old_interval = sys.getcheckinterval()
sys.setcheckinterval(1)
try:
for i in xrange(1, 1000):
t = threading.Thread(target=lambda: None)
t.start()
t.join()
l = enum()
self.assertFalse(
t in l, "#1703448 triggered after %d trials: %s" % (i, l))
finally:
sys.setcheckinterval(old_interval)
def main():
global _config
parser = argparse.ArgumentParser(description="")
parser.add_argument(
'config_path',
type=str,
help="path to the config file",
)
args = parser.parse_args()
_config = Config.from_file(args.config_path)
t = threading.Thread(target=run_server)
t.start()
t.join()
def test_rude_shutdown__write():
if test_support.verbose:
print "test_rude_shutdown__variant ..."
from eventlet.green import threading
# Some random port to connect to.
PORT = [9934]
listener_ready = threading.Event()
listener_gone = threading.Event()
# `listener` runs in a thread. It opens a socket listening on PORT, and
# sits in an accept() until the main thread connects. Then it rudely
# closes the socket, and sets Event `listener_gone` to let the main thread
# know the socket is gone.
def listener():
s = socket.socket()
PORT[0] = test_support.bind_port(s, '', PORT[0])
s.listen(5)
listener_ready.set()
s.accept()
s = None # reclaim the socket object, which also closes it
listener_gone.set()
def connector():
listener_ready.wait()
s = socket.socket()
s.connect(('localhost', PORT[0]))
listener_gone.wait()
try:
ssl_sock = socket.ssl(s)
ssl_sock.write("hello")
except socket.sslerror:
pass
else:
raise test_support.TestFailed(
'connecting to closed SSL socket should have failed')
t = threading.Thread(target=listener)
t.start()
connector()
t.join()
def __init__(self,
num_threads=DEFAULT_NUM_THREADS,
daemon_threads=True,
max_q_in_depth=None,
full_sleep_time=None):
self.num_threads = num_threads
self.max_q_in_depth = max_q_in_depth
self.full_sleep_time = full_sleep_time or DEFAULT_QUEUE_FULL_SLEEP_TIME
if self.max_q_in_depth:
self._q_in = Queue.Queue(self.max_q_in_depth)
else:
self._q_in = Queue.Queue()
self._stopped = False
self._threads = {}
for i in xrange(self.num_threads):
name = 'thread-%d' % i
self._threads[name] = threading.Thread(target=self._process_q,
name=name)
self._threads[name].start()
def __init__(self, cms_id, **kwargs):
super(VsdClientImpl, self).__init__()
self.restproxy = restproxy.RESTProxyServer(**kwargs)
response = self.restproxy.generate_nuage_auth()
if self.__renew_auth_key:
threading.Thread(target=self._auth_key_renewal,
args=[response]).start()
self.verify_cms(cms_id)
cms_id_helper.CMS_ID = cms_id
self.net_part = netpartition.NuageNetPartition(self.restproxy)
self.policygroups = policygroups.NuagePolicyGroups(self.restproxy)
self.redirecttargets = policygroups.NuageRedirectTargets(
self.restproxy)
self.l2domain = l2domain.NuageL2Domain(self.restproxy,
self.policygroups)
self.domain = domain.NuageDomain(self.restproxy, self.policygroups)
self.vm = vm.NuageVM(self.restproxy, self)
self.dhcp_options = dhcpoptions.NuageDhcpOptions(self.restproxy)
self.nuagegw = gateway.NuageGateway(self.restproxy, self.policygroups)
self.trunk = trunk.NuageTrunk(self.restproxy)
if not stable.exists(engine):
engine.connect().execute('''
CREATE TABLE account_status
(
uname character varying(255) NOT NULL ,
is_login boolean NOT NULL DEFAULT false,
last_login_time timestamp with time zone DEFAULT now(),
chost bigint[] NOT NULL DEFAULT '{0,0}'::bigint[],
CONSTRAINT account_status_uname_fkey FOREIGN KEY (uname)
REFERENCES account (uname) MATCH SIMPLE
ON UPDATE NO ACTION ON DELETE NO ACTION
)
''')
errqueue = Queue()
statqueue = Queue()
errlog = ErrLog('srv_log')
statlog = StatLog('srv_stat')
errworker = threading.Thread(target=logger_worker,args=(errqueue,errlog))
errworker.daemon = True
errworker.start()
statworker = threading.Thread(target=logger_worker,args=(statqueue,statlog))
statworker.daemon = True
statworker.start()
srv =EpollServer(port,errqueue,statqueue)
srv.run()
def spawn(self, func, *args, **kwargs):
t = green_threading.Thread(target=func, args=args, kwargs=kwargs)
t.daemon = True
t.start()
return t
def run():
threads = [threading.Thread(target=t) for i in range(300)]
for th in threads:
th.start()
for th in threads:
th.join()
def process_loop(self, hbuf):
gc.collect()
if check_packet_vaild(hbuf): # 校验包头
errqueue.put(','.join(
['sock',
'%d' % self.fileno, 'check_packet_vaild', hbuf]))
errqueue.put(hbuf)
return False
hattr = get_packet_head_class(hbuf[:STUN_HEADER_LENGTH])
if not hattr:
errqueue.put('sock %d,recv wrong head' % self.fileno)
return False
if stun_get_type(hattr.method) == STUN_METHOD_SEND:
if hattr.srcsock == 0xFFFFFFFF:
errqueue.put('sock %d,recv forward packet not srcsock,buf %s' %
(self.fileno, hbuf))
return False
dstsock = hattr.srcsock
self.dstsock = hattr.srcsock
if hattr.sequence[:2] == '03':
eventlet.sleep(0.01)
self.sbuf = self.send_data_to_app('02%s' % hattr.sequence[2:])
statqueue.put(
"%s,sock %d,recv from app number of hex(%s), buf: %s" %
(str(self.sock.getsockname()), self.fileno,
hattr.sequence[2:], hbuf))
#下面是我方主动发数据
elif hattr.sequence[:2] == '02':
rnum = int(hattr.sequence[2:], 16)
if self.mynum > 0xFFFFFF:
self.mynum = 0
errqueue.put(
'socket %d,packet counter is over 0xFFFFFF once' %
self.fileno)
elif self.mynum == rnum:
self.mynum += 1
statqueue.put(
"%s,sock %d,recv my confirm num %d is ok,data: %s" %
(str(
self.sock.getsockname()), self.fileno, rnum, hbuf))
else:
errqueue.put(
'sock %d,losing packet,recv number %d, my counter %d'
% (self.fileno, rnum, self.mynum))
self.sbuf = self.send_data_to_app('03%06x' % self.mynum)
self.timer_queue.put(0)
return self.write_sock()
p = parser_stun_package(hbuf[STUN_HEADER_LENGTH:-8])
if not p:
statqueue.put(','.join([
'sock',
'%d' % self.fileno, 'server packet is wrong,rdict is empty'
]))
return False # 出错了
if not stun_is_success_response_str(hattr.method):
errqueue.put(','.join(['sock','%d' % self.fileno,'server error sbuf',\
'method',hattr.method]))
return False
hattr.method = stun_get_type(hattr.method)
rdict = p[0]
if hattr.method == STUN_METHOD_ALLOCATE:
#statqueue.put('sock %d, login' % self.fileno)
"""
登录成功
"""
statqueue.put('sock %d,uuid %s login' % (self.fileno, self.uid))
try:
stat = rdict[STUN_ATTRIBUTE_STATE]
self.srcsock = int(stat[:8], 16)
except KeyError:
pass
elif hattr.method == STUN_METHOD_INFO:
try:
stat = rdict[STUN_ATTRIBUTE_STATE]
except KeyError:
errqueue.put("sock %d,recv not state,%s,rdict %s" %
(self.fileno, str(self.sock.getsockname()), hbuf))
else:
self.dstsock = int(stat[:8], 16)
self.sbuf = self.send_data_to_app('03%06x' % self.mynum)
pt = threading.Thread(target=self.retransmit_packet)
pt.start()
self.timer_queue.put(0)
return self.write_sock()
return False
def process_loop(self, rbuf):
gc.collect()
if check_packet_vaild(rbuf): # 校验包头
errqueue.put(','.join(
['sock',
'%d' % self.fileno, 'check_packet_vaild', rbuf]))
errqueue.put(rbuf)
return False
hattr = get_packet_head_class(rbuf[:STUN_HEADER_LENGTH])
if not hattr:
errqueue.put('sock %d,recv wrong head' % self.fileno)
return False
if stun_get_type(hattr.method) == STUN_METHOD_DATA: # 小机回应
if hattr.srcsock == 0xFFFFFFFF:
errqueue.put(
'sock %d, recv forward packet not srcsock,buf %s' %
(self.fileno, rbuf))
return False
self.dstsock = hattr.srcsock
if hattr.sequence[:2] == '03':
statqueue.put(
"%s,sock %d,recv from dev number of hex(%s), buf: %s" %
(str(self.sock.getsockname()), self.fileno,
hattr.sequence[2:], rbuf))
self.sbuf = self.stun_send_data_to_devid('02%s' %
hattr.sequence[2:])
elif hattr.sequence[:2] == '02':
n = int(hattr.sequence[2:], 16)
if n > 0xFFFFFF:
self.mynum = 0
errqueue.put('packet counter over 0xFFFFFF once')
elif n == self.mynum:
self.mynum += 1
statqueue.put(
"%s,sock %d,recv dev confirm num %d ok,data: %s" %
(str(self.sock.getsockname()), self.fileno, n, rbuf))
else:
errqueue.put(
'sock %d,lost packet,recv num %d,my counter %d' %
(self.fileno, n, self.mynum))
self.sbuf = self.stun_send_data_to_devid('03%06x' % self.mynum)
self.timer_queue.put(0)
statqueue.put("sock %d,send packet of %d to dev,data %s" %
(self.fileno, n, self.sbuf))
return self.write_sock()
if not stun_is_success_response_str(hattr.method):
if cmp(hattr.method[-2:], STUN_METHOD_REGISTER[-2:]):
errqueue.put(','.join(['sock','%d'% self.fileno,'recv server error',\
'method',hattr.method,rbuf]))
return False
else:
self.sbuf = self.stun_login_request()
return self.write_sock()
hattr.method = stun_get_type(hattr.method)
p = parser_stun_package(rbuf[STUN_HEADER_LENGTH:-8]) # 去头去尾
if p is None:
return False
rdict = p[0]
if not cmp(hattr.method, STUN_METHOD_BINDING):
stat = rdict[STUN_ATTRIBUTE_STATE]
self.srcsock = int(stat[:8], 16)
# 下面绑定一些UUID
#if len(self.ulist) > 1:
# self.sbuf = stun_bind_uuids(''.join(self.ulist))
#else:
statqueue.put('sock %d,uname %s login' % (self.fileno, self.user))
self.sbuf = self.stun_bind_single_uuid()
elif hattr.method == STUN_METHOD_REGISTER:
self.sbuf = self.stun_login_request()
elif hattr.method == STUN_METHOD_REFRESH:
return False
elif hattr.method == STUN_METHOD_CHANNEL_BIND:
# 绑定小机命令o
# 开启重传线程
p = threading.Thread(target=self.retransmit_packet)
p.start()
try:
self.dstsock = int(rdict[STUN_ATTRIBUTE_RUUID][-8:], 16)
if self.dstsock != 0xFFFFFFFF:
self.sbuf = self.stun_send_data_to_devid('03%06x' %
self.mynum)
statqueue.put(
'sock %d,start send packet to dev %d,buf %s' %
(self.fileno, self.dstsock, self.sbuf))
self.timer_queue.put(0)
else:
return False
except KeyError:
errqueue.put('sock %d,recv server packet not RUUID ,buf %s' %
(self.fileno, rbuf))
# elif rdict.has_key(STUN_ATTRIBUTE_MRUUID):
# mlist = split_mruuid(rdict[STUN_ATTRIBUTE_MRUUID])
# for n in mlist:
# eventlet.sleep(0.2)
# dstsock = int(n[-8:],16)
# if dstsock != 0xFFFFFFFF:
# pass
# #send_forward_buf(sock,srcsock,dstsock)
# return False
elif hattr.method == STUN_METHOD_INFO:
try:
self.dstsock = int(rdict[STUN_ATTRIBUTE_RUUID][-8:], 16)
self.sbuf = self.stun_send_data_to_devid('03%06x' % self.mynum)
statqueue.put('sock %d,start send packet to dev %d,buf: %s' %
(self.fileno, self.dstsock, self.sbuf))
self.timer_queue.put(0)
except KeyError:
errqueue.put('sock %d,recv server packet not RUUID ,buf %s' %
(self.fileno, rbuf))
# elif rdict.has_key(STUN_ATTRIBUTE_MRUUID):
# mlist = split_mruuid(rdict[STUN_ATTRIBUTE_MRUUID])
# for n in mlist:
# eventlet.sleep(0.2)
# dstsock = int(n[-8:],16)
# if dstsock != 0xFFFFFFFF:
# pass
# #send_forward_buf(sock,srcsock,dstsock)
# return False
elif hattr.method == STUN_METHOD_PULL:
pass
elif hattr.method == STUN_METHOD_MODIFY:
pass
elif hattr.method == STUN_METHOD_DELETE:
pass
else:
pass
return self.write_sock()
def spider(data):
global current_sid
global spider_async
print('spider sid is 0x',request.sid)
current_sid = request.sid
GSym.set_value('current_sid', current_sid)
# flask web container thread reenterable,here make sure at the same time only one index page could spider
# since message queue is introduced,thread semaphore here become dangerous,should re-implement by database inquiry
if not spider_semaphore.acquire(blocking=False):
print("not get spider_semaphore")
emit('progress', 80)
return
print(data)
print("get spider_semaphore")
# emit('progress', 0)
# emit('progress', 1)
# emit('progress', 1)
# UpdateSpiderProgress(2)
StartSeason = (data['date_start_month'] - 1) // 3 + 1
EndSeason = (data['date_end_month'] - 1) // 3 + 1
print("spider on pid and ppid", os.getpid(), os.getppid())
'''
spider_async = multiprocessing.Process(
target=StockModal.Spider.Spider_main,
kwargs={'StartYear':data['date_start_year'],
'EndYear':data['date_end_year'],'StartSeason':StartSeason,'EndSeason':EndSeason
}
)
spider_async.start()
'''
# client_g[request.sid]['out_que']=Queue()
# client_g[request.sid]['spider_progress_que'] = Queue()
client_g[request.sid]['spider_thread'] = threading.Thread(target=StockModal.Spider.Spider_main, name='Spider_main',
kwargs={'sid': current_sid, # no choice, sync weired reconnection of sid by ref
'StartYear': data['date_start_year'],
'EndYear': data['date_end_year'],
'StartSeason': StartSeason,
'EndSeason': EndSeason,
#'progress_que':client_g[request.sid]['spider_progress_que'],
#'out_que':client_g[request.sid]['out_que'],
'semaphore': spider_semaphore}
)
client_g[request.sid]['spider_thread'].setDaemon(True)
client_g[request.sid]['spider_thread'].start()
'''
client_g[request.sid]['sp_progress_thread']= threading.Thread(target=sp_progress_thread,
name='sp_progress_thread',
args=(request.sid,client_g[request.sid]['spider_progress_que'],))
client_g[request.sid]['sp_progress_thread'].setDaemon(True)
client_g[request.sid]['sp_progress_thread'].start()
'''
# start_background_task also works,but parameter assign got bug deep inside,
# args and kwargs will induce err in 3.4 lib,could only use serial assigning,and start could'nt be call manually
'''client_g[request.sid]['spider_thread'] = socketio.start_background_task(StockModal.Spider.Spider_main,
data['date_start_year'],
data['date_end_year'],
StartSeason,
EndSeason,
client_g[request.sid][
'spider_progress_que'],
client_g[request.sid]['out_que'],
spider_semaphore) #name='Spider_main',
#client_g[request.sid]['spider_thread'].setDaemon(True)
#client_g[request.sid]['spider_thread'].start()
client_g[request.sid]['sp_progress_thread']= socketio.start_background_task(sp_progress_thread,
#name='sp_progress_thread',
request.sid,client_g[request.sid]['spider_progress_que'])
#client_g[request.sid]['sp_progress_thread'].setDaemon(True)
#client_g[request.sid]['sp_progress_thread'].start()
'''
print('handle spider finished')
def Save_DB():
print('Save_DB')
Save_DB_Thread=threading.Thread(target=StockModal.GeneratorSINA.StartSaveDB, name='StartSaveDB')
Save_DB_Thread.setDaemon(True)
Save_DB_Thread.start()
print('Save_DB_Thread started')
consumer_key = os.environ['TWITTER_CONSUMER_KEY']
consumer_secret = os.environ['TWITTER_CONSUMER_SECRET']
token = os.environ['TWITTER_TOKEN']
token_secret = os.environ['TWITTER_TOKEN_SECRET']
def consume_tweet(t):
if 'place' in t and t['place'] is not None:
print('Sending tweet')
socketio.emit('tweet', t['place'], room='tweets')
stream = TwitterStream(consumer_key, consumer_secret, token, token_secret,
consume_tweet)
threading.Thread(target=stream.statuses.sample, daemon=True).start()
@app.route('/favicon.ico')
def favicon():
return send_from_directory(os.path.join(app.root_path, 'static'),
'favicon.ico',
mimetype='image/vnd.microsoft.icon')
@app.route('/')
def index():
return render_template('index.html')
@socketio.on('connect')
def make_thread(*args, **kwargs):
return threading.Thread(*args, **kwargs)
def run():
global _config
global _dump
global _traces
parser = argparse.ArgumentParser(description="")
parser.add_argument(
'config_path',
type=str,
help="path to the config file",
)
parser.add_argument(
'--dataset_size',
type=str,
help="config override",
)
parser.add_argument(
'--test',
type=str2bool,
help="confg override",
)
args = parser.parse_args()
_config = Config.from_file(args.config_path)
if args.dataset_size is not None:
_config.override(
'prooftrace_dataset_size',
args.dataset_size,
)
if args.test:
dataset_dir = os.path.join(
os.path.expanduser(_config.get('prooftrace_dataset_dir')),
_config.get('prooftrace_dataset_size'),
"test_traces",
)
else:
dataset_dir = os.path.join(
os.path.expanduser(_config.get('prooftrace_dataset_dir')),
_config.get('prooftrace_dataset_size'),
"train_traces",
)
ptre_path = os.path.join(dataset_dir, 'traces.embeds')
Log.out("Loading ProofTraceEmbeds", {
'path': ptre_path,
})
with gzip.open(ptre_path, 'rb') as f:
embeds = pickle.load(f)
_dump = {
'embeds': dict(embeds),
}
files = [os.path.join(dataset_dir, f) for f in os.listdir(dataset_dir)]
for p in files:
if re.search("\\.actions$", p) is None:
continue
Log.out("Loading ProofTraceActions", {
'path': p,
})
with gzip.open(p, 'rb') as f:
ptra = pickle.load(f)
_traces[ptra.name()] = {'actions': []}
for i in range(ptra.len()):
action = dict(ptra.actions()[i])
argument = dict(ptra.arguments()[i])
if 'hyp' in argument:
action['hyp'] = argument['hyp']
action['ccl'] = argument['ccl']
action['hash'] = argument['hash']
_traces[ptra.name()]['actions'].append(action)
t = threading.Thread(target=run_server)
t.start()
t.join()
