def fetch_page(session, url):
resp = yield from session.get(url)
if resp.status == 200:
text = yield from resp.text()
alog.info(f"GET {resp.url} HTTP/1.0 {resp.status} OK")
yield from session.close()
def check_token():
alog.info('check_token', 'check_token')
param_raw = {}
def getValue(key, default=''):
value = request.GET.get(key, default).strip()
if value is None or len(value) == 0:
value = default
param_raw.setdefault(key, value)
else:
param_raw.setdefault(key, value)
return value
try:
signature = getValue('signature')
timestamp = getValue('timestamp')
nonce = getValue('nonce')
echostr = getValue('echostr')
token = "xxxx" #请按照公众平台官网\基本配置中信息填写
list = [token, timestamp, nonce]
list.sort()
sha1 = hashlib.sha1()
map(sha1.update, list)
hashcode = sha1.hexdigest()
print("handle/GET func: hashcode:" + hashcode + " signature:" +
signature)
if hashcode == signature:
alog.info('check ok:' + echostr, 'check_token')
return echostr
else:
return ""
except Exception as Argument:
return Argument
def GET(self):
try:
data = web.input()
alog.debug(data, 'wx_handle::GET', 'input')
if len(data) == 0:
return "hello, this is handle view"
signature = data.signature
timestamp = data.timestamp
nonce = data.nonce
echostr = data.echostr
token = "haojiage" #请按照公众平台官网\基本配置中信息填写
list = [token.encode('utf-8'), timestamp.encode('utf-8'), nonce.encode('utf-8')]
list.sort()
alog.info(list, 'wx_handle::GET', 'list')
sha1 = hashlib.sha1()
#map(sha1.update, list)
for i in list:
sha1.update(i)
hashcode = sha1.hexdigest()
alog.info("handle/GET func: hashcode:"+hashcode+' signature:'+signature, 'wx_handle::GET', 'check')
if hashcode == signature:
return echostr
else:
return ""
except Exception as Argument:
return Argument
def goodprice(codes):
alog.info(codes, 'goodprice')
if len(codes) == 0:
return u'goodprice/<codes>'
calc = GoodPriceCalc()
price_data = calc.calc(codes)
return calc.to_json(price_data, indent=4)
def _latent_generator(self, z_pre, E_xs, E_xe, u, l_id, reuse):
# u: Dx1
# z_pre: N x T2 x H x W x C
# E_xs, E_xe: N x 1 x H × W × C, N x 1 x H × W × C
[N, T2, H, W, C]= z_pre.get_shape().as_list()
with tf.variable_scope("latent_layer" + str(l_id), reuse=reuse):
u_l = self.get_layer_u(u, l_id, reuse, z_pre.shape[0]) # [4,64]
u_l = tf.expand_dims(u_l, 0)
print("-------print u_l.shape------") ##adl
import alog ##adl
from pprint import pprint ##adl
alog.info("u_l.shape") ##adl
print(">>> type(u_l.shape) = ", type(u_l.shape)) ##adl
if hasattr(u_l.shape, "shape"): ##adl
print(">>> u_l.shape.shape", u_l.shape.shape) ##adl
if type(u_l.shape) is list: ##adl
print(">>> len(u_l.shape) = ", len(u_l.shape)) ##adl
pprint(u_l.shape) ##adl
else: ##adl
pprint(u_l.shape) ##adl
print("------------------------\n") ##adl
out = conv1d(u_l, self.dim_C * 3, k=3, d=1, name="g_s_e_3conv")
_g_s, _g_e, n_l = tf.split(out, 3, axis=-1)
g_s = tf.sigmoid(_g_s)
g_e = tf.sigmoid(_g_e)
var_list = [g_s.shape, E_xs.shape, z_pre.shape, n_l.shape] ##adl
for idx,x in enumerate(var_list): ##adl
var_names = "g_s.shape, E_xs.shape, z_pre.shape, n_l.shape".split(",") ##adl
cur_name = var_names[idx] ##adl
print("-------print "+ cur_name + "------") ##adl
import alog ##adl
from pprint import pprint ##adl
alog.info(cur_name) ##adl
print(">>> type(x) = ", type(x)) ##adl
if hasattr(x, "shape"): ##adl
print(">>> " + cur_name + ".shape", x.shape) ##adl
if type(x) is list: ##adl
print(">>> len(" + cur_name + ") = ", len(x)) ##adl
pprint(x) ##adl
else: ##adl
pprint(x) ##adl
pass ##adl
print("------------------------\n") ##adl
# g_s [1, 2, 64]
g_s = tf.reshape(g_s, [1, T2, 1, 1, C])
g_e = tf.reshape(g_e, [1, T2, 1, 1, C])
n_l = tf.reshape(n_l, [1, T2, 1, 1, C])
# raise ValueError
z_in = g_s * E_xs + g_e * E_xe + tf.maximum(1 - g_s - g_e, 0) * z_pre + n_l
# raise ValueError
z_in2 = conv3d(z_in, self.dim_C, k_h=3, k_w=3, k_dep=3, name="conv3d1")
z_in2 = tf.nn.leaky_relu(z_in2)
z_in3 = conv3d(z_in2, self.dim_C, k_h=3, k_w=3, k_dep=3, name="conv3d2")
z_l = tf.nn.leaky_relu(z_pre + z_in3)
return z_l
def szpe():
ckey = 'szpe'
try:
value = DataGether._cache(ckey)
return float(value)
except:
pe = ShenZhen.szpe()
alog.info("http get:" + str(pe), 'good_price', 'DataGether::szpe')
DataGether._cache(ckey, str(pe))
return pe
def chinabond10():
ckey = 'chinabond10'
try:
value = DataGether._cache(ckey)
return float(value)
except:
chinabond10 = chinabond.chinabond()
chinabond10 = chinabond10 / 100
alog.info("http get:" + str(chinabond10), 'good_price',
'DataGether::chinabond10')
DataGether._cache(ckey, str(chinabond10))
return chinabond10
def get_layer_u(self, u, l_id, reuse, n_bz):
T_l = [4, 4, 4, 4, 4, 4, 4, 4,
8, 8, 8, 8, 8, 8, 8, 8,
16, 16, 16, 16, 16, 16, 16, 16]
#
# print("-------print u.shape------") ##adl
# import alog ##adl
# from pprint import pprint ##adl
# alog.info("u.shape") ##adl
# print(">>> type(u.shape) = ", type(u.shape)) ##adl
# if hasattr(u.shape, "shape"): ##adl
# print(">>> u.shape.shape", u.shape.shape) ##adl
# if type(u.shape) is list: ##adl
# print(">>> len(u.shape) = ", len(u.shape)) ##adl
# pprint(u.shape) ##adl
# else: ##adl
# pprint(u.shape) ##adl
# print("------------------------\n") ##adl
# raise ValueError
with tf.variable_scope("layer_u" + str(l_id), reuse=reuse):
w = tf.get_variable('w',
[T_l[l_id] * self.dim_C, self.dim_D],
initializer=tf.contrib.layers.xavier_initializer())
biases = tf.get_variable('biases', [T_l[l_id] * self.dim_C, 1],
initializer=tf.constant_initializer(0.0))
var_list = [w.shape, u.shape] ##adl
for idx,x in enumerate(var_list): ##adl
var_names = "w.shape, u.shape".split(",") ##adl
cur_name = var_names[idx] ##adl
print("-------print "+ cur_name + "------") ##adl
import alog ##adl
from pprint import pprint ##adl
alog.info(cur_name) ##adl
print(">>> type(x) = ", type(x)) ##adl
if hasattr(x, "shape"): ##adl
print(">>> " + cur_name + ".shape", x.shape) ##adl
if type(x) is list: ##adl
print(">>> len(" + cur_name + ") = ", len(x)) ##adl
pprint(x) ##adl
else: ##adl
pprint(x) ##adl
pass ##adl
print("------------------------\n") ##adl
ul = tf.matmul(w, u) + biases
ul2 = tf.reshape(ul, [T_l[l_id], self.dim_C])
return ul2
def async_syntax():
"""async syntax
"""
start = time.time()
#loop = asyncio.get_event_loop()
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
tasks = [
asyncio.ensure_future(async_fetch_page(f"{url}{u}")) for u in URLS
]
loop.run_until_complete(asyncio.wait(tasks))
loop.close()
alog.info(f'async syntax Coroutines took {time.time() - start:.1f} sec')
def _send_message(self, method, params):
self._method = method
self._id += 1
message = \
{
"jsonrpc" : "2.0",
"id" : self._id,
"method" : self._method,
"params" : params
}
msg = json.dumps(message)
alog.info("sending request: " + str(msg))
self.send(msg)
async def __call__(self, scope, receive, send):
# locals inside the app function (send_wrapper) can't be assigned to,
# as the interpreter detects the assignment and thus creates a new
# local variable within that function, with that name.
instance = {'http_status_code': None}
def send_wrapper(response):
if response['type'] == 'http.response.start':
instance['http_status_code'] = response['status']
return send(response)
if scope['type'] != 'http':
alog.info(
f"ASGI scope of type {scope['type']} is not supported yet")
await self.app(scope, receive, send)
return
try:
metric_name = self.metric_namer(scope)
except AttributeError as e:
alog.error(
f"Unable to extract metric name from asgi scope: {scope}, skipping statsd timing"
)
alog.error(f" -> exception: {e}")
await self.app(scope, receive, send)
return
def emit(stats):
statsd_tags = [
f"http_status:{instance['http_status_code']}",
f"http_method:{scope['method']}"
]
self.client.timing(f"{metric_name}",
stats.time,
tags=statsd_tags + ["time:wall"])
self.client.timing(f"{metric_name}",
stats.cpu_time,
tags=statsd_tags + ["time:cpu"])
with TimingStats(metric_name) as stats:
try:
await self.app(scope, receive, send_wrapper)
except Exception:
stats.stop()
instance['http_status_code'] = 500
emit(stats)
raise
emit(stats)
def quote_ttm(symbol):
alog.info('111')
#xueqiu.set_token('xq_a_token=a2a6ce3addb6502c2e7618a455fc6ae2e48d9544')
resp = xueqiu.fetch(symbol, 'quote_detail', params={})
#jsresp = json.loads(resp, encoding='utf-8')
#util.print_json(resp, 'xueqiu.fetch quote_detail:'+symbol)
quote = resp['data']['quote']
data = {CN_symbol: symbol}
data[CN_name] = quote['name']
data[CN_current] = str(quote['current'])
data[CN_pe_ttm] = str(quote['pe_ttm'])
data[CN_dividend] = str(quote['dividend'])
data[CN_dividend_yield] = str(quote['dividend_yield'])
data[CN_high52w] = str(quote['high52w'])
data[CN_low52w] = str(quote['low52w'])
#util.print_json(data, '11')
return data
def average_gradients(tower_grads):
average_grads = []
for grad_and_vars in zip(*tower_grads):
# Note that each grad_and_vars looks like the following:
# ((grad0_gpu0, var0_gpu0), ... , (grad0_gpuN, var0_gpuN))
grads = []
for g, _ in grad_and_vars:
# Add 0 dimension to the gradients to represent the tower.
var_list = [g, _] ##adl
for idx, x in enumerate(var_list): ##adl
var_names = "g, _".split(",") ##adl
cur_name = var_names[idx] ##adl
print("-------print " + cur_name + "------") ##adl
import alog ##adl
from pprint import pprint ##adl
alog.info(cur_name) ##adl
print(">>> type(x) = ", type(x)) ##adl
if hasattr(x, "shape"): ##adl
print(">>> " + cur_name + ".shape", x.shape) ##adl
if type(x) is list: ##adl
print(">>> len(" + cur_name + ") = ", len(x)) ##adl
pprint(x) ##adl
else: ##adl
pprint(x) ##adl
pass ##adl
print("------------------------\n") ##adl
expanded_g = tf.expand_dims(g, 0)
# Append on a 'tower' dimension which we will average over below.
grads.append(expanded_g)
# Average over the 'tower' dimension.
grad = tf.concat(axis=0, values=grads)
grad = tf.reduce_mean(grad, 0)
# Keep in mind that the Variables are redundant because they are shared
# across towers. So .. we will just return the first tower's pointer to
# the Variable.
v = grad_and_vars[0][1]
grad_and_var = (grad, v)
average_grads.append(grad_and_var)
return average_grads
def gen_url(self, test):
if test:
base_url = 'https://test.deribit.com/ws/api/v2'
else:
base_url = 'https://www.deribit.com/api/v2'
url_parts = list(urlparse(base_url))
# query_string = ''
# if self.heartbeat:
# query_string = '?heartbeat=true'
# url = "wss://{}/realtime{}".format(url_parts[1], query_string)
url = "wss://{}/ws/api/v2".format(url_parts[1])
alog.info(url)
return url
def yieldfrom_syntax():
start = time.time()
#loop = asyncio.get_event_loop()
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
# yield from syntax
session = aiohttp.ClientSession(loop=loop)
tasks = [
#loop.create_task(fetch_page(session, f"{url}/foo")),
#loop.create_task(fetch_page(session, f"{url}/bar"))
asyncio.ensure_future(fetch_page(session, f"{url}{u}")) for u in URLS
]
loop.run_until_complete(asyncio.wait(tasks))
loop.close()
alog.info(
f'yieldfrom syntax Coroutines took {time.time() - start:.1f} sec')
async def produce(loop, queue, n, fake_msg):
"""
Create fake rs232 data into queue, it's blocking
:param queue
:param n
"""
index = 0
try:
async for data in pv200(n, fake_msg):
# produce rs232 data
alog.info(f"producing {index}/{n}, id:{id(data)}, {data.split(',')[2]}")
index += 1
# simulate i/o operation data
await asyncio.sleep(random.random())
# put into list
await queue.put(data)
except Exception as e:
alog.info(e)
def on_message(self, message):
"""Handler for parsing WS messages."""
response = json.loads(message)
# if response
if "id" in response:
if "result" in response:
result = response["result"]
elif "error" in response:
alog.error(response["error"])
return
else:
return
id = response["id"]
if id == self._id:
if self._method == "public/auth":
self._authInfo["access_token"] = result["access_token"]
self._authInfo["expires_in"] = result["expires_in"]
self._authInfo["refresh_token"] = result["refresh_token"]
self._authInfo["scope"] = result["scope"]
self._authInfo["token_type"] = result["token_type"]
alog.info("access_token: {}".format(self._authInfo["access_token"]))
self.emit('subscribe')
# wst = threading.Thread(target=self.__on_expires_in)
# wst.daemon = True
# wst.start()
return
elif self._method == "public/subscribe" or self._method == "private/subscribe":
self.emit('subscribed', result)
return
alog.info(message)
if "method" in response:
method = response["method"]
if method == "subscription":
params = response["params"]
channel = params["channel"]
#if "user.orders" in channel:
self.emit('orders', channel, params["data"])
def header(self):
"""Return auth headers. Will use API Keys if present in settings."""
auth_header = []
if self.should_auth:
alog.info("Authenticating with API Key.")
# To auth to the WS using an API key, we generate a signature
# of a nonce and the WS API endpoint.
alog.debug(settings.BITMEX_API_KEY)
nonce = generate_nonce()
api_signature = generate_signature(settings.BITMEX_API_SECRET,
'GET', '/realtime', nonce, '')
auth_header = [
"api-nonce: " + str(nonce), "api-signature: " + api_signature,
"api-key:" + settings.BITMEX_API_KEY
]
return auth_header
async def produce_rs232(queue, n=None, **kwargs):
"""
Creat the real rs232 data into queue, is non-blocking
:param queue
:param n
:param **kwargs
"""
reader, writer = await serial_asyncio.open_serial_connection(url=com, **kwargs)
buffers = ''
index = 0
while True:
data = await reader.readline()
buffers += data.decode('utf-8')
alog.info(buffers)
buf = buffers.split('\r')[-2].split(',')
if len(buf) < rs232_len:
alog.info(f"Drop not illeage {len(buf)}")
buffers = ''
elif '\r' in buffers:
buf = buffers.split('\r')
last_received, buffers = buffers.split('\r')[-2:]
buffers = buffers.strip() # clean \n
alog.info(f"producing {index}/{n}, id:{id(last_received)}, {last_received.split(',')[2]}")
index += 1
await queue.put(last_received)
def POST(self):
try:
alog.debug('hello', 'wx_handle', 'WXHandle::POST')
webData = web.data()
alog.info("Handle Post webdata is " + str(webData), 'wx_handle', 'WXHandle::POST')
recMsg = wx_receive.parse_xml(webData)
if isinstance(recMsg, wx_receive.Msg) and recMsg.MsgType == 'text':
toUser = recMsg.FromUserName
fromUser = recMsg.ToUserName
alog.debug('handle_type:'+recMsg.handle_type, 'WXHandle::POST')
if recMsg.handle_type == 'goodprice':
calc = GoodPriceCalc()
price_data = calc.calc(recMsg.Content)
content = calc.to_text(price_data, '\n')
elif 'xueqiu_token' == recMsg.handle_type:
xueqiu.set_token('xq_a_token='+recMsg.Content)
content = u'设置雪球token成功:' + xueqiu.get_token()
else:
content = u'不支持命令:' + recMsg.Content
replyMsg = wx_reply.TextMsg(toUser, fromUser, content)
s = replyMsg.send()
alog.debug(s, 'wx_handle', 'POST reply')
return s
else:
alog.info('not support', 'WXHandle::POST', u'recMsg.MsgType')
return "success"
except Exception as Argment:
alog.info(Argment, 'WXHandle::POST', u'error')
return Argment
def get_auth(self):
'''Return auth headers. Will use API Keys if present in settings.'''
alog.debug('shouldAuth: %s' % self.shouldAuth)
if self.shouldAuth:
alog.info("Authenticating with API Key.")
# To auth to the WS using an API key, we generate a signature
# of a nonce and the WS API endpoint.
alog.debug(settings.GDAX_API_KEY)
nonce = generate_nonce()
api_signature = generate_signature(
settings.GDAX_API_SECRET, 'GET', '/realtime', nonce, '')
auth = [
"api-nonce: " + str(nonce),
"api-signature: " + api_signature,
"api-key:" + settings.GDAX_API_KEY
]
alog.debug(auth)
return auth
else:
return []
def goodprice_wx():
xmldata = touni(request._get_body_string())
alog.info(xmldata, 'goodprice_wx')
#try:
if True:
recMsg = wx_receive.parse_xml(xmldata)
if isinstance(recMsg, wx_receive.Msg) and recMsg.MsgType == 'text':
toUser = recMsg.FromUserName
fromUser = recMsg.ToUserName
recv_content = touni(recMsg.Content)
calc = GoodPriceCalc()
alog.debug(recv_content, 'goodprice_wx', 'recMsg.Content')
price_data = calc.calc(recv_content)
content = calc.to_text(price_data, '<br>')
replyMsg = wx_reply.TextMsg(toUser, fromUser, recv_content)
s = replyMsg.send()
alog.debug(s, 'goodprice_wx', 'reply')
return s
else:
alog.info(s, 'goodprice_wx', u'不支持')
return "success"
""" except Exception as Argment:
async def consume(queue):
"""
Post rs232 data into server
:param queue
"""
async with aiohttp.ClientSession() as session:
try:
while True:
# wait for an data from producer
data = await queue.get()
# process the item
alog.info(f"consuming id:{id(data)}")
status, result = await async_post(session, url_boards, data)
if status != 201:
alog.error(f"error: id:{id(data)}, {status}, {result}")
await queue.put(data)
# simulate i/o operation using sleep
await asyncio.sleep(random.random())
# Notify the queue that the item has been processed
queue.task_done()
except Exception as e:
alog.info(f"Finish: {e}.")
def connect_websocket(self):
"""Connect to the websocket in a thread."""
alog.debug("### Connecting Websocket ###")
# setup websocket.run_forever arguments
wsRunArgs = {
'sslopt': {"cert_reqs": ssl.CERT_NONE}
}
if self.heartbeatEnabled:
wsRunArgs['ping_timeout'] = 20
wsRunArgs['ping_interval'] = 60
alog.debug("websocket.run_forever: %s" % (wsRunArgs))
# Run the websocket on another thread and enable heartbeat
self.wst = threading.Thread(
target=lambda: self.websocket_run_forever(wsRunArgs)
)
self.wst.daemon = True
self.wst.start()
alog.info("### Started thread")
self.wait_for_connection()
def calc(self, codes, use_cache=True):
code_list = codes.split(',')
if len(codes) < 5 or len(code_list) < 1:
return {'code': {'error': 'codes is error:' + codes}}
dts = datetime.datetime.now().strftime('%Y%m%d')
cachefile = '.cache/' + codes + '-' + dts + '.json'
if use_cache is True:
try:
redata = util.load_json(cachefile)
alog.info('hit cache:' + codes + ' cache file:' + cachefile,
'good_price', 'calc')
return redata
except:
pass
self.chinabond10 = DataGether.chinabond10()
self.market_pe = DataGether.szpe()
market_data = {
CN_chinabond10: "%.3f%%" % (self.chinabond10 * 100),
CN_szpe: str(self.market_pe)
}
alog.info(
'market_pe:' + str(self.market_pe) + ' chinabond10:' +
str(self.chinabond10), 'good_price', 'calc')
redata = {}
for code in code_list:
symbol = util.code2symbal(code)
alog.info(symbol + '...', 'good_price', 'calc')
if True:
try:
#if True:
ttm_data = DataGether.quote_ttm(symbol)
#util.print_json(ttm_data)
itemdata = {}
itemdata['a'] = ttm_data
itemdata['b'] = market_data
itemdata['c'] = self.calc_good_price(symbol, ttm_data)
redata[symbol] = itemdata
#else:
except Exception as e1:
print("[req_quote] error:" + str(e1))
itemdata = {}
itemdata['a'] = {CN_symbol: symbol}
itemdata['b'] = market_data
itemdata['c'] = {}
redata[symbol] = itemdata
continue
else:
ttm_data = DataGether.quote_ttm(symbol)
#util.print_json(ttm_data)
itemdata = self.calc_good_price(symbol, ttm_data)
redata[symbol] = itemdata
time.sleep(2)
util.dump_json(cachefile, redata)
return redata
def header(self):
"""Return auth headers. Will use API Keys if present in settings."""
auth_header = []
alog.info(f'### should auth {self.should_auth} ###')
if self.should_auth:
alog.info("Authenticating with API Key.")
# To auth to the WS using an API key, we generate a signature
# of a nonce and the WS API endpoint.
alog.info((self.api_key, self.api_secret))
nonce = generate_nonce()
api_signature = generate_signature(self.api_secret, 'GET',
'/realtime', nonce, '')
auth_header = [
"api-nonce: " + str(nonce), "api-signature: " + api_signature,
"api-key:" + self.api_key
]
alog.info(alog.pformat(auth_header))
return auth_header
def record_history(self, user, retdata, symbal):
alog.debug(user + ' ' + symbal, 'good_price::record_history')
util.print_json(retdata)
if len(user) <= 0:
alog.info("user is empty", 'good_price::record_history')
return
dts = datetime.datetime.now().strftime('%Y-%m-%d')
if symbal in retdata.keys():
try:
itemdata = retdata[symbal]['basic']
ctx = '\t'.join([
user, dts, symbal, itemdata[CN_name], itemdata[CN_current],
itemdata[CN_pe_ttm]
])
util.dump_str("data/user/goodprice_history_%s.txt" % (user),
ctx + '\n',
mode='a')
except:
alog.info("histroy data error", 'good_price::record_history')
else:
alog.info(symbal + "not in retdata", 'good_price::record_history')
return
def on_close(self):
alog.info('Websocket Closed')
def on_close(self, info):
alog.info('Websocket Closed: ' + str(info))
def on_open(self, *kwargs):
alog.info("Websocket opened")
alog.debug("Websocket Opened. args: " + str(kwargs))
self.emit('open')
