def fetch_page(session, url): resp = yield from session.get(url) if resp.status == 200: text = yield from resp.text() alog.info(f"GET {resp.url} HTTP/1.0 {resp.status} OK") yield from session.close()
def check_token(): alog.info('check_token', 'check_token') param_raw = {} def getValue(key, default=''): value = request.GET.get(key, default).strip() if value is None or len(value) == 0: value = default param_raw.setdefault(key, value) else: param_raw.setdefault(key, value) return value try: signature = getValue('signature') timestamp = getValue('timestamp') nonce = getValue('nonce') echostr = getValue('echostr') token = "xxxx" #请按照公众平台官网\基本配置中信息填写 list = [token, timestamp, nonce] list.sort() sha1 = hashlib.sha1() map(sha1.update, list) hashcode = sha1.hexdigest() print("handle/GET func: hashcode:" + hashcode + " signature:" + signature) if hashcode == signature: alog.info('check ok:' + echostr, 'check_token') return echostr else: return "" except Exception as Argument: return Argument
def GET(self): try: data = web.input() alog.debug(data, 'wx_handle::GET', 'input') if len(data) == 0: return "hello, this is handle view" signature = data.signature timestamp = data.timestamp nonce = data.nonce echostr = data.echostr token = "haojiage" #请按照公众平台官网\基本配置中信息填写 list = [token.encode('utf-8'), timestamp.encode('utf-8'), nonce.encode('utf-8')] list.sort() alog.info(list, 'wx_handle::GET', 'list') sha1 = hashlib.sha1() #map(sha1.update, list) for i in list: sha1.update(i) hashcode = sha1.hexdigest() alog.info("handle/GET func: hashcode:"+hashcode+' signature:'+signature, 'wx_handle::GET', 'check') if hashcode == signature: return echostr else: return "" except Exception as Argument: return Argument
def goodprice(codes): alog.info(codes, 'goodprice') if len(codes) == 0: return u'goodprice/<codes>' calc = GoodPriceCalc() price_data = calc.calc(codes) return calc.to_json(price_data, indent=4)
def _latent_generator(self, z_pre, E_xs, E_xe, u, l_id, reuse): # u: Dx1 # z_pre: N x T2 x H x W x C # E_xs, E_xe: N x 1 x H × W × C, N x 1 x H × W × C [N, T2, H, W, C]= z_pre.get_shape().as_list() with tf.variable_scope("latent_layer" + str(l_id), reuse=reuse): u_l = self.get_layer_u(u, l_id, reuse, z_pre.shape[0]) # [4,64] u_l = tf.expand_dims(u_l, 0) print("-------print u_l.shape------") ##adl import alog ##adl from pprint import pprint ##adl alog.info("u_l.shape") ##adl print(">>> type(u_l.shape) = ", type(u_l.shape)) ##adl if hasattr(u_l.shape, "shape"): ##adl print(">>> u_l.shape.shape", u_l.shape.shape) ##adl if type(u_l.shape) is list: ##adl print(">>> len(u_l.shape) = ", len(u_l.shape)) ##adl pprint(u_l.shape) ##adl else: ##adl pprint(u_l.shape) ##adl print("------------------------\n") ##adl out = conv1d(u_l, self.dim_C * 3, k=3, d=1, name="g_s_e_3conv") _g_s, _g_e, n_l = tf.split(out, 3, axis=-1) g_s = tf.sigmoid(_g_s) g_e = tf.sigmoid(_g_e) var_list = [g_s.shape, E_xs.shape, z_pre.shape, n_l.shape] ##adl for idx,x in enumerate(var_list): ##adl var_names = "g_s.shape, E_xs.shape, z_pre.shape, n_l.shape".split(",") ##adl cur_name = var_names[idx] ##adl print("-------print "+ cur_name + "------") ##adl import alog ##adl from pprint import pprint ##adl alog.info(cur_name) ##adl print(">>> type(x) = ", type(x)) ##adl if hasattr(x, "shape"): ##adl print(">>> " + cur_name + ".shape", x.shape) ##adl if type(x) is list: ##adl print(">>> len(" + cur_name + ") = ", len(x)) ##adl pprint(x) ##adl else: ##adl pprint(x) ##adl pass ##adl print("------------------------\n") ##adl # g_s [1, 2, 64] g_s = tf.reshape(g_s, [1, T2, 1, 1, C]) g_e = tf.reshape(g_e, [1, T2, 1, 1, C]) n_l = tf.reshape(n_l, [1, T2, 1, 1, C]) # raise ValueError z_in = g_s * E_xs + g_e * E_xe + tf.maximum(1 - g_s - g_e, 0) * z_pre + n_l # raise ValueError z_in2 = conv3d(z_in, self.dim_C, k_h=3, k_w=3, k_dep=3, name="conv3d1") z_in2 = tf.nn.leaky_relu(z_in2) z_in3 = conv3d(z_in2, self.dim_C, k_h=3, k_w=3, k_dep=3, name="conv3d2") z_l = tf.nn.leaky_relu(z_pre + z_in3) return z_l
def szpe(): ckey = 'szpe' try: value = DataGether._cache(ckey) return float(value) except: pe = ShenZhen.szpe() alog.info("http get:" + str(pe), 'good_price', 'DataGether::szpe') DataGether._cache(ckey, str(pe)) return pe
def chinabond10(): ckey = 'chinabond10' try: value = DataGether._cache(ckey) return float(value) except: chinabond10 = chinabond.chinabond() chinabond10 = chinabond10 / 100 alog.info("http get:" + str(chinabond10), 'good_price', 'DataGether::chinabond10') DataGether._cache(ckey, str(chinabond10)) return chinabond10
def get_layer_u(self, u, l_id, reuse, n_bz): T_l = [4, 4, 4, 4, 4, 4, 4, 4, 8, 8, 8, 8, 8, 8, 8, 8, 16, 16, 16, 16, 16, 16, 16, 16] # # print("-------print u.shape------") ##adl # import alog ##adl # from pprint import pprint ##adl # alog.info("u.shape") ##adl # print(">>> type(u.shape) = ", type(u.shape)) ##adl # if hasattr(u.shape, "shape"): ##adl # print(">>> u.shape.shape", u.shape.shape) ##adl # if type(u.shape) is list: ##adl # print(">>> len(u.shape) = ", len(u.shape)) ##adl # pprint(u.shape) ##adl # else: ##adl # pprint(u.shape) ##adl # print("------------------------\n") ##adl # raise ValueError with tf.variable_scope("layer_u" + str(l_id), reuse=reuse): w = tf.get_variable('w', [T_l[l_id] * self.dim_C, self.dim_D], initializer=tf.contrib.layers.xavier_initializer()) biases = tf.get_variable('biases', [T_l[l_id] * self.dim_C, 1], initializer=tf.constant_initializer(0.0)) var_list = [w.shape, u.shape] ##adl for idx,x in enumerate(var_list): ##adl var_names = "w.shape, u.shape".split(",") ##adl cur_name = var_names[idx] ##adl print("-------print "+ cur_name + "------") ##adl import alog ##adl from pprint import pprint ##adl alog.info(cur_name) ##adl print(">>> type(x) = ", type(x)) ##adl if hasattr(x, "shape"): ##adl print(">>> " + cur_name + ".shape", x.shape) ##adl if type(x) is list: ##adl print(">>> len(" + cur_name + ") = ", len(x)) ##adl pprint(x) ##adl else: ##adl pprint(x) ##adl pass ##adl print("------------------------\n") ##adl ul = tf.matmul(w, u) + biases ul2 = tf.reshape(ul, [T_l[l_id], self.dim_C]) return ul2
def async_syntax(): """async syntax """ start = time.time() #loop = asyncio.get_event_loop() loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) tasks = [ asyncio.ensure_future(async_fetch_page(f"{url}{u}")) for u in URLS ] loop.run_until_complete(asyncio.wait(tasks)) loop.close() alog.info(f'async syntax Coroutines took {time.time() - start:.1f} sec')
def _send_message(self, method, params): self._method = method self._id += 1 message = \ { "jsonrpc" : "2.0", "id" : self._id, "method" : self._method, "params" : params } msg = json.dumps(message) alog.info("sending request: " + str(msg)) self.send(msg)
async def __call__(self, scope, receive, send): # locals inside the app function (send_wrapper) can't be assigned to, # as the interpreter detects the assignment and thus creates a new # local variable within that function, with that name. instance = {'http_status_code': None} def send_wrapper(response): if response['type'] == 'http.response.start': instance['http_status_code'] = response['status'] return send(response) if scope['type'] != 'http': alog.info( f"ASGI scope of type {scope['type']} is not supported yet") await self.app(scope, receive, send) return try: metric_name = self.metric_namer(scope) except AttributeError as e: alog.error( f"Unable to extract metric name from asgi scope: {scope}, skipping statsd timing" ) alog.error(f" -> exception: {e}") await self.app(scope, receive, send) return def emit(stats): statsd_tags = [ f"http_status:{instance['http_status_code']}", f"http_method:{scope['method']}" ] self.client.timing(f"{metric_name}", stats.time, tags=statsd_tags + ["time:wall"]) self.client.timing(f"{metric_name}", stats.cpu_time, tags=statsd_tags + ["time:cpu"]) with TimingStats(metric_name) as stats: try: await self.app(scope, receive, send_wrapper) except Exception: stats.stop() instance['http_status_code'] = 500 emit(stats) raise emit(stats)
def quote_ttm(symbol): alog.info('111') #xueqiu.set_token('xq_a_token=a2a6ce3addb6502c2e7618a455fc6ae2e48d9544') resp = xueqiu.fetch(symbol, 'quote_detail', params={}) #jsresp = json.loads(resp, encoding='utf-8') #util.print_json(resp, 'xueqiu.fetch quote_detail:'+symbol) quote = resp['data']['quote'] data = {CN_symbol: symbol} data[CN_name] = quote['name'] data[CN_current] = str(quote['current']) data[CN_pe_ttm] = str(quote['pe_ttm']) data[CN_dividend] = str(quote['dividend']) data[CN_dividend_yield] = str(quote['dividend_yield']) data[CN_high52w] = str(quote['high52w']) data[CN_low52w] = str(quote['low52w']) #util.print_json(data, '11') return data
def average_gradients(tower_grads): average_grads = [] for grad_and_vars in zip(*tower_grads): # Note that each grad_and_vars looks like the following: # ((grad0_gpu0, var0_gpu0), ... , (grad0_gpuN, var0_gpuN)) grads = [] for g, _ in grad_and_vars: # Add 0 dimension to the gradients to represent the tower. var_list = [g, _] ##adl for idx, x in enumerate(var_list): ##adl var_names = "g, _".split(",") ##adl cur_name = var_names[idx] ##adl print("-------print " + cur_name + "------") ##adl import alog ##adl from pprint import pprint ##adl alog.info(cur_name) ##adl print(">>> type(x) = ", type(x)) ##adl if hasattr(x, "shape"): ##adl print(">>> " + cur_name + ".shape", x.shape) ##adl if type(x) is list: ##adl print(">>> len(" + cur_name + ") = ", len(x)) ##adl pprint(x) ##adl else: ##adl pprint(x) ##adl pass ##adl print("------------------------\n") ##adl expanded_g = tf.expand_dims(g, 0) # Append on a 'tower' dimension which we will average over below. grads.append(expanded_g) # Average over the 'tower' dimension. grad = tf.concat(axis=0, values=grads) grad = tf.reduce_mean(grad, 0) # Keep in mind that the Variables are redundant because they are shared # across towers. So .. we will just return the first tower's pointer to # the Variable. v = grad_and_vars[0][1] grad_and_var = (grad, v) average_grads.append(grad_and_var) return average_grads
def gen_url(self, test): if test: base_url = 'https://test.deribit.com/ws/api/v2' else: base_url = 'https://www.deribit.com/api/v2' url_parts = list(urlparse(base_url)) # query_string = '' # if self.heartbeat: # query_string = '?heartbeat=true' # url = "wss://{}/realtime{}".format(url_parts[1], query_string) url = "wss://{}/ws/api/v2".format(url_parts[1]) alog.info(url) return url
def yieldfrom_syntax(): start = time.time() #loop = asyncio.get_event_loop() loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) # yield from syntax session = aiohttp.ClientSession(loop=loop) tasks = [ #loop.create_task(fetch_page(session, f"{url}/foo")), #loop.create_task(fetch_page(session, f"{url}/bar")) asyncio.ensure_future(fetch_page(session, f"{url}{u}")) for u in URLS ] loop.run_until_complete(asyncio.wait(tasks)) loop.close() alog.info( f'yieldfrom syntax Coroutines took {time.time() - start:.1f} sec')
async def produce(loop, queue, n, fake_msg): """ Create fake rs232 data into queue, it's blocking :param queue :param n """ index = 0 try: async for data in pv200(n, fake_msg): # produce rs232 data alog.info(f"producing {index}/{n}, id:{id(data)}, {data.split(',')[2]}") index += 1 # simulate i/o operation data await asyncio.sleep(random.random()) # put into list await queue.put(data) except Exception as e: alog.info(e)
def on_message(self, message): """Handler for parsing WS messages.""" response = json.loads(message) # if response if "id" in response: if "result" in response: result = response["result"] elif "error" in response: alog.error(response["error"]) return else: return id = response["id"] if id == self._id: if self._method == "public/auth": self._authInfo["access_token"] = result["access_token"] self._authInfo["expires_in"] = result["expires_in"] self._authInfo["refresh_token"] = result["refresh_token"] self._authInfo["scope"] = result["scope"] self._authInfo["token_type"] = result["token_type"] alog.info("access_token: {}".format(self._authInfo["access_token"])) self.emit('subscribe') # wst = threading.Thread(target=self.__on_expires_in) # wst.daemon = True # wst.start() return elif self._method == "public/subscribe" or self._method == "private/subscribe": self.emit('subscribed', result) return alog.info(message) if "method" in response: method = response["method"] if method == "subscription": params = response["params"] channel = params["channel"] #if "user.orders" in channel: self.emit('orders', channel, params["data"])
def header(self): """Return auth headers. Will use API Keys if present in settings.""" auth_header = [] if self.should_auth: alog.info("Authenticating with API Key.") # To auth to the WS using an API key, we generate a signature # of a nonce and the WS API endpoint. alog.debug(settings.BITMEX_API_KEY) nonce = generate_nonce() api_signature = generate_signature(settings.BITMEX_API_SECRET, 'GET', '/realtime', nonce, '') auth_header = [ "api-nonce: " + str(nonce), "api-signature: " + api_signature, "api-key:" + settings.BITMEX_API_KEY ] return auth_header
async def produce_rs232(queue, n=None, **kwargs): """ Creat the real rs232 data into queue, is non-blocking :param queue :param n :param **kwargs """ reader, writer = await serial_asyncio.open_serial_connection(url=com, **kwargs) buffers = '' index = 0 while True: data = await reader.readline() buffers += data.decode('utf-8') alog.info(buffers) buf = buffers.split('\r')[-2].split(',') if len(buf) < rs232_len: alog.info(f"Drop not illeage {len(buf)}") buffers = '' elif '\r' in buffers: buf = buffers.split('\r') last_received, buffers = buffers.split('\r')[-2:] buffers = buffers.strip() # clean \n alog.info(f"producing {index}/{n}, id:{id(last_received)}, {last_received.split(',')[2]}") index += 1 await queue.put(last_received)
def POST(self): try: alog.debug('hello', 'wx_handle', 'WXHandle::POST') webData = web.data() alog.info("Handle Post webdata is " + str(webData), 'wx_handle', 'WXHandle::POST') recMsg = wx_receive.parse_xml(webData) if isinstance(recMsg, wx_receive.Msg) and recMsg.MsgType == 'text': toUser = recMsg.FromUserName fromUser = recMsg.ToUserName alog.debug('handle_type:'+recMsg.handle_type, 'WXHandle::POST') if recMsg.handle_type == 'goodprice': calc = GoodPriceCalc() price_data = calc.calc(recMsg.Content) content = calc.to_text(price_data, '\n') elif 'xueqiu_token' == recMsg.handle_type: xueqiu.set_token('xq_a_token='+recMsg.Content) content = u'设置雪球token成功:' + xueqiu.get_token() else: content = u'不支持命令:' + recMsg.Content replyMsg = wx_reply.TextMsg(toUser, fromUser, content) s = replyMsg.send() alog.debug(s, 'wx_handle', 'POST reply') return s else: alog.info('not support', 'WXHandle::POST', u'recMsg.MsgType') return "success" except Exception as Argment: alog.info(Argment, 'WXHandle::POST', u'error') return Argment
def get_auth(self): '''Return auth headers. Will use API Keys if present in settings.''' alog.debug('shouldAuth: %s' % self.shouldAuth) if self.shouldAuth: alog.info("Authenticating with API Key.") # To auth to the WS using an API key, we generate a signature # of a nonce and the WS API endpoint. alog.debug(settings.GDAX_API_KEY) nonce = generate_nonce() api_signature = generate_signature( settings.GDAX_API_SECRET, 'GET', '/realtime', nonce, '') auth = [ "api-nonce: " + str(nonce), "api-signature: " + api_signature, "api-key:" + settings.GDAX_API_KEY ] alog.debug(auth) return auth else: return []
def goodprice_wx(): xmldata = touni(request._get_body_string()) alog.info(xmldata, 'goodprice_wx') #try: if True: recMsg = wx_receive.parse_xml(xmldata) if isinstance(recMsg, wx_receive.Msg) and recMsg.MsgType == 'text': toUser = recMsg.FromUserName fromUser = recMsg.ToUserName recv_content = touni(recMsg.Content) calc = GoodPriceCalc() alog.debug(recv_content, 'goodprice_wx', 'recMsg.Content') price_data = calc.calc(recv_content) content = calc.to_text(price_data, '<br>') replyMsg = wx_reply.TextMsg(toUser, fromUser, recv_content) s = replyMsg.send() alog.debug(s, 'goodprice_wx', 'reply') return s else: alog.info(s, 'goodprice_wx', u'不支持') return "success" """ except Exception as Argment:
async def consume(queue): """ Post rs232 data into server :param queue """ async with aiohttp.ClientSession() as session: try: while True: # wait for an data from producer data = await queue.get() # process the item alog.info(f"consuming id:{id(data)}") status, result = await async_post(session, url_boards, data) if status != 201: alog.error(f"error: id:{id(data)}, {status}, {result}") await queue.put(data) # simulate i/o operation using sleep await asyncio.sleep(random.random()) # Notify the queue that the item has been processed queue.task_done() except Exception as e: alog.info(f"Finish: {e}.")
def connect_websocket(self): """Connect to the websocket in a thread.""" alog.debug("### Connecting Websocket ###") # setup websocket.run_forever arguments wsRunArgs = { 'sslopt': {"cert_reqs": ssl.CERT_NONE} } if self.heartbeatEnabled: wsRunArgs['ping_timeout'] = 20 wsRunArgs['ping_interval'] = 60 alog.debug("websocket.run_forever: %s" % (wsRunArgs)) # Run the websocket on another thread and enable heartbeat self.wst = threading.Thread( target=lambda: self.websocket_run_forever(wsRunArgs) ) self.wst.daemon = True self.wst.start() alog.info("### Started thread") self.wait_for_connection()
def calc(self, codes, use_cache=True): code_list = codes.split(',') if len(codes) < 5 or len(code_list) < 1: return {'code': {'error': 'codes is error:' + codes}} dts = datetime.datetime.now().strftime('%Y%m%d') cachefile = '.cache/' + codes + '-' + dts + '.json' if use_cache is True: try: redata = util.load_json(cachefile) alog.info('hit cache:' + codes + ' cache file:' + cachefile, 'good_price', 'calc') return redata except: pass self.chinabond10 = DataGether.chinabond10() self.market_pe = DataGether.szpe() market_data = { CN_chinabond10: "%.3f%%" % (self.chinabond10 * 100), CN_szpe: str(self.market_pe) } alog.info( 'market_pe:' + str(self.market_pe) + ' chinabond10:' + str(self.chinabond10), 'good_price', 'calc') redata = {} for code in code_list: symbol = util.code2symbal(code) alog.info(symbol + '...', 'good_price', 'calc') if True: try: #if True: ttm_data = DataGether.quote_ttm(symbol) #util.print_json(ttm_data) itemdata = {} itemdata['a'] = ttm_data itemdata['b'] = market_data itemdata['c'] = self.calc_good_price(symbol, ttm_data) redata[symbol] = itemdata #else: except Exception as e1: print("[req_quote] error:" + str(e1)) itemdata = {} itemdata['a'] = {CN_symbol: symbol} itemdata['b'] = market_data itemdata['c'] = {} redata[symbol] = itemdata continue else: ttm_data = DataGether.quote_ttm(symbol) #util.print_json(ttm_data) itemdata = self.calc_good_price(symbol, ttm_data) redata[symbol] = itemdata time.sleep(2) util.dump_json(cachefile, redata) return redata
def header(self): """Return auth headers. Will use API Keys if present in settings.""" auth_header = [] alog.info(f'### should auth {self.should_auth} ###') if self.should_auth: alog.info("Authenticating with API Key.") # To auth to the WS using an API key, we generate a signature # of a nonce and the WS API endpoint. alog.info((self.api_key, self.api_secret)) nonce = generate_nonce() api_signature = generate_signature(self.api_secret, 'GET', '/realtime', nonce, '') auth_header = [ "api-nonce: " + str(nonce), "api-signature: " + api_signature, "api-key:" + self.api_key ] alog.info(alog.pformat(auth_header)) return auth_header
def record_history(self, user, retdata, symbal): alog.debug(user + ' ' + symbal, 'good_price::record_history') util.print_json(retdata) if len(user) <= 0: alog.info("user is empty", 'good_price::record_history') return dts = datetime.datetime.now().strftime('%Y-%m-%d') if symbal in retdata.keys(): try: itemdata = retdata[symbal]['basic'] ctx = '\t'.join([ user, dts, symbal, itemdata[CN_name], itemdata[CN_current], itemdata[CN_pe_ttm] ]) util.dump_str("data/user/goodprice_history_%s.txt" % (user), ctx + '\n', mode='a') except: alog.info("histroy data error", 'good_price::record_history') else: alog.info(symbal + "not in retdata", 'good_price::record_history') return
def on_close(self): alog.info('Websocket Closed')
def on_close(self, info): alog.info('Websocket Closed: ' + str(info))
def on_open(self, *kwargs): alog.info("Websocket opened") alog.debug("Websocket Opened. args: " + str(kwargs)) self.emit('open')