def get_quotation(): page_num = 1 page_size = 10000 fields = "f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f26,f22,f33,f11,f62,f128,f136,f115,f152" # k = ['f43', 'f44', 'f45', 'f46', 'f60', 'f71', 'f47', 'f48', 'f49', 'f161', 'f50', 'f55', 'f59', 'f84', 'f86', # 'f92', 'f116', 'f126', 'f152', 'f167', 'f164', 'f168', 'f169', 'f170', 'f171', 'f172'] # fields = ",".join(keys) start_url = "http://{h}.push2.eastmoney.com/api/qt/clist/get?" \ "cb=jQuery1124012264592664044649_1565663112714&pn={pn}&pz={pz}&po=1&np=1" \ "&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=m:105,m:106,m:107" \ "&fields={fields}" \ "&_={t}" url = start_url.format(h=random.randint(1, 100), pn=page_num, pz=page_size, t=str(time.time()).replace(".", "")[:13], fields=fields) resp = s.get(url, headers=get_headers()) data = json_loads(resp.text).get("data") quotations = handler_quotation(data.get("diff")) save_data(quotations, filename=filename) total = int(data.get("total")) logger.info("获取行情数据 {} 条".format(total))
def penqueue(uid, type, priority, body, mdb=None): """ 追加一条准备状态的队列,同一个用户会不断的追加 @body as dict, 追加队列的body必须时一个词典,格式如下 { 'email': '接收的邮箱地址', 'values': '追加值,必须是一个列表' } """ assert (isinstance(body, dict)) assert ('email' in body) assert ('values' in body and isinstance(body['values'], list)) # 获取用户处于准备中的队列记录 q = Queue.get_user_status(uid, type, 'P', mdb=mdb) if q is None: # 初始化一条队列记录 q = web.utils.storage(id=0, user_id=uid, type=type, priority=priority, status='P', body=body) else: q.body = utils.json_loads(q.body) q.body['email'] = body['email'] q.body['values'].extend(body['values']) q.body = utils.json_dumps(q.body) # 覆盖更新之前的队列记录 return Queue.replace(q, mdb=mdb)
async def hsm_status(self, h=None): # refresh HSM status b4 = STATUS.hsm.get('active', False) try: b4_nlc = STATUS.hsm.get('next_local_code') h = h or (await self.send_recv(CCProtocolPacker.hsm_status())) STATUS.hsm = h = json_loads(h) STATUS.notify_watchers() except MissingColdcard: h = {} if h.get('next_local_code') and STATUS.psbt_hash: if b4_nlc != h.next_local_code: STATUS.local_code = calc_local_pincode( a2b_hex(STATUS.psbt_hash), h.next_local_code) else: # won't be required STATUS.local_code = None # has it just transitioned into HSM mode? if STATUS.connected and STATUS.hsm.active and not b4: await self.activated_hsm() return STATUS.hsm
async def hsm_status(self, h=None): # refresh HSM status b4 = STATUS.hsm.get('active', False) try: b4_nlc = STATUS.hsm.get('next_local_code') h = h or (await self.send_recv(CCProtocolPacker.hsm_status())) STATUS.hsm = h = json_loads(h) STATUS.notify_watchers() except MissingColdcard: h = {} if h.get('next_local_code') and STATUS.psbt_hash: if b4_nlc != h.next_local_code: STATUS.local_code = calc_local_pincode( a2b_hex(STATUS.psbt_hash), h.next_local_code) else: # won't be required STATUS.local_code = None if ('summary' in h) and h.summary and not BP.get( 'priv_over_ux') and not BP.get('summary'): logging.info("Captured CC's summary of the policy") BP['summary'] = h.summary BP.save() # has it just transitioned into HSM mode? if STATUS.connected and STATUS.hsm.active and not b4: await self.activated_hsm() return STATUS.hsm
def update_key_from_api(self): content = utils.get_content(conf.GET_KEY_API) keys_record = utils.json_loads(content) if not keys_record: self.log.error('get keys from api failed %s' % keys_record) return False dao.upsert_wechat_key(self.mydqldb, json.dumps(keys_record)) return True
def test_login(self): uid = 1 online_model.delete(uid) session_key = online_model.add(uid) data = [SERVICE_LOGIN, CMD_LOGIN, "1", VERSION_SUB, CONFIG.game_id, 0, session_key] self.send(data) stream = self.__socket.recv(1024) result = utils.json_loads(stream) self.assertTrue(not result) data = [SERVICE_LOGIN, CMD_LOGIN, VERSION_MAIN, VERSION_SUB, CONFIG.game_id, 0, session_key] self.send(data) stream = self.__socket.recv(1024) result = utils.json_loads(stream) self.assertTrue(len(result) > 0) self.assertTrue(result[2] == OK)
def get_videos(url, cw=None): print_ = get_print(cw) info = {} user_id = re.find(r'twitch.tv/([^/?]+)', url, err='no user_id') print(user_id) session = Session() r = session.get(url) s = cut_pair(re.find(r'headers *: *({.*)', r.text, err='no headers')) print(s) headers = json_loads(s) payload = [{ 'operationName': 'ClipsCards__User', 'variables': { 'login': user_id, 'limit': 20, 'criteria': { 'filter': 'ALL_TIME' } }, 'extensions': { 'persistedQuery': { 'version': 1, 'sha256Hash': 'b73ad2bfaecfd30a9e6c28fada15bd97032c83ec77a0440766a56fe0bd632777' } }, }] videos = [] cursor = None cursor_new = None while True: if cursor: payload[0]['variables']['cursor'] = cursor r = session.post('https://gql.twitch.tv/gql', json=payload, headers=headers) #print(r) data = r.json() for edge in data[0]['data']['user']['clips']['edges']: url_video = edge['node']['url'] info['name'] = edge['node']['broadcaster']['displayName'] video = Video(url_video) video.id = int(edge['node']['id']) videos.append(video) cursor_new = edge['cursor'] print_('videos: {} / cursor: {}'.format(len(videos), cursor)) if cursor == cursor_new: print_('same cursor') break if cursor_new is None: break cursor = cursor_new if not videos: raise Exception('no videos') info['videos'] = sorted(videos, key=lambda video: video.id, reverse=True) return info
def deserialize_overrides(override_list): def splitter(string): if isinstance(string, list): pair = string # already split into pairs, return what we have return pair ensure('|' in string, "override key and value must be seperated by a pipe '|'") first, rest = string.split('|', 1) ensure(rest.strip(), "a value must be provided. use 'null' without quotes to use an empty value") return first, rest pairs = lmap(splitter, override_list) return OrderedDict([(key, utils.json_loads(val)) for key, val in pairs])
def get_news_list(self, biz, uin, key, pass_ticket='', frommsgid=''): # 获取公众号文章列表 msgsend_url = "https://mp.weixin.qq.com/mp/profile_ext?action=home&__biz=%s&scene=124&" \ "devicetype=android-22&version=26050732&lang=zh_CN&nettype=" \ "WIFI&a8scene=3&pass_ticket=%s&wx_header=1 " %(biz, pass_ticket) try: if not hasattr(self, 'cookies'): self.cookies = {} if not hasattr(self, 'wxparams'): self.wxparams = {} time.sleep(random.randint(5, 10)) # print cookies self.headers['X-WECHAT-KEY'] = key self.headers['X-WECHAT-UIN'] = urllib.quote(uin) content = utils.get_content(msgsend_url, cookies=self.cookies, headers=self.headers, verify=False) # set cookies. if not content: self.log.warn( 'get new list failed, biz=%s, errmsg=response error %s ' % (biz, uin)) return [] news_json, msg_token = self.parse_article_list(content) if msg_token: self.wxparams['appmsg_token'] = msg_token if not news_json: r = re.search(RE_ERRMSG, content) self.log.warn(content) self.log.warn( "parse no article list biz %s, uin %s, errmsg: %s" % (biz, uin, r.group(1) if r else '')) # 如果出现页面无法打开, api中的uin不能使用。 if '<h2 class="weui_msg_title">页面无法打开</h2>' in content \ or '<p>请在微信客户端打开链接。</p>' in content \ or '<h2 class="weui_msg_title">操作频繁,请稍后再试</h2>' in content: return [] news_json = news_json.strip() if news_json.startswith("'"): news_json = news_json[1:-1] news_json = utils.json_loads(news_json) if not news_json: self.log.warn('failed to loads json, biz=%s, uin=%s' % (biz, uin)) return [] self.log.info("crawl news list success len:%s" % len(news_json.get('list', []))) return news_json.get('list', []) except Exception as e: self.log.error('get news list Exception:%s, biz:%s, uin:%s' % (traceback.format_exc(), biz, uin)) return []
async def rx_handler(ses, ws, orig_request): # Block on receive, handle each message as it comes in. # see pp/aiohttp/client_ws.py async def tx_resp(_ws=ws, **resp): logging.debug(f"Send resp: {resp}") await _ws.send_str(json_dumps(resp)) async for msg in ws: if msg.type != web.WSMsgType.TEXT: raise TypeError('expected text') try: assert len(msg.data) < 20000 req = json_loads(msg.data) if '_ping' in req: # connection keep alive, simple await tx_resp(_pong=1) continue # Caution: lots of sensitive values here XXX #logging.info("WS api data: %r" % req) except Exception as e: logging.critical("Junk data on WS", exc_info=1) break # the connection # do something with the request failed = True try: await ws_api_handler(ses, tx_resp, req, orig_request) failed = False except SystemExit: raise except KeyboardInterrupt: break except HTMLErrorMsg as exc: # pre-formated text for display msg = exc.args[0] except RuntimeError as exc: # covers CCProtoError and similar msg = str(exc) or str(type(exc).__name__) except BaseException as exc: logging.exception("API fail: req=%r" % req) msg = str(exc) or str(type(exc).__name__) if failed: # standard error response await tx_resp(show_modal=True, html=jinja2.escape(msg), selector='.js-api-fail')
def get_index_data(): url = "http://58.push2.eastmoney.com/api/qt/clist/get?" \ "cb=jQuery1124005752417505401741_1565678085560&pn=1&pz=20&po=1&np=1" \ "&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3" \ "&fs=i:100.NDX,i:100.DJIA,i:100.SPX" \ "&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f26,f22,f33,f11,f62,f128,f136,f115,f152,f124,f107" \ "&_=1565678085561" resp = s.get(url, headers=get_headers()) data = json_loads(resp.text).get("data") index_data = handler_index(data.get("diff")) save_data(index_data, filename=filename) total = data.get("total") logger.info("获取数据 {} 条".format(total))
def deserialize_overrides(override_list): def splitter(string): if isinstance(string, list): pair = string # already split into pairs, return what we have return pair ensure('|' in string, "override key and value must be seperated by a pipe '|'", ValueError) first, rest = string.split('|', 1) ensure( rest.strip(), "a value must be provided. use 'null' without quotes to use an empty value", ValueError) return first, rest pairs = lmap(splitter, override_list) return OrderedDict([(key, utils.json_loads(val)) for key, val in pairs])
def json(self): """ If the ``Content-Type`` header is ``application/json`` or ``application/json-rpc``, this property holds the parsed content of the request body. Only requests smaller than :attr:`MEMFILE_MAX` are processed to avoid memory exhaustion. Invalid JSON raises a 400 error response. """ ctype = self.environ.get('CONTENT_TYPE', '').lower().split(';')[0] if ctype in ('application/json', 'application/json-rpc'): b = self._get_body_string() if not b: return None try: return json_loads(b) except (ValueError, TypeError): raise HTTPError(400, 'Invalid JSON') return None
def get_wechat_key(self): if self.wechat_key and int( time.time()) - self.last_wechat_key_time < conf.WECHAT_KEY_TTL: return self.wechat_key if int(self.idx) == 0: self.update_key_from_api() wechat_record = dao.get_wechat_key(self.mydqldb) if int(time.time()) - utils.get_timestamp( str(wechat_record.last_update_time)) > conf.WECHAT_KEY_TTL: self.log.info('sleep 10 seconds indx: %s' % self.idx) time.sleep(10) return self.get_wechat_key() if not wechat_record: self.log.error('get wechat key error! %s ' % wechat_record) return None response_info = utils.json_loads(wechat_record.content) wechat_keys = response_info.get('data') self.wechat_key = wechat_keys[self.idx] return self.wechat_key
def broadcast_txn(txn): # take bytes and get them shared over P2P to the world # - raise w/ text about what happened if it fails # - limited docs: <https://github.com/Blockstream/esplora/blob/master/API.md> ses = requests.session() ses.proxies = dict(http=settings.TOR_SOCKS) ses.headers.clear() # hide user-agent url = settings.EXPLORA url += '/api/tx' if not STATUS.is_testnet else '/testnet/api/tx' assert '.onion/' in url, 'dude, your privacy' logging.warning(f"Sending txn via: {url}") resp = ses.post(url, data=b2a_hex(txn).decode('ascii')) msg = resp.text if not resp.ok: # content is like: # sendrawtransaction RPC error: {"code":-22,"message":"TX decode failed"} # which is a text thing, including some JSON from bitcoind? if '"message":' in msg: try: prefix, rest = msg.split(': ', 1) j = json_loads(rest) if prefix == 'sendrawtransaction RPC error': msg = j.message else: msg = prefix + ': ' + j.message except: pass msg = f"Transaction broadcast FAILED: {msg}" logging.error(msg) return msg # untested msg = f"Transaction broadcast success: {msg}" logging.info(msg) return msg
def open(self, key): # Given a private key (via storage locker) open a Nacl secret box # and use that for the data. self.set_secret(key) try: with open(self.filename, 'rb') as fp: d = self.box.decrypt(fp.read()) d = json_loads(d) except FileNotFoundError: logging.info("%s: not found (probably fine)" % self.filename) return True self.update(d) # copy a setting to status (XXX feels wrong) from status import STATUS STATUS.tor_enabled = self.get('tor_enabled', False) logging.info(f"Got bunker settings from: {self.filename}")
def on_line_received(self, p, line): if not line or len(line) < 2: p.send('data empty') p.close() return ret = utils.json_loads(line) if not ret or len(ret) < 2: utils.log('receive data error: ' + line, 'receive_error.log') p.send('decode fail') p.close() return if not self.is_legal_command(ret): print 'command is illegal', line p.send('cmd illegal') p.close() return result = self.distribute(p, ret) if result: p.send(result)
def get(self, key): value = self.client.get(key) if value: return utils.json_loads(value)
async def ws_api_handler(ses, send_json, req, orig_request): # handle_api # # Handle incoming requests over websocket; send back results. # req = already json parsed request coming in # send_json() = means to send the response back # action = req.action args = getattr(req, 'args', None) #logging.warn("API action=%s (%r)" % (action, args)) # MAJOR info leak XXX logging.debug(f"API action={action}") if action == '_connected': logging.info("Websocket connected: %r" % args) # can send special state update at this point, depending on the page elif action == 'start_hsm_btn': await Connection().hsm_start() await send_json(show_flash_msg=APPROVE_CTA) elif action == 'delete_user': name, = args assert 1 <= len(name) <= MAX_USERNAME_LEN, "bad username length" await Connection().delete_user(name.encode('utf8')) # assume it worked, so UX updates right away try: STATUS.hsm.users.remove(name) except ValueError: pass STATUS.notify_watchers() elif action == 'create_user': name, authmode, new_pw = args assert 1 <= len(name) <= MAX_USERNAME_LEN, "bad username length" assert ',' not in name, "no commas in names" if authmode == 'totp': mode = USER_AUTH_TOTP | USER_AUTH_SHOW_QR new_pw = '' elif authmode == 'rand_pw': mode = USER_AUTH_HMAC | USER_AUTH_SHOW_QR new_pw = '' elif authmode == 'give_pw': mode = USER_AUTH_HMAC else: raise ValueError(authmode) await Connection().create_user(name.encode('utf8'), mode, new_pw) # assume it worked, so UX updates right away try: STATUS.hsm.users = list(set(STATUS.hsm.users + [name])) except ValueError: pass STATUS.notify_watchers() elif action == 'submit_policy': # get some JSON w/ everything the user entered. p, save_copy = args proposed = policy.web_cleanup(json_loads(p)) policy.update_sl(proposed) await Connection().hsm_start(proposed) STATUS.notify_watchers() await send_json(show_flash_msg=APPROVE_CTA) if save_copy: d = policy.desensitize(proposed) await send_json(local_download=dict(data=json_dumps(d, indent=2), filename=f'hsm-policy-{STATUS.xfp}.json.txt')) elif action == 'download_policy': proposed = policy.web_cleanup(json_loads(args[0])) await send_json(local_download=dict(data=json_dumps(proposed, indent=2), filename=f'hsm-policy-{STATUS.xfp}.json.txt')) elif action == 'import_policy': # they are uploading a JSON capture, but need values we can load in Vue proposed = args[0] cooked = policy.web_cookup(proposed) await send_json(vue_app_cb=dict(update_policy=cooked), show_flash_msg="Policy file imported.") elif action == 'pick_onion_addr': from torsion import TOR addr, pk = await TOR.pick_onion_addr() await send_json(vue_app_cb=dict(new_onion_addr=[addr, pk])) elif action == 'pick_master_pw': pw = b64encode(os.urandom(12)).decode('ascii') pw = pw.replace('/', 'S').replace('+', 'p') assert '=' not in pw await send_json(vue_app_cb=dict(new_master_pw=pw)) elif action == 'new_bunker_config': from torsion import TOR # save and apply config values nv = json_loads(args[0]) assert 4 <= len(nv.master_pw) < 200, "Master password must be at least 4 chars long" # copy in simple stuff for fn in [ 'tor_enabled', 'master_pw', 'easy_captcha', 'allow_reboots']: if fn in nv: BP[fn] = nv[fn] # update onion stuff only if PK is known (ie. they changed it) if nv.get('onion_pk', False) or False: for fn in [ 'onion_addr', 'onion_pk']: if fn in nv: BP[fn] = nv[fn] BP.save() await send_json(show_flash_msg="Bunker settings encrypted and saved to disk.") STATUS.tor_enabled = BP['tor_enabled'] STATUS.notify_watchers() if not BP['tor_enabled']: await TOR.stop_tunnel() elif BP.get('onion_pk') and not (STATUS.force_local_mode or STATUS.setup_mode) \ and TOR.get_current_addr() != BP.get('onion_addr'): # disconnect/reconnect await TOR.start_tunnel() elif action == 'sign_message': # sign a short text message # - lots more checking could be done here, but CC does it anyway msg_text, path, addr_fmt = args addr_fmt = AF_P2WPKH if addr_fmt != 'classic' else AF_CLASSIC try: sig, addr = await Connection().sign_text_msg(msg_text, path, addr_fmt) except: # get the spinner to stop: error msg will be "refused by policy" typically await send_json(vue_app_cb=dict(msg_signing_result='(failed)')) raise sig = b64encode(sig).decode('ascii').replace('\n', '') await send_json(vue_app_cb=dict(msg_signing_result=f'{sig}\n{addr}')) elif action == 'upload_psbt': # receiving a PSBT for signing size, digest, contents = args psbt = b64decode(contents) assert len(psbt) == size, "truncated/padded in transit" assert sha256(psbt).hexdigest() == digest, "corrupted in transit" STATUS.import_psbt(psbt) STATUS.notify_watchers() elif action == 'clear_psbt': STATUS.clear_psbt() STATUS.notify_watchers() elif action == 'preview_psbt': STATUS.psbt_preview = 'Wait...' STATUS.notify_watchers() try: txt = await Connection().sign_psbt(STATUS._pending_psbt, flags=STXN_VISUALIZE) txt = txt.decode('ascii') # force some line splits, especially for bech32, 32-byte values (p2wsh) probs = re.findall(r'([a-zA-Z0-9]{36,})', txt) for p in probs: txt = txt.replace(p, p[0:30] + '\u22ef\n\u22ef' + p[30:]) STATUS.psbt_preview = txt except: # like if CC doesn't like the keys, whatever .. STATUS.psbt_preview = None raise finally: STATUS.notify_watchers() elif action == 'auth_set_name': idx, name = args assert 0 <= len(name) <= MAX_USERNAME_LEN assert 0 <= idx < len(STATUS.pending_auth) STATUS.pending_auth[idx].name = name STATUS.notify_watchers() elif action == 'auth_offer_guess': idx, ts, guess = args assert 0 <= idx < len(STATUS.pending_auth) STATUS.pending_auth[idx].totp = ts STATUS.pending_auth[idx].has_guess = 'x'*len(guess) STATUS._auth_guess[idx] = guess STATUS.notify_watchers() elif action == 'submit_psbt': # they want to sign it now expect_hash, send_immediately, finalize, wants_dl = args assert expect_hash == STATUS.psbt_hash, "hash mismatch" if send_immediately: assert finalize, "must finalize b4 send" logging.info("Starting to sign...") STATUS.busy_signing = True STATUS.notify_watchers() try: dev = Connection() # do auth steps first (no feedback given) for pa, guess in zip(STATUS.pending_auth, STATUS._auth_guess): if pa.name and guess: await dev.user_auth(pa.name, guess, int(pa.totp), a2b_hex(STATUS.psbt_hash)) STATUS.reset_pending_auth() try: result = await dev.sign_psbt(STATUS._pending_psbt, finalize=finalize) logging.info("Done signing") msg = "Transaction signed." if send_immediately: msg += '<br><br>' + broadcast_txn(result) await send_json(show_modal=True, html=Markup(msg), selector='.js-api-success') result = (b2a_hex(result) if finalize else b64encode(result)).decode('ascii') fname = 'transaction.txt' if finalize else ('signed-%s.psbt'%STATUS.psbt_hash[-6:]) if wants_dl: await send_json(local_download=dict(data=result, filename=fname, is_b64=(not finalize))) await dev.hsm_status() except CCUserRefused: logging.error("Coldcard refused to sign txn") await dev.hsm_status() r = STATUS.hsm.get('last_refusal', None) if not r: raise HTMLErroMsg('Refused by local user.') else: raise HTMLErrorMsg(f"Rejected by Coldcard.<br><br>{r}") finally: STATUS.busy_signing = False STATUS.notify_watchers() elif action == 'shutdown_bunker': await send_json(show_flash_msg="Bunker is shutdown.") await asyncio.sleep(0.25) logging.warn("User-initiated shutdown") asyncio.get_running_loop().stop() sys.exit(0) elif action == 'leave_setup_mode': # During setup process, they want to go Tor mode; which I consider leaving # setup mode ... in particular, logins are required. # - button label is "Start Tor" tho ... so user doesn't see it that way assert STATUS.setup_mode, 'not in setup mode?' assert BP['tor_enabled'], 'Tor not enabled (need to save?)' addr = BP['onion_addr'] assert addr and '.onion' in addr, "Bad address?" STATUS.setup_mode = False await send_json(show_flash_msg="Tor hidden service has been enabled. " "It may take a few minutes for the website to become available") STATUS.notify_watchers() from torsion import TOR logging.info(f"Starting hidden service: %s" % addr) asyncio.create_task(TOR.start_tunnel()) elif action == 'logout_everyone': # useful for running battles... # - changes crypto key for cookies, so they are all invalid immediately. from aiohttp_session.nacl_storage import NaClCookieStorage import nacl logging.warning("Logout of everyone!") # reset all session cookies storage = orig_request.get('aiohttp_session_storage') assert isinstance(storage, NaClCookieStorage) storage._secretbox = nacl.secret.SecretBox(os.urandom(32)) # kick everyone off (bonus step) for w in web_sockets: try: await send_json(redirect='/logout', _ws=w) await w.close() except: pass else: raise NotImplementedError(action)