def get_sina_Market_json(market='sh', showtime=True, num='1000', retry_count=3, pause=0.001): start_t = time.time() # url="http://vip.stock.finance.sina.com.cn/quotes_service/api/json_v2.php/Market_Center.getHQNodeData?page=1&num=50&sort=changepercent&asc=0&node=sh_a&symbol=" # SINA_REAL_PRICE_DD = '%s%s/quotes_service/api/json_v2.php/Market_Center.getHQNodeData?page=1&num=%s&sort=changepercent&asc=0&node=%s&symbol=%s' """ 一次性获取最近一个日交易日所有股票的交易数据 return ------- DataFrame 属性:代码,名称,涨跌幅,现价,开盘价,最高价,最低价,最日收盘价,成交量,换手率 """ # ct._write_head() if market=='all': url_list=[] # for m in ct.SINA_Market_KEY.values(): for m in ['sh_a','sz_a']: list=_get_sina_Market_url(m, num=num) for l in list:url_list.append(l) # print url_list else: url_list=_get_sina_Market_url(ct.SINA_Market_KEY[market], num=num) # print url_list log.debug("Market_jsonURL: %s" % url_list[0]) # print url_list # print "url:",url_list df = pd.DataFrame() # data['code'] = symbol # df = df.append(data, ignore_index=True) # results = cct.to_mp_run(_parsing_Market_price_json, url_list) results = cct.to_asyncio_run(url_list, _parsing_Market_price_json) if len(results)>0: df = df.append(results, ignore_index=True) # df['volume']= df['volume'].apply(lambda x:x/100) df['ratio']=df['ratio'].apply(lambda x:round(x,1)) df['percent']=df['percent'].apply(lambda x:round(x,1)) df=df.drop_duplicates() # print df[:1] # for url in url_list: # # print url # data = _parsing_Market_price_json(url) # # print data[:1] # df = df.append(data, ignore_index=True) # # break if df is not None: # for i in range(2, ct.PAGE_NUM[0]): # newdf = _parsing_dayprice_json(i) # df = df.append(newdf, ignore_index=True) # print len(df.index) if showtime: print ( "Market-df:%s %s" % (format((time.time() - start_t), '.1f'), len(df))), # print type(df) return df else: if showtime:print ("no data Market-df:%s" % (format((time.time() - start_t), '.2f'))) return []
def get_sina_all_json_dd(vol='0', type='3', num='10000', retry_count=3, pause=0.001): start_t = time.time() # url="http://vip.stock.finance.sina.com.cn/quotes_service/api/json_v2.php/Market_Center.getHQNodeData?page=1&num=50&sort=changepercent&asc=0&node=sh_a&symbol=" # SINA_REAL_PRICE_DD = '%s%s/quotes_service/api/json_v2.php/Market_Center.getHQNodeData?page=1&num=%s&sort=changepercent&asc=0&node=%s&symbol=%s' """ 一次性获取最近一个日交易日所有股票的交易数据 return ------- DataFrame 属性:代码,名称,涨跌幅,现价,开盘价,最高价,最低价,最日收盘价,成交量,换手率 """ # ct._write_head() url_list = _get_sina_json_dd_url(vol, type, num) # print url_list df = pd.DataFrame() # data['code'] = symbol # df = df.append(data, ignore_index=True) if len(url_list)>0: log.info("json_dd_url:%s"%url_list[0]) data = cct.to_asyncio_run(url_list, _parsing_sina_dd_price_json) # data = cct.to_mp_run(_parsing_sina_dd_price_json, url_list) # data = cct.to_mp_run_async(_parsing_sina_dd_price_json, url_list) # if len(url_list)>cct.get_cpu_count(): # divs=cct.get_cpu_count() # else: # divs=len(url_list) # # if len(url_list)>=divs: # print len(url_list), # dl=cct.get_div_list(url_list,divs) # data=cct.to_mp_run_async(cct.to_asyncio_run,dl,_parsing_sina_dd_price_json) # else: # data=cct.to_asyncio_run(url_list,_parsing_sina_dd_price_json) if len(data)>0: df = df.append(data, ignore_index=True) log.debug("dd.columns:%s" % df.columns.values) #['code' 'name' 'ticktime' 'price' 'volume' 'prev_price' 'kind'] log.debug("get_sina_all_json_dd:%s" % df[:1]) if not df.empty: # for i in range(2, ct.PAGE_NUM[0]): # newdf = _parsing_dayprice_json(i) # df = df.append(newdf, ignore_index=True) # print len(df.index) print (" dd-df: %0.2f" % ((time.time() - start_t))), return df else: print print ("no data json-df: %0.2f"%((time.time() - start_t))), return '' else: print ("Url null json-df: %0.2f "%((time.time() - start_t))), return ''