def run(self): http_session = get_http_session() # 抓取沪市 ETF 列表 url = 'http://query.sse.com.cn/commonQuery.do?sqlId=COMMON_SSE_ZQPZ_ETFLB_L_NEW' response = request_get(http_session, url, headers=DEFAULT_SH_ETF_LIST_HEADER) response_dict = demjson.decode(response.text) df = pd.DataFrame(response_dict.get('result', [])) self.persist_etf_list(df, exchange='sh') self.logger.info('沪市 ETF 列表抓取完成...') # 抓取沪市 ETF 成分股 self.download_sh_etf_component(df, http_session) self.logger.info('沪市 ETF 成分股抓取完成...') # 抓取深市 ETF 列表 url = 'http://www.szse.cn/api/report/ShowReport?SHOWTYPE=xlsx&CATALOGID=1945' response = request_get(http_session, url) df = pd.read_excel(io.BytesIO(response.content), dtype=str) self.persist_etf_list(df, exchange='sz') self.logger.info('深市 ETF 列表抓取完成...') # 抓取深市 ETF 成分股 self.download_sz_etf_component(df, http_session) self.logger.info('深市 ETF 成分股抓取完成...')
def run(self): http_session = get_http_session() url = 'http://query.sse.com.cn/security/stock/downloadStockListFile.do?csrcCode=&stockCode=&areaName=&stockType=1' resp = request_get(http_session, url, headers=DEFAULT_SH_HEADER) self.download_stock_list(response=resp, exchange='sh') url = 'http://www.szse.cn/api/report/ShowReport?SHOWTYPE=xlsx&CATALOGID=1110&TABKEY=tab1&random=0.20932135244582617' resp = request_get(http_session, url, headers=DEFAULT_SZ_HEADER) self.download_stock_list(response=resp, exchange='sz')
def record(self, entity, start, end, size, timestamps, http_session): # 此 url 不支持分页,如果超过我们想取的条数,则只能取最大条数 if start is None or size > self.default_size: size = 8000 param = { 'security_item': entity, 'level': self.level.value, 'size': size } security_item = param['security_item'] size = param['size'] url = ChinaETFDayKdataRecorder.url.format(security_item.exchange, security_item.code, size) response = request_get(http_session, url) response_json = demjson.decode(response.text) if response_json is None or len(response_json) == 0: return [] df = pd.DataFrame(response_json) df.rename(columns={'day': 'timestamp'}, inplace=True) df['timestamp'] = pd.to_datetime(df['timestamp']) df['name'] = security_item.name df['provider'] = 'sina' df['level'] = param['level'] return df.to_dict(orient='records')
def run(self): # get stock blocks from sina http_session = get_http_session() for category, url in self.category_map_url.items(): resp = request_get(http_session, url) resp.encoding = 'GBK' tmp_str = resp.text json_str = tmp_str[tmp_str.index('{'):tmp_str.index('}') + 1] tmp_json = json.loads(json_str) the_list = [] for code in tmp_json: name = tmp_json[code].split(',')[1] entity_id = f'block_cn_{code}' the_list.append({ 'id': entity_id, 'entity_id': entity_id, 'entity_type': EntityType.Block.value, 'exchange': 'cn', 'code': code, 'name': name, 'category': category.value }) if the_list: df = pd.DataFrame.from_records(the_list) df_to_db(df=df, region=Region.CHN, data_schema=self.data_schema, provider=self.provider, force_update=True) self.logger.info(f"finish record sina blocks:{category.value}")
def populate_sh_etf_type(df: pd.DataFrame, http_session): """ 填充沪市 ETF 代码对应的 TYPE 到列表数据中 :param df: ETF 列表数据 :return: 包含 ETF 对应 TYPE 的列表数据 """ query_url = 'http://query.sse.com.cn/infodisplay/queryETFNewAllInfo.do?' \ 'isPagination=false&type={}&pageHelp.pageSize=25' type_df = pd.DataFrame() for etf_class in [1, 2]: url = query_url.format(etf_class) response = request_get(http_session, url, headers=DEFAULT_SH_ETF_LIST_HEADER) response_dict = demjson.decode(response.text) response_df = pd.DataFrame(response_dict.get('result', [])) response_df = response_df[['fundid1', 'etftype']] type_df = pd.concat([type_df, response_df]) result_df = df.copy() result_df = result_df.sort_values(by='FUND_ID').reset_index(drop=True) type_df = type_df.sort_values(by='fundid1').reset_index(drop=True) result_df['ETF_TYPE'] = type_df['etftype'] return result_df
def run(self): http_session = get_http_session() for category, url in self.category_map_url.items(): resp = request_get(http_session, url) results = json_callback_param(resp.text) the_list = [] for result in results: items = result.split(',') code = items[1] name = items[2] entity_id = f'block_cn_{code}' the_list.append({ 'id': entity_id, 'entity_id': entity_id, 'entity_type': EntityType.Block.value, 'exchange': 'cn', 'code': code, 'name': name, 'category': category.value }) if the_list: df = pd.DataFrame.from_records(the_list) df_to_db(df=df, region=Region.CHN, data_schema=self.data_schema, provider=self.provider, force_update=True) self.logger.info(f"finish record sina blocks:{category.value}")
def record(self, entity, start, end, size, timestamps, http_session): json_results = [] for timestamp in timestamps: timestamp_str = to_time_str(timestamp) url = self.url.format(timestamp_str) response = request_get(http_session, url, headers=DEFAULT_SH_SUMMARY_HEADER) results = demjson.decode(response.text[response.text.index("(") + 1:response.text.index(")")])['result'] result = [result for result in results if result['productType'] == '1'] if result and len(result) == 1: result_json = result[0] # 有些较老的数据不存在,默认设为0.0 json_results.append({ 'provider': Provider.Exchange.value, 'timestamp': timestamp, 'name': '上证指数', 'pe': to_float(result_json['profitRate'], 0.0), 'total_value': to_float(result_json['marketValue1'] + '亿', 0.0), 'total_tradable_vaule': to_float(result_json['negotiableValue1'] + '亿', 0.0), 'volume': to_float(result_json['trdVol1'] + '万', 0.0), 'turnover': to_float(result_json['trdAmt1'] + '亿', 0.0), 'turnover_rate': to_float(result_json['exchangeRate'], 0.0), }) if len(json_results) > self.batch_size: return json_results return json_results
def download_sz_etf_component(self, df: pd.DataFrame, http_session): query_url = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vII_NewestComponent/indexid/{}.phtml' self.parse_sz_etf_underlying_index(df) for _, etf in df.iterrows(): underlying_index = etf['拟合指数'] etf_code = etf['证券代码'] if len(underlying_index) == 0: self.logger.info(f'{etf["证券简称"]} - {etf_code} 非 A 股市场指数,跳过...') continue url = query_url.format(underlying_index) response = request_get(http_session, url) response.encoding = 'gbk' try: dfs = pd.read_html(response.text, header=1) except ValueError as error: self.logger.error( f'HTML parse error: {error}, response: {response.text}') continue if len(dfs) < 4: continue response_df = dfs[3].copy() response_df = response_df.dropna(axis=1, how='any') response_df['品种代码'] = response_df['品种代码'].apply( lambda x: f'{x:06d}') etf_id = f'etf_sz_{etf_code}' response_df = response_df[['品种代码', '品种名称']].copy() response_df.rename(columns={ '品种代码': 'stock_code', '品种名称': 'stock_name' }, inplace=True) response_df['entity_id'] = etf_id response_df['entity_type'] = EntityType.ETF.value response_df['exchange'] = 'sz' response_df['code'] = etf_code response_df['name'] = etf['证券简称'] response_df['timestamp'] = now_pd_timestamp(Region.CHN) response_df['stock_id'] = response_df['stock_code'].apply( lambda code: china_stock_code_to_id(code)) response_df['id'] = response_df['stock_id'].apply( lambda x: f'{etf_id}_{x}') df_to_db(df=response_df, region=Region.CHN, data_schema=self.data_schema, provider=self.provider) self.logger.info(f'{etf["证券简称"]} - {etf_code} 成分股抓取完成...') self.sleep()
def run(self): http_session = get_http_session() exchanges = ['NYSE', 'NASDAQ', 'AMEX'] for exchange in exchanges: url = 'https://old.nasdaq.com/screening/companies-by-name.aspx?letter=0&render=download&exchange={}'.format(exchange) resp = request_get(http_session, url, headers=YAHOO_STOCK_LIST_HEADER) self.download_stock_list(response=resp, exchange=exchange)
def record(self, entity, start, end, size, timestamps, http_session): the_quarters = get_year_quarters(start, now_pd_timestamp(Region.CHN)) if not is_same_date(entity.timestamp, start) and len(the_quarters) > 1: the_quarters = the_quarters[1:] param = { 'security_item': entity, 'quarters': the_quarters, 'level': self.level.value } security_item = param['security_item'] quarters = param['quarters'] level = param['level'] result_df = pd.DataFrame() for year, quarter in quarters: query_url = self.url.format(security_item.code, year, quarter) response = request_get(http_session, query_url) response.encoding = 'gbk' try: dfs = pd.read_html(response.text) except ValueError as error: self.logger.error( f'skip ({year}-{quarter:02d}){security_item.code}{security_item.name}({error})' ) self.sleep() continue if len(dfs) < 5: self.sleep() continue df = dfs[4].copy() df = df.iloc[1:] df.columns = [ 'timestamp', 'open', 'high', 'close', 'low', 'volume', 'turnover' ] df['name'] = security_item.name df['level'] = level df['timestamp'] = pd.to_datetime(df['timestamp']) df['provider'] = Provider.Sina result_df = pd.concat([result_df, df]) self.logger.info( f'({security_item.code}{security_item.name})({year}-{quarter:02d})' ) self.sleep() result_df = result_df.sort_values(by='timestamp') return result_df.to_dict(orient='records')
def record(self, entity, start, end, size, timestamps, http_session): url = self.generate_url(category=entity.category, code=entity.code, number=size) resp = request_get(http_session, url) opendate = "opendate" avg_price = "avg_price" avg_changeratio = 'avg_changeratio' turnover = 'turnover' netamount = 'netamount' ratioamount = 'ratioamount' r0_net = 'r0_net' r0_ratio = 'r0_ratio' r0x_ratio = 'r0x_ratio' cnt_r0x_ratio = 'cnt_r0x_ratio' json_list = [] try: json_list = eval(resp.text) except Exception as e: resp.encoding = 'GBK' self.logger.error(resp.text) self.sleep() result_list = [] for item in json_list: result_list.append({ 'name': entity.name, 'timestamp': to_pd_timestamp(item['opendate']), 'close': to_float(item['avg_price']), 'change_pct': to_float(item['avg_changeratio']), 'turnover_rate': to_float(item['turnover']) / 10000, 'net_inflows': to_float(item['netamount']), 'net_inflow_rate': to_float(item['ratioamount']), 'net_main_inflows': to_float(item['r0_net']), 'net_main_inflow_rate': to_float(item['r0_ratio']) }) return result_list
def download_sh_etf_component(self, df: pd.DataFrame, http_session): """ ETF_CLASS => 1. 单市场 ETF 2.跨市场 ETF 3. 跨境 ETF 5. 债券 ETF 6. 黄金 ETF :param df: ETF 列表数据 :return: None """ query_url = 'http://query.sse.com.cn/infodisplay/queryConstituentStockInfo.do?' \ 'isPagination=false&type={}&etfClass={}' etf_df = df[(df['ETF_CLASS'] == '1') | (df['ETF_CLASS'] == '2')] etf_df = self.populate_sh_etf_type(etf_df, http_session) for _, etf in etf_df.iterrows(): url = query_url.format(etf['ETF_TYPE'], etf['ETF_CLASS']) response = request_get(http_session, url, headers=DEFAULT_SH_ETF_LIST_HEADER) response_dict = demjson.decode(response.text) response_df = pd.DataFrame(response_dict.get('result', [])) etf_code = etf['FUND_ID'] etf_id = f'etf_sh_{etf_code}' response_df = response_df[['instrumentId', 'instrumentName']].copy() response_df.rename(columns={ 'instrumentId': 'stock_code', 'instrumentName': 'stock_name' }, inplace=True) response_df['entity_id'] = etf_id response_df['entity_type'] = EntityType.ETF.value response_df['exchange'] = 'sh' response_df['code'] = etf_code response_df['name'] = etf['FUND_NAME'] response_df['timestamp'] = now_pd_timestamp(Region.CHN) response_df['stock_id'] = response_df['stock_code'].apply( lambda code: china_stock_code_to_id(code)) response_df['id'] = response_df['stock_id'].apply( lambda x: f'{etf_id}_{x}') df_to_db(df=response_df, region=Region.CHN, data_schema=self.data_schema, provider=self.provider) self.logger.info(f'{etf["FUND_NAME"]} - {etf_code} 成分股抓取完成...') self.sleep()
def fetch_csi_index_component(self, df: pd.DataFrame, http_session): """ 抓取上证、中证指数成分股 """ query_url = 'http://www.csindex.com.cn/uploads/file/autofile/cons/{}cons.xls' for _, index in df.iterrows(): index_code = index['code'] url = query_url.format(index_code) try: response = request_get(http_session, url) response.raise_for_status() except requests.HTTPError as error: self.logger.error( f'{index["name"]} - {index_code} 成分股抓取错误 ({error})') continue response_df = pd.read_excel(io.BytesIO(response.content)) response_df = response_df[[ '成分券代码Constituent Code', '成分券名称Constituent Name' ]].rename( columns={ '成分券代码Constituent Code': 'stock_code', '成分券名称Constituent Name': 'stock_name' }) index_id = f'index_cn_{index_code}' response_df['entity_id'] = index_id response_df['entity_type'] = EntityType.Index.value response_df['exchange'] = 'cn' response_df['code'] = index_code response_df['name'] = index['name'] response_df['timestamp'] = now_pd_timestamp(Region.CHN) response_df['stock_id'] = response_df['stock_code'].apply( lambda x: china_stock_code_to_id(str(x))) response_df['id'] = response_df['stock_id'].apply( lambda x: f'{index_id}_{x}') df_to_db(df=response_df, region=Region.CHN, data_schema=self.data_schema, provider=self.provider, force_update=True) self.logger.info(f'{index["name"]} - {index_code} 成分股抓取完成...') self.sleep()
def fetch_cni_index_component(self, df: pd.DataFrame, http_session): """ 抓取国证指数成分股 """ query_url = 'http://www.cnindex.com.cn/docs/yb_{}.xls' for _, index in df.iterrows(): index_code = index['code'] url = query_url.format(index_code) try: response = request_get(http_session, url) response.raise_for_status() except requests.HTTPError as error: self.logger.error( f'{index["name"]} - {index_code} 成分股抓取错误 ({error})') continue response_df = pd.read_excel(io.BytesIO(response.content), dtype='str') index_id = f'index_cn_{index_code}' try: response_df = response_df[['样本股代码']] except KeyError: response_df = response_df[['证券代码']] response_df['entity_id'] = index_id response_df['entity_type'] = EntityType.Index.value response_df['exchange'] = 'cn' response_df['code'] = index_code response_df['name'] = index['name'] response_df['timestamp'] = now_pd_timestamp(Region.CHN) response_df.columns = ['stock_code'] response_df['stock_id'] = response_df['stock_code'].apply( lambda x: china_stock_code_to_id(str(x))) response_df['id'] = response_df['stock_id'].apply( lambda x: f'{index_id}_{x}') df_to_db(df=response_df, region=Region.CHN, data_schema=self.data_schema, provider=self.provider) self.logger.info(f'{index["name"]} - {index_code} 成分股抓取完成...') self.sleep()
def fetch_csi_index(self, http_session) -> None: """ 抓取上证、中证指数列表 """ url = 'http://www.csindex.com.cn/zh-CN/indices/index' \ '?page={}&page_size={}&data_type=json&class_1=1&class_2=2&class_7=7&class_10=10' index_list = [] page = 1 page_size = 50 while True: query_url = url.format(page, page_size) response = request_get(http_session, query_url) response_dict = demjson.decode(response.text) response_index_list = response_dict.get('list', []) if len(response_index_list) == 0: break index_list.extend(response_index_list) self.logger.info(f'上证、中证指数第 {page} 页抓取完成...') page += 1 self.sleep() df = pd.DataFrame(index_list) df = df[[ 'base_date', 'base_point', 'index_code', 'indx_sname', 'online_date', 'class_eseries' ]].copy() df.columns = [ 'timestamp', 'base_point', 'code', 'name', 'list_date', 'class_eseries' ] df['category'] = df['class_eseries'].apply( lambda x: x.split(' ')[0].lower()) df = df.drop('class_eseries', axis=1) df = df.loc[df['code'].str.contains(r'^\d{6}$')] self.persist_index(df) self.logger.info('上证、中证指数列表抓取完成...') # 抓取上证、中证指数成分股 self.fetch_csi_index_component(df, http_session) self.logger.info('上证、中证指数成分股抓取完成...')
def fetch_szse_index(self, http_session) -> None: """ 抓取深证指数列表 """ url = 'http://www.szse.cn/api/report/ShowReport?SHOWTYPE=xlsx&CATALOGID=1812_zs&TABKEY=tab1' response = request_get(http_session, url) df = pd.read_excel(io.BytesIO(response.content), dtype='str') df.columns = ['code', 'name', 'timestamp', 'base_point', 'list_date'] df['category'] = 'szse' df = df.loc[df['code'].str.contains(r'^\d{6}$')] self.persist_index(df) self.logger.info('深证指数列表抓取完成...') # 抓取深证指数成分股 self.fetch_szse_index_component(df, http_session) self.logger.info('深证指数成分股抓取完成...')
def fetch_szse_index_component(self, df: pd.DataFrame, http_session): """ 抓取深证指数成分股 """ query_url = 'http://www.szse.cn/api/report/ShowReport?SHOWTYPE=xlsx&CATALOGID=1747_zs&TABKEY=tab1&ZSDM={}' for _, index in df.iterrows(): index_code = index['code'] url = query_url.format(index_code) response = request_get(http_session, url) response_df = pd.read_excel(io.BytesIO(response.content), dtype='str') index_id = f'index_cn_{index_code}' response_df['entity_id'] = index_id response_df['entity_type'] = EntityType.Index.value response_df['exchange'] = 'cn' response_df['code'] = index_code response_df['name'] = index['name'] response_df['timestamp'] = now_pd_timestamp(Region.CHN) response_df.rename(columns={ '证券代码': 'stock_code', '证券简称': 'stock_name' }, inplace=True) response_df['stock_id'] = response_df['stock_code'].apply( lambda x: china_stock_code_to_id(str(x))) response_df['id'] = response_df['stock_id'].apply( lambda x: f'{index_id}_{x}') df_to_db(df=response_df, region=Region.CHN, data_schema=self.data_schema, provider=self.provider) self.logger.info(f'{index["name"]} - {index_code} 成分股抓取完成...') self.sleep()
def record(self, entity, start, end, size, timestamps, http_session): for page in range(1, 5): resp = request_get( http_session, self.category_stocks_url.format(page, entity.code)) try: if resp.text == 'null' or resp.text is None: break category_jsons = demjson.decode(resp.text) the_list = [] for category in category_jsons: stock_code = category['code'] stock_id = china_stock_code_to_id(stock_code) block_id = entity.id the_list.append({ 'id': '{}_{}'.format(block_id, stock_id), 'entity_id': block_id, 'entity_type': EntityType.Block.value, 'exchange': entity.exchange, 'code': entity.code, 'name': entity.name, 'timestamp': now_pd_timestamp(Region.CHN), 'stock_id': stock_id, 'stock_code': stock_code, 'stock_name': category['name'], }) if the_list: df = pd.DataFrame.from_records(the_list) df_to_db(df=df, region=Region.CHN, data_schema=self.data_schema, provider=self.provider, force_update=True) self.logger.info('finish recording BlockStock:{},{}'.format( entity.category, entity.name)) except Exception as e: self.logger.error("error:{},resp.text:{}".format(e, resp.text)) self.sleep()
def fetch_cni_index(self, http_session) -> None: """ 抓取国证指数列表 """ url = 'http://www.cnindex.com.cn/zstx/jcxl/' response = request_get(http_session, url) response.encoding = 'utf-8' dfs = pd.read_html(response.text) # 第 9 个 table 之后为非股票指数 dfs = dfs[1:9] result_df = pd.DataFrame() for df in dfs: header = df.iloc[0] df = df[1:] df.columns = header df.astype('str') result_df = pd.concat([result_df, df]) result_df = result_df.drop('样本股数量', axis=1) result_df.columns = [ 'name', 'code', 'timestamp', 'base_point', 'list_date' ] result_df['timestamp'] = result_df['timestamp'].apply( lambda x: x.replace('-', '')) result_df['list_date'] = result_df['list_date'].apply( lambda x: x.replace('-', '')) result_df['category'] = 'csi' result_df = result_df.loc[result_df['code'].str.contains(r'^\d{6}$')] self.persist_index(result_df) self.logger.info('国证指数列表抓取完成...') # 抓取国证指数成分股 self.fetch_cni_index_component(result_df, http_session) self.logger.info('国证指数成分股抓取完成...')
def record(self, entity, start, end, size, timestamps, http_session): resp = request_get(http_session, self.category_stocks_url.format(entity.code, '1')) try: results = json_callback_param(resp.text) the_list = [] for result in results: items = result.split(',') stock_code = items[1] stock_id = china_stock_code_to_id(stock_code) block_id = entity.id the_list.append({ 'id': '{}_{}'.format(block_id, stock_id), 'entity_id': block_id, 'entity_type': EntityType.Block.value, 'exchange': entity.exchange, 'code': entity.code, 'name': entity.name, 'timestamp': now_pd_timestamp(Region.CHN), 'stock_id': stock_id, 'stock_code': stock_code, 'stock_name': items[2], }) if the_list: df = pd.DataFrame.from_records(the_list) df_to_db(df=df, region=Region.CHN, data_schema=self.data_schema, provider=self.provider, force_update=True) self.logger.info('finish recording block:{},{}'.format( entity.category, entity.name)) except Exception as e: self.logger.error("error:{}, resp.text:{}".format(e, resp.text)) self.sleep()
def record(self, entity, start, end, size, timestamps, http_session): the_url = self.url.format( "{}".format(entity.code), level_flag(self.level), size, now_time_str(region=Region.CHN, fmt=TIME_FORMAT_DAY1)) resp = request_get(http_session, the_url) results = json_callback_param(resp.text) kdatas = [] if results: klines = results['data']['klines'] # TODO: ignore the last unfinished kdata now,could control it better if need for result in klines[:-1]: # "2000-01-28,1005.26,1012.56,1173.12,982.13,3023326,3075552000.00" # time,open,close,high,low,volume,turnover fields = result.split(',') the_timestamp = to_pd_timestamp(fields[0]) the_id = generate_kdata_id(entity_id=entity.id, timestamp=the_timestamp, level=self.level) kdatas.append( dict(id=the_id, timestamp=the_timestamp, entity_id=entity.id, code=entity.code, name=entity.name, level=self.level.value, open=to_float(fields[1]), close=to_float(fields[2]), high=to_float(fields[3]), low=to_float(fields[4]), volume=to_float(fields[5]), turnover=to_float(fields[6]))) return kdatas
def fetch_cumulative_net_value(self, security_item, start, end, http_session) -> pd.DataFrame: query_url = 'http://api.fund.eastmoney.com/f10/lsjz?' \ 'fundCode={}&pageIndex={}&pageSize=200&startDate={}&endDate={}' page = 1 df = pd.DataFrame() while True: url = query_url.format(security_item.code, page, to_time_str(start), to_time_str(end)) response = request_get(http_session, url, headers=EASTMONEY_ETF_NET_VALUE_HEADER) response_json = demjson.decode(response.text) response_df = pd.DataFrame(response_json['Data']['LSJZList']) # 最后一页 if not pd_is_not_null(response_df): break response_df['FSRQ'] = pd.to_datetime(response_df['FSRQ']) response_df['JZZZL'] = pd.to_numeric(response_df['JZZZL'], errors='coerce') response_df['LJJZ'] = pd.to_numeric(response_df['LJJZ'], errors='coerce') response_df = response_df.fillna(0) response_df.set_index('FSRQ', inplace=True, drop=True) df = pd.concat([df, response_df]) page += 1 self.sleep() return df
def record(self, entity, start, end, size, timestamps, http_session): param = { 'url': self.generate_url(code='{}{}'.format(entity.exchange, entity.code), number=size), 'security_item': entity } resp = request_get(http_session, param['url']) # {opendate:"2019-04-29",trade:"10.8700",changeratio:"-0.0431338",turnover:"74.924",netamount:"-2903349.8500", # ratioamount:"-0.155177",r0:"0.0000",r1:"2064153.0000",r2:"6485031.0000",r3:"10622169.2100",r0_net:"0.0000", # r1_net:"2064153.0000",r2_net:"-1463770.0000",r3_net:"-3503732.8500"} # opendate = "opendate" # trade = "trade" # changeratio = 'changeratio' # turnover = 'turnover' # netamount = 'netamount' # ratioamount = 'ratioamount' # r0 = 'r0' # r1 = 'r1' # r2 = 'r2' # r3 = 'r3' # r0_net = 'r0_net' # r1_net = 'r1_net' # r2_net = 'r2_net' # r3_net = 'r3_net' json_list = [] try: json_list = eval(resp.text) except Exception as e: resp.encoding = 'GBK' self.logger.error("text:{}, error:{}".format(resp.text, e)) self.sleep() result_list = [] for item in json_list: amount = to_float(item['r0']) + to_float(item['r1']) + to_float( item['r2']) + to_float(item['r3']) result = { 'timestamp': to_pd_timestamp(item['opendate']), 'name': entity.name, 'close': to_float(item['trade']), 'change_pct': to_float(item['changeratio']), 'turnover_rate': to_float(item['turnover']) / 10000, 'net_inflows': to_float(item['netamount']), 'net_inflow_rate': to_float(item['ratioamount']), # # 主力=超大单+大单 # net_main_inflows = Column(Float) # net_main_inflow_rate = Column(Float) # # 超大单 # net_huge_inflows = Column(Float) # net_huge_inflow_rate = Column(Float) # # 大单 # net_big_inflows = Column(Float) # net_big_inflow_rate = Column(Float) # # # 中单 # net_medium_inflows = Column(Float) # net_medium_inflow_rate = Column(Float) # # 小单 # net_small_inflows = Column(Float) # net_small_inflow_rate = Column(Float) 'net_main_inflows': to_float(item['r0_net']) + to_float(item['r1_net']), 'net_huge_inflows': to_float(item['r0_net']), 'net_big_inflows': to_float(item['r1_net']), 'net_medium_inflows': to_float(item['r2_net']), 'net_small_inflows': to_float(item['r3_net']), } if amount != 0: result['net_main_inflow_rate'] = (to_float( item['r0_net']) + to_float(item['r1_net'])) / amount result['net_huge_inflow_rate'] = to_float( item['r0_net']) / amount result['net_big_inflow_rate'] = to_float( item['r1_net']) / amount result['net_medium_inflow_rate'] = to_float( item['r2_net']) / amount result['net_small_inflow_rate'] = to_float( item['r3_net']) / amount result_list.append(result) return result_list