def get_trade_rank(self, date): url = 'http://www.shfe.com.cn/data/dailydata/kx/pm%s.dat' % date_convert( date, '%Y-%m-%d', '%Y%m%d') response = self.do_request(url, None) rsp = json.loads(response) code = rsp['o_code'] msg = rsp['o_msg'] if code != 0: return None, msg if 'report_date' in rsp.keys(): date = rsp['report_date'] else: date = date_convert(date, '%Y-%m-%d', "%Y%m%d") records = rsp['o_cursor'] df = pd.DataFrame(records) df['date'] = date for col in df.columns: df[col] = df[col].apply(lambda x: format_field(x)) df['RANK'] = df['RANK'].apply(lambda x: int(x)) df = df[(df['RANK'] > 0) & (df['RANK'] <= 20)] df.rename(columns=SHF_name_map, inplace=True) df = df[list(SHF_name_map.values())] return df, ""
def get_trade_rank(self, date): date_int = int(date_convert(date, '%Y-%m-%d', "%Y%m%d")) url = 'http://www.dce.com.cn/publicweb/quotesdata/exportMemberDealPosiQuotesBatchData.html' # url = 'http://www.dce.com.cn/publicweb/quotesdata/exportMemberDealPosiQuotesData.html' year, month, day = split_date(date, '%Y-%m-%d') data = { "year": year, "month": month - 1, # 脑残程序员的设计 "day": day, "batchExportFlag": 'batch', } response = self.do_request(url, data, "POST", type='binary') zip_ref = zipfile.ZipFile(io.BytesIO(response)) df_list = [] for finfo in zip_ref.infolist(): file = zip_ref.open(finfo, 'r') if date_int > 20151231: df = self._parse_trade_file(file) else: df = self._parse_trade_file(file, old=True) df_list.append(df) df_result = _concat_df(df_list) df_result['date'] = date df_result.reset_index(level=['名次'], inplace=True) _rename_df(df_result) return transform(df_result), ""
def _get_url_by_date(self, date): year, month, day = split_date(date, '%Y-%m-%d') date_int = int(date_convert(date, '%Y-%m-%d', "%Y%m%d")) url = 'http://old.czce.com.cn/portal/DFSStaticFiles/Future/%d/%d/FutureDataHolding.txt' url_old = 'http://old.czce.com.cn/portal/exchange/%d/datatradeholding/%d.txt' if date < '2015-10-01': return url_old % (year, date_int) else: return url % (year, date_int)
def get_rzrq_info(self, date): date = date_convert(date, '%Y%m%d', '%Y-%m-%d') df_total = self._get_rzrq_total(date) df_detail = self._get_rzrq_detail(date) if df_total is not None: df_total['date'] = date if df_detail is not None: df_detail['date'] = date return df_total, df_detail
def get_trade_rank(self, date): url = 'http://www.czce.com.cn/portal/DFSStaticFiles/Future/%d/%d/FutureDataHolding.txt' year, month, day = split_date(date, '%Y-%m-%d') date_int = int(date_convert(date, '%Y-%m-%d', "%Y%m%d")) url = url % (year, date_int) response = self.do_request(url, None, "GET", type='binary') df = self._parse_trade_file(io.StringIO(response.decode('gbk'))) df['date'] = date _rename_df(df) return df, ""
def get_pledge_info(self, date): date2 = date_convert(date, '%Y-%m-%d', '%Y%m%d') url = 'http://query.sse.com.cn/exportExcel/exportStockPledgeExcle.do?tradeDate=%s' % (date2) response = self.do_request(url, None, method='GET', type='binary') if response is not None: excel = pd.ExcelFile(io.BytesIO(response)) df_total = excel.parse('交易金额汇总').dropna() df_detail = excel.parse('交易数量明细').dropna() df_total['date'] = date df_detail['date'] = date return df_total, df_detail else: return None, None
def get_rzrq_info(self, date): date2 = date_convert(date, '%Y-%m-%d', '%Y%m%d') url = 'http://www.sse.com.cn/market/dealingdata/overview/margin/a/rzrqjygk%s.xls' % (date2) response = self.do_request(url, None, method='GET', type='binary') if response is not None: excel = pd.ExcelFile(io.BytesIO(response)) df_total = excel.parse('汇总信息').dropna() df_detail = excel.parse('明细信息').dropna() df_total['date'] = date df_detail['date'] = date return df_total, df_detail else: return None, None
def get_trade_rank(self, date): url = 'http://www.shfe.com.cn/data/dailydata/kx/pm%s.dat' % date_convert( date, '%Y-%m-%d', '%Y%m%d') response = self.do_request(url, None) rsp = json.loads(response) code = rsp['o_code'] msg = rsp['o_msg'] if code != 0: return None, msg date = rsp['report_date'] records = rsp['o_cursor'] df = pd.DataFrame(records) df['date'] = date for col in df.columns: df[col] = df[col].apply(lambda x: format_field(x)) df['RANK'] = df['RANK'].apply(lambda x: int(x)) df = df[(df['RANK'] > 0) & (df['RANK'] <= 20)] df.rename(columns={"CJ1": "成交量"}, inplace=True) df.rename(columns={"CJ1_CHG": "成交量增减"}, inplace=True) df.rename(columns={"PARTICIPANTABBR1": "成交量期货公司"}, inplace=True) df.rename(columns={"CJ2": "持买仓量"}, inplace=True) df.rename(columns={"CJ2_CHG": "持买仓量增减"}, inplace=True) df.rename(columns={"PARTICIPANTABBR2": "持买仓量期货公司"}, inplace=True) df.rename(columns={"CJ3": "持卖仓量"}, inplace=True) df.rename(columns={"CJ3_CHG": "持卖仓量增减"}, inplace=True) df.rename(columns={"PARTICIPANTABBR3": "持卖仓量期货公司"}, inplace=True) df.rename(columns={"RANK": "名次"}, inplace=True) df.rename(columns={"INSTRUMENTID": "symbol"}, inplace=True) df.drop([ 'PARTICIPANTID1', 'PARTICIPANTID2', 'PARTICIPANTID3', 'PRODUCTNAME', 'PRODUCTSORTNO' ], axis=1, inplace=True) return df, ""
def get_trade_rank(self, date): url = 'http://old.czce.com.cn/portal/DFSStaticFiles/Future/%d/%d/FutureDataHolding.txt' url_old = 'http://old.czce.com.cn/portal/exchange/%d/datatradeholding/%d.txt' year, month, day = split_date(date, '%Y-%m-%d') date_int = int(date_convert(date, '%Y-%m-%d', "%Y%m%d")) if date_int < 20151001: url_old = url_old % (year, date_int) response = self.do_request(url_old, None, "GET", type='binary') df = self._parse_trade_file(io.StringIO(response.decode('gbk')), old=True) df['date'] = date _rename_df(df) else: url = url % (year, date_int) response = self.do_request(url, None, "GET", type='binary') df = self._parse_trade_file(io.StringIO(response.decode('gbk'))) df['date'] = date _rename_df(df) df = df[df.variety == df.symbol] return transform(df), ""
def get_lgt_share(self, market='SH', date=None): url = "http://sc.hkexnews.hk/TuniS/www.hkexnews.hk/sdw/search/mutualmarket_c.aspx?t=%s" % ( market) today = get_current_day() if date is None: date = today search_date = datetime.datetime.strptime(date, "%Y-%m-%d") year, month, day = search_date.year, search_date.month, search_date.day data = { 'today': date_convert(today, "%Y-%m-%d", "%Y%m%d"), 'ddlShareholdingDay': "%02d" % day, 'ddlShareholdingMonth': "%02d" % month, 'ddlShareholdingYear': year, 'sortBy': '', 'alertMsg': '', 'btnSearch.x': 23, 'btnSearch.y': 5, } aspx_param = self.get_aspx_param(url) data.update(aspx_param) self.add_headers({ "Content-Type": "application/x-www-form-urlencoded", }) rsp = self.do_request(url, data, "POST") # 2. 开始解析返回数据,并从html中提取需要的内容 data = list() soup = BeautifulSoup(rsp, "html5lib") divs = soup.find_all('div') result_date = "" for div in divs: if div.has_attr('id') and 'pnlResult' in div['id']: res_div = div.find_all('div') if (len(res_div)) > 0: result_date = HKExAgent.clear_text(res_div[0].text) result_date = result_date.replace("持股日期: ", "").strip() result_date = date_convert(result_date, "%d/%m/%Y", "%Y-%m-%d") rows = div.table.findAll('tr') for row in rows: cols = row.findAll('td') if len(cols) == 4: code = HKExAgent.clear_text(cols[0].text) name = HKExAgent.clear_text(cols[1].text) share_num = HKExAgent.clear_text(cols[2].text) percent = HKExAgent.clear_text(cols[3].text) data.append({ "code": HKExAgent.process_code(market, code), "name": name, "share_num": share_num, "percent": percent, "date": result_date, }) df = pd.DataFrame(data) return df
def get_trade_rank(self, date): url = 'http://www.shfe.com.cn/data/dailydata/kx/pm%s.dat' % date_convert( date, '%Y-%m-%d', '%Y%m%d') response = self.do_request(url, None) rsp = json.loads(response.encode('utf-8')) code = rsp['o_code'] msg = rsp['o_msg'] date_int = int(date_convert(date, '%Y-%m-%d', "%Y%m%d")) if code != 0: return None, msg if 'report_date' in rsp.keys(): date = rsp['report_date'] else: date = str(date_int) records = rsp['o_cursor'] # print(records) # input() df = pd.DataFrame(records) df['date'] = date[:4] + '-' + date[4:6] + '-' + date[6:] for col in df.columns: df[col] = df[col].apply(lambda x: format_field(x)) df['RANK'] = df['RANK'].apply(lambda x: int(x)) df = df[(df['RANK'] > 0) & (df['RANK'] <= 20)] df.rename(columns={"CJ1": "成交量"}, inplace=True) df.rename(columns={"CJ1_CHG": "成交量增减"}, inplace=True) df.rename(columns={"PARTICIPANTABBR1": "成交量期货公司"}, inplace=True) df.rename(columns={"CJ2": "持买仓量"}, inplace=True) df.rename(columns={"CJ2_CHG": "持买仓量增减"}, inplace=True) df.rename(columns={"PARTICIPANTABBR2": "持买仓量期货公司"}, inplace=True) df.rename(columns={"CJ3": "持卖仓量"}, inplace=True) df.rename(columns={"CJ3_CHG": "持卖仓量增减"}, inplace=True) df.rename(columns={"PARTICIPANTABBR3": "持卖仓量期货公司"}, inplace=True) df.rename(columns={"RANK": "名次"}, inplace=True) df.rename(columns={"INSTRUMENTID": "symbol"}, inplace=True) try: df.drop([ 'PARTICIPANTID1', 'PARTICIPANTID2', 'PARTICIPANTID3', 'PRODUCTNAME', 'PRODUCTSORTNO' ], axis=1, inplace=True) except: try: df.drop(['PRODUCTSORTNO', 'PRODUCTNAME'], axis=1, inplace=True) except: df.drop(['PRODUCTSORTNO'], axis=1, inplace=True) # df['variety'] = df['symbol'].apply(lambda x: re.search(r'([\s\S]*?)\d', x).group(1)) df['variety'] = df['symbol'].apply( lambda x: x.replace(' ', '').rstrip(string.digits)) data2 = pd.DataFrame() for name, data in df.groupby('symbol'): data = data.copy() # 丢掉合计列 # data.loc['合计'] = \ data.iloc[:, :6].apply(lambda x: x.replace('', np.nan)).apply( lambda x: x.sum(skipna=True)) # print(data) data2 = pd.concat([data2, data]) cols = list(data2) cols.insert(0, cols.pop(cols.index('名次'))) cols.insert(-1, cols.pop(cols.index('symbol'))) cols.append(cols.pop(cols.index('date'))) cols.insert(1, cols.pop(cols.index('成交量期货公司'))) cols.insert(4, cols.pop(cols.index('持买仓量期货公司'))) cols.insert(7, cols.pop(cols.index('持卖仓量期货公司'))) df = data2.loc[:, cols] # input() # print(df) return transform(df), ""
def get_lgt_share(self, market='SH', date=None): url = "http://sc.hkexnews.hk/TuniS/www.hkexnews.hk/sdw/search/mutualmarket_c.aspx?t=%s" % ( market) today = get_current_day() if date is None: date = today data = { 'today': date_convert(today, "%Y-%m-%d", "%Y%m%d"), 'sortBy': 'stockcode', 'sortDirection': 'asc', 'alertMsg': '', 'txtShareholdingDate': date_convert(date, "%Y-%m-%d", "%Y/%m/%d"), 'btnSearch': "搜寻", } aspx_param = self.get_aspx_param(url) data.update(aspx_param) self.add_headers({ "Content-Type": "application/x-www-form-urlencoded", }) rsp = self.do_request(url, data, "POST") # 2. 开始解析返回数据,并从html中提取需要的内容 data = list() soup = BeautifulSoup(rsp, "html5lib") divs = soup.find_all('div') result_date = "" for div in divs: if div.has_attr('id') and 'pnlResult' in div['id']: res_span = div.find_all('span') if (len(res_span)) > 0: result_date = HKExAgent.clear_text(res_span[0].text) result_date = result_date.replace("持股日期:", "").strip() result_date = date_convert(result_date, "%Y/%m/%d", "%Y-%m-%d") rows = div.table.findAll('tr') for row in rows: cols = row.findAll('td') if len(cols) == 4: code = HKExAgent.clear_text(cols[0].text.replace( "股份代号:", "")) name = HKExAgent.clear_text(cols[1].text.replace( "股份名称:", "")) share_num = HKExAgent.clear_text(cols[2].text.replace( "于中央结算系统的持股量:", "")) percent = HKExAgent.clear_text(cols[3].text.replace( "占于深交所上市及交易的A股总数的百分比:", "").replace("占于上交所上市及交易的A股总数的百分比:", "")) data.append({ "code": HKExAgent.process_code(market, code), "name": name, "share_num": share_num, "percent": percent, "date": result_date, }) df = pd.DataFrame(data) return df