def get_trade_rank(self, date):
url = 'http://www.shfe.com.cn/data/dailydata/kx/pm%s.dat' % date_convert(
date, '%Y-%m-%d', '%Y%m%d')
response = self.do_request(url, None)
rsp = json.loads(response)
code = rsp['o_code']
msg = rsp['o_msg']
if code != 0:
return None, msg
if 'report_date' in rsp.keys():
date = rsp['report_date']
else:
date = date_convert(date, '%Y-%m-%d', "%Y%m%d")
records = rsp['o_cursor']
df = pd.DataFrame(records)
df['date'] = date
for col in df.columns:
df[col] = df[col].apply(lambda x: format_field(x))
df['RANK'] = df['RANK'].apply(lambda x: int(x))
df = df[(df['RANK'] > 0) & (df['RANK'] <= 20)]
df.rename(columns=SHF_name_map, inplace=True)
df = df[list(SHF_name_map.values())]
return df, ""
def get_trade_rank(self, date):
date_int = int(date_convert(date, '%Y-%m-%d', "%Y%m%d"))
url = 'http://www.dce.com.cn/publicweb/quotesdata/exportMemberDealPosiQuotesBatchData.html'
# url = 'http://www.dce.com.cn/publicweb/quotesdata/exportMemberDealPosiQuotesData.html'
year, month, day = split_date(date, '%Y-%m-%d')
data = {
"year": year,
"month": month - 1, # 脑残程序员的设计
"day": day,
"batchExportFlag": 'batch',
}
response = self.do_request(url, data, "POST", type='binary')
zip_ref = zipfile.ZipFile(io.BytesIO(response))
df_list = []
for finfo in zip_ref.infolist():
file = zip_ref.open(finfo, 'r')
if date_int > 20151231:
df = self._parse_trade_file(file)
else:
df = self._parse_trade_file(file, old=True)
df_list.append(df)
df_result = _concat_df(df_list)
df_result['date'] = date
df_result.reset_index(level=['名次'], inplace=True)
_rename_df(df_result)
return transform(df_result), ""
def _get_url_by_date(self, date):
year, month, day = split_date(date, '%Y-%m-%d')
date_int = int(date_convert(date, '%Y-%m-%d', "%Y%m%d"))
url = 'http://old.czce.com.cn/portal/DFSStaticFiles/Future/%d/%d/FutureDataHolding.txt'
url_old = 'http://old.czce.com.cn/portal/exchange/%d/datatradeholding/%d.txt'
if date < '2015-10-01':
return url_old % (year, date_int)
else:
return url % (year, date_int)
def get_rzrq_info(self, date):
date = date_convert(date, '%Y%m%d', '%Y-%m-%d')
df_total = self._get_rzrq_total(date)
df_detail = self._get_rzrq_detail(date)
if df_total is not None:
df_total['date'] = date
if df_detail is not None:
df_detail['date'] = date
return df_total, df_detail
def get_trade_rank(self, date):
url = 'http://www.czce.com.cn/portal/DFSStaticFiles/Future/%d/%d/FutureDataHolding.txt'
year, month, day = split_date(date, '%Y-%m-%d')
date_int = int(date_convert(date, '%Y-%m-%d', "%Y%m%d"))
url = url % (year, date_int)
response = self.do_request(url, None, "GET", type='binary')
df = self._parse_trade_file(io.StringIO(response.decode('gbk')))
df['date'] = date
_rename_df(df)
return df, ""
def get_pledge_info(self, date):
date2 = date_convert(date, '%Y-%m-%d', '%Y%m%d')
url = 'http://query.sse.com.cn/exportExcel/exportStockPledgeExcle.do?tradeDate=%s' % (date2)
response = self.do_request(url, None, method='GET', type='binary')
if response is not None:
excel = pd.ExcelFile(io.BytesIO(response))
df_total = excel.parse('交易金额汇总').dropna()
df_detail = excel.parse('交易数量明细').dropna()
df_total['date'] = date
df_detail['date'] = date
return df_total, df_detail
else:
return None, None
def get_rzrq_info(self, date):
date2 = date_convert(date, '%Y-%m-%d', '%Y%m%d')
url = 'http://www.sse.com.cn/market/dealingdata/overview/margin/a/rzrqjygk%s.xls' % (date2)
response = self.do_request(url, None, method='GET', type='binary')
if response is not None:
excel = pd.ExcelFile(io.BytesIO(response))
df_total = excel.parse('汇总信息').dropna()
df_detail = excel.parse('明细信息').dropna()
df_total['date'] = date
df_detail['date'] = date
return df_total, df_detail
else:
return None, None
def get_trade_rank(self, date):
url = 'http://www.shfe.com.cn/data/dailydata/kx/pm%s.dat' % date_convert(
date, '%Y-%m-%d', '%Y%m%d')
response = self.do_request(url, None)
rsp = json.loads(response)
code = rsp['o_code']
msg = rsp['o_msg']
if code != 0:
return None, msg
date = rsp['report_date']
records = rsp['o_cursor']
df = pd.DataFrame(records)
df['date'] = date
for col in df.columns:
df[col] = df[col].apply(lambda x: format_field(x))
df['RANK'] = df['RANK'].apply(lambda x: int(x))
df = df[(df['RANK'] > 0) & (df['RANK'] <= 20)]
df.rename(columns={"CJ1": "成交量"}, inplace=True)
df.rename(columns={"CJ1_CHG": "成交量增减"}, inplace=True)
df.rename(columns={"PARTICIPANTABBR1": "成交量期货公司"}, inplace=True)
df.rename(columns={"CJ2": "持买仓量"}, inplace=True)
df.rename(columns={"CJ2_CHG": "持买仓量增减"}, inplace=True)
df.rename(columns={"PARTICIPANTABBR2": "持买仓量期货公司"}, inplace=True)
df.rename(columns={"CJ3": "持卖仓量"}, inplace=True)
df.rename(columns={"CJ3_CHG": "持卖仓量增减"}, inplace=True)
df.rename(columns={"PARTICIPANTABBR3": "持卖仓量期货公司"}, inplace=True)
df.rename(columns={"RANK": "名次"}, inplace=True)
df.rename(columns={"INSTRUMENTID": "symbol"}, inplace=True)
df.drop([
'PARTICIPANTID1', 'PARTICIPANTID2', 'PARTICIPANTID3',
'PRODUCTNAME', 'PRODUCTSORTNO'
],
axis=1,
inplace=True)
return df, ""
def get_trade_rank(self, date):
url = 'http://old.czce.com.cn/portal/DFSStaticFiles/Future/%d/%d/FutureDataHolding.txt'
url_old = 'http://old.czce.com.cn/portal/exchange/%d/datatradeholding/%d.txt'
year, month, day = split_date(date, '%Y-%m-%d')
date_int = int(date_convert(date, '%Y-%m-%d', "%Y%m%d"))
if date_int < 20151001:
url_old = url_old % (year, date_int)
response = self.do_request(url_old, None, "GET", type='binary')
df = self._parse_trade_file(io.StringIO(response.decode('gbk')),
old=True)
df['date'] = date
_rename_df(df)
else:
url = url % (year, date_int)
response = self.do_request(url, None, "GET", type='binary')
df = self._parse_trade_file(io.StringIO(response.decode('gbk')))
df['date'] = date
_rename_df(df)
df = df[df.variety == df.symbol]
return transform(df), ""
def get_lgt_share(self, market='SH', date=None):
url = "http://sc.hkexnews.hk/TuniS/www.hkexnews.hk/sdw/search/mutualmarket_c.aspx?t=%s" % (
market)
today = get_current_day()
if date is None:
date = today
search_date = datetime.datetime.strptime(date, "%Y-%m-%d")
year, month, day = search_date.year, search_date.month, search_date.day
data = {
'today': date_convert(today, "%Y-%m-%d", "%Y%m%d"),
'ddlShareholdingDay': "%02d" % day,
'ddlShareholdingMonth': "%02d" % month,
'ddlShareholdingYear': year,
'sortBy': '',
'alertMsg': '',
'btnSearch.x': 23,
'btnSearch.y': 5,
}
aspx_param = self.get_aspx_param(url)
data.update(aspx_param)
self.add_headers({
"Content-Type": "application/x-www-form-urlencoded",
})
rsp = self.do_request(url, data, "POST")
# 2. 开始解析返回数据，并从html中提取需要的内容
data = list()
soup = BeautifulSoup(rsp, "html5lib")
divs = soup.find_all('div')
result_date = ""
for div in divs:
if div.has_attr('id') and 'pnlResult' in div['id']:
res_div = div.find_all('div')
if (len(res_div)) > 0:
result_date = HKExAgent.clear_text(res_div[0].text)
result_date = result_date.replace("持股日期: ", "").strip()
result_date = date_convert(result_date, "%d/%m/%Y",
"%Y-%m-%d")
rows = div.table.findAll('tr')
for row in rows:
cols = row.findAll('td')
if len(cols) == 4:
code = HKExAgent.clear_text(cols[0].text)
name = HKExAgent.clear_text(cols[1].text)
share_num = HKExAgent.clear_text(cols[2].text)
percent = HKExAgent.clear_text(cols[3].text)
data.append({
"code":
HKExAgent.process_code(market, code),
"name":
name,
"share_num":
share_num,
"percent":
percent,
"date":
result_date,
})
df = pd.DataFrame(data)
return df
def get_trade_rank(self, date):
url = 'http://www.shfe.com.cn/data/dailydata/kx/pm%s.dat' % date_convert(
date, '%Y-%m-%d', '%Y%m%d')
response = self.do_request(url, None)
rsp = json.loads(response.encode('utf-8'))
code = rsp['o_code']
msg = rsp['o_msg']
date_int = int(date_convert(date, '%Y-%m-%d', "%Y%m%d"))
if code != 0:
return None, msg
if 'report_date' in rsp.keys():
date = rsp['report_date']
else:
date = str(date_int)
records = rsp['o_cursor']
# print(records)
# input()
df = pd.DataFrame(records)
df['date'] = date[:4] + '-' + date[4:6] + '-' + date[6:]
for col in df.columns:
df[col] = df[col].apply(lambda x: format_field(x))
df['RANK'] = df['RANK'].apply(lambda x: int(x))
df = df[(df['RANK'] > 0) & (df['RANK'] <= 20)]
df.rename(columns={"CJ1": "成交量"}, inplace=True)
df.rename(columns={"CJ1_CHG": "成交量增减"}, inplace=True)
df.rename(columns={"PARTICIPANTABBR1": "成交量期货公司"}, inplace=True)
df.rename(columns={"CJ2": "持买仓量"}, inplace=True)
df.rename(columns={"CJ2_CHG": "持买仓量增减"}, inplace=True)
df.rename(columns={"PARTICIPANTABBR2": "持买仓量期货公司"}, inplace=True)
df.rename(columns={"CJ3": "持卖仓量"}, inplace=True)
df.rename(columns={"CJ3_CHG": "持卖仓量增减"}, inplace=True)
df.rename(columns={"PARTICIPANTABBR3": "持卖仓量期货公司"}, inplace=True)
df.rename(columns={"RANK": "名次"}, inplace=True)
df.rename(columns={"INSTRUMENTID": "symbol"}, inplace=True)
try:
df.drop([
'PARTICIPANTID1', 'PARTICIPANTID2', 'PARTICIPANTID3',
'PRODUCTNAME', 'PRODUCTSORTNO'
],
axis=1,
inplace=True)
except:
try:
df.drop(['PRODUCTSORTNO', 'PRODUCTNAME'], axis=1, inplace=True)
except:
df.drop(['PRODUCTSORTNO'], axis=1, inplace=True)
# df['variety'] = df['symbol'].apply(lambda x: re.search(r'([\s\S]*?)\d', x).group(1))
df['variety'] = df['symbol'].apply(
lambda x: x.replace(' ', '').rstrip(string.digits))
data2 = pd.DataFrame()
for name, data in df.groupby('symbol'):
data = data.copy()
# 丢掉合计列
# data.loc['合计'] = \
data.iloc[:, :6].apply(lambda x: x.replace('', np.nan)).apply(
lambda x: x.sum(skipna=True))
# print(data)
data2 = pd.concat([data2, data])
cols = list(data2)
cols.insert(0, cols.pop(cols.index('名次')))
cols.insert(-1, cols.pop(cols.index('symbol')))
cols.append(cols.pop(cols.index('date')))
cols.insert(1, cols.pop(cols.index('成交量期货公司')))
cols.insert(4, cols.pop(cols.index('持买仓量期货公司')))
cols.insert(7, cols.pop(cols.index('持卖仓量期货公司')))
df = data2.loc[:, cols]
# input()
# print(df)
return transform(df), ""
def get_lgt_share(self, market='SH', date=None):
url = "http://sc.hkexnews.hk/TuniS/www.hkexnews.hk/sdw/search/mutualmarket_c.aspx?t=%s" % (
market)
today = get_current_day()
if date is None:
date = today
data = {
'today': date_convert(today, "%Y-%m-%d", "%Y%m%d"),
'sortBy': 'stockcode',
'sortDirection': 'asc',
'alertMsg': '',
'txtShareholdingDate': date_convert(date, "%Y-%m-%d", "%Y/%m/%d"),
'btnSearch': "搜寻",
}
aspx_param = self.get_aspx_param(url)
data.update(aspx_param)
self.add_headers({
"Content-Type": "application/x-www-form-urlencoded",
})
rsp = self.do_request(url, data, "POST")
# 2. 开始解析返回数据，并从html中提取需要的内容
data = list()
soup = BeautifulSoup(rsp, "html5lib")
divs = soup.find_all('div')
result_date = ""
for div in divs:
if div.has_attr('id') and 'pnlResult' in div['id']:
res_span = div.find_all('span')
if (len(res_span)) > 0:
result_date = HKExAgent.clear_text(res_span[0].text)
result_date = result_date.replace("持股日期:", "").strip()
result_date = date_convert(result_date, "%Y/%m/%d",
"%Y-%m-%d")
rows = div.table.findAll('tr')
for row in rows:
cols = row.findAll('td')
if len(cols) == 4:
code = HKExAgent.clear_text(cols[0].text.replace(
"股份代号:", ""))
name = HKExAgent.clear_text(cols[1].text.replace(
"股份名称:", ""))
share_num = HKExAgent.clear_text(cols[2].text.replace(
"于中央结算系统的持股量:", ""))
percent = HKExAgent.clear_text(cols[3].text.replace(
"占于深交所上市及交易的A股总数的百分比:",
"").replace("占于上交所上市及交易的A股总数的百分比:", ""))
data.append({
"code":
HKExAgent.process_code(market, code),
"name":
name,
"share_num":
share_num,
"percent":
percent,
"date":
result_date,
})
df = pd.DataFrame(data)
return df
