def _parse_fund_data(url, fund_type='open'):
ct._write_console()
try:
request = Request(url)
text = urlopen(request, timeout=10).read()
if text == 'null':
return None
text = text.decode('gbk') if ct.PY3 else text
text = text.split('data:')[1].split(',exec_time')[0]
reg = re.compile(r'\,(.*?)\:')
text = reg.sub(r',"\1":', text)
text = text.replace('"{symbol', '{"symbol')
text = text.replace('{symbol', '{"symbol"')
if ct.PY3:
jstr = json.dumps(text)
else:
jstr = json.dumps(text, encoding='gbk')
org_js = json.loads(jstr)
fund_df = pd.DataFrame(pd.read_json(org_js, dtype={'symbol': object}),
columns=ct.NAV_COLUMNS[fund_type])
fund_df.fillna(0, inplace=True)
return fund_df
except Exception as er:
print(str(er))
def _get_nav_histroy_num(code, start, end, ismonetary=False):
"""
获取基金历史净值数量
--------
货币和证券型基金采用的url不同，需要增加基金类型判断
"""
ct._write_console()
if ismonetary:
request = Request(
ct.SINA_NAV_HISTROY_COUNT_CUR_URL %
(ct.P_TYPE['http'], ct.DOMAINS['ssf'], code, start, end))
else:
request = Request(
ct.SINA_NAV_HISTROY_COUNT_URL %
(ct.P_TYPE['http'], ct.DOMAINS['ssf'], code, start, end))
text = urlopen(request, timeout=10).read()
text = text.decode('gbk')
org_js = json.loads(text)
status_code = int(org_js['result']['status']['code'])
if status_code != 0:
status = str(org_js['result']['status']['msg'])
raise ValueError(status)
nums = org_js['result']['data']['total_num']
return int(nums)
def _get_nav_histroy_num(code, start, end, ismonetary=False):
"""
获取基金历史净值数量
--------
货币和证券型基金采用的url不同，需要增加基金类型判断
"""
ct._write_console()
if ismonetary:
request = Request(ct.SINA_NAV_HISTROY_COUNT_CUR_URL %
(ct.P_TYPE['http'], ct.DOMAINS['ssf'],
code, start, end))
else:
request = Request(ct.SINA_NAV_HISTROY_COUNT_URL %
(ct.P_TYPE['http'], ct.DOMAINS['ssf'],
code, start, end))
text = urlopen(request, timeout=10).read()
text = text.decode('gbk')
org_js = json.loads(text)
status_code = int(org_js['result']['status']['code'])
if status_code != 0:
status = str(org_js['result']['status']['msg'])
raise ValueError(status)
nums = org_js['result']['data']['total_num']
return int(nums)
def _parse_fund_data(url, fund_type='open'):
ct._write_console()
try:
request = Request(url)
text = urlopen(request, timeout=10).read()
if text == 'null':
return None
text = text.decode('gbk') if ct.PY3 else text
text = text.split('data:')[1].split(',exec_time')[0]
reg = re.compile(r'\,(.*?)\:')
text = reg.sub(r',"\1":', text)
text = text.replace('"{symbol', '{"symbol')
text = text.replace('{symbol', '{"symbol"')
if ct.PY3:
jstr = json.dumps(text)
else:
jstr = json.dumps(text, encoding='gbk')
org_js = json.loads(jstr)
fund_df = pd.DataFrame(pd.read_json(org_js, dtype={'symbol': object}),
columns=ct.NAV_COLUMNS[fund_type])
fund_df.fillna(0, inplace=True)
return fund_df
except Exception as er:
print(str(er))
def _get_fund_num(url, retry_count=3, pause=0.01, timeout=10):
"""
获取基金数量
"""
for _ in range(retry_count):
time.sleep(pause)
ct._write_console()
try:
request = Request(url)
text = urlopen(request, timeout=timeout).read()
text = text.decode('gbk')
if text == 'null':
raise ValueError('get fund num error')
text = text.split('((')[1].split('))')[0]
reg = re.compile(r'\,(.*?)\:')
text = reg.sub(r',"\1":', text)
text = text.replace('{total_num', '{"total_num"')
text = text.replace('null', '0')
org_js = json.loads(text)
nums = org_js["total_num"]
return int(nums)
except Exception as er:
print(str(er))
return 0
def _parse_nav_history_data(code,
start,
end,
nums,
ismonetary=False,
retry_count=3,
pause=0.01):
if (nums == 0):
return None
for _ in range(retry_count):
time.sleep(pause)
#try:
ct._write_console()
if ismonetary:
request = Request(
ct.SINA_NAV_HISTROY_DATA_CUR_URL %
(ct.P_TYPE['http'], ct.DOMAINS['ssf'], code, start, end, nums))
else:
request = Request(
ct.SINA_NAV_HISTROY_DATA_URL %
(ct.P_TYPE['http'], ct.DOMAINS['ssf'], code, start, end, nums))
text = urlopen(request, timeout=10).read()
text = text.decode('gbk')
js = json.loads(text)
status_code = int(js['result']['status']['code'])
if 0 != status_code:
status = str(js['result']['status']['msg'])
raise ValueError(status)
data = js['result']['data']['data']
if 'jjjz' in data[0].keys():
df = pd.DataFrame(data, columns=ct.NAV_HIS_JJJZ)
df['jjjz'] = df['jjjz'].astype(float)
df['ljjz'] = df['ljjz'].astype(float)
df['pre_jz'] = df['jjjz'].shift(1)
df['change'] = (df['jjjz'] / df['pre_jz'] - 1) * 100
df = df.drop('pre_jz', axis=1)
df.rename(columns=ct.DICT_NAV_EQUITY, inplace=True)
else:
df = pd.DataFrame(data, columns=ct.NAV_HIS_NHSY)
df['nhsyl'] = df['nhsyl'].astype(float)
df['dwsy'] = df['dwsy'].astype(float)
df.rename(columns=ct.DICT_NAV_MONETARY, inplace=True)
df.fillna(0, inplace=True)
if df['date'].dtypes == np.object:
df['date'] = df['date'].astype(np.datetime64)
df = df.set_index('date')
return df
raise IOError(ct.NETWORK_URL_ERROR_MSG)
def _parse_nav_history_data(code, start, end, nums, ismonetary=False, retry_count=3, pause=0.01, timeout=10):
if nums == 0:
return None
for _ in range(retry_count):
time.sleep(pause)
# try:
ct._write_console()
if ismonetary:
request = Request(ct.SINA_NAV_HISTROY_DATA_CUR_URL %
(ct.P_TYPE['http'], ct.DOMAINS['ssf'],
code, start, end, nums))
else:
request = Request(ct.SINA_NAV_HISTROY_DATA_URL %
(ct.P_TYPE['http'], ct.DOMAINS['ssf'],
code, start, end, nums))
text = urlopen(request, timeout=timeout).read()
text = text.decode('gbk')
org_js = json.loads(text)
status_code = int(org_js['result']['status']['code'])
if status_code != 0:
status = str(org_js['result']['status']['msg'])
raise ValueError(status)
data = org_js['result']['data']['data']
if 'jjjz' in data[0].keys():
fund_df = pd.DataFrame(data, columns=ct.NAV_HIS_JJJZ)
fund_df['jjjz'] = fund_df['jjjz'].astype(float)
fund_df['ljjz'] = fund_df['ljjz'].astype(float)
fund_df.rename(columns=ct.DICT_NAV_EQUITY, inplace=True)
else:
fund_df = pd.DataFrame(data, columns=ct.NAV_HIS_NHSY)
fund_df['nhsyl'] = fund_df['nhsyl'].astype(float)
fund_df['dwsy'] = fund_df['dwsy'].astype(float)
fund_df.rename(columns=ct.DICT_NAV_MONETARY, inplace=True)
#fund_df.fillna(0, inplace=True)
if fund_df['date'].dtypes == np.object:
fund_df['date'] = pd.to_datetime(fund_df['date'])
fund_df = fund_df.set_index('date')
fund_df = fund_df.sort_index(ascending=False)
fund_df['pre_value'] = fund_df['value'].shift(-1)
fund_df['change'] = (fund_df['value'] / fund_df['pre_value'] - 1) * 100
fund_df = fund_df.drop('pre_value', axis=1)
return fund_df
raise IOError(ct.NETWORK_URL_ERROR_MSG)
def _get_fund_num(url):
"""
获取基金数量
"""
ct._write_console()
try:
request = Request(url)
text = urlopen(request, timeout=10).read()
text = text.decode('gbk')
if text == 'null':
raise ValueError('get fund num error')
text = text.split('((')[1].split('))')[0]
reg = re.compile(r'\,(.*?)\:')
text = reg.sub(r',"\1":', text)
text = text.replace('{total_num', '{"total_num"')
text = text.replace('null', '0')
org_js = json.loads(text)
nums = org_js["total_num"]
return int(nums)
except Exception as er:
print(str(er))
def _get_detail(tag, retry_count=3, pause=0.001):
for _ in range(retry_count):
time.sleep(pause)
try:
ct._write_console()
request = Request(ct.SINA_DATA_DETAIL_URL%(ct.P_TYPE['http'],
ct.DOMAINS['vsf'], ct.PAGES['jv'],
tag))
text = urlopen(request, timeout=10).read()
text = text.decode('gbk')
except _network_error_classes:
pass
else:
reg = re.compile(r'\,(.*?)\:')
text = reg.sub(r',"\1":', text)
text = text.replace('"{symbol', '{"symbol')
text = text.replace('{symbol', '{"symbol"')
jstr = json.dumps(text)
js = json.loads(jstr)
df = pd.DataFrame(pd.read_json(js, dtype={'code':object}), columns=ct.THE_FIELDS)
df = df[ct.FOR_CLASSIFY_B_COLS]
return df
raise IOError(ct.NETWORK_URL_ERROR_MSG)
def _get_detail(tag, retry_count=3, pause=0.001):
for _ in range(retry_count):
time.sleep(pause)
try:
ct._write_console()
request = Request(
ct.SINA_DATA_DETAIL_URL %
(ct.P_TYPE['http'], ct.DOMAINS['vsf'], ct.PAGES['jv'], tag))
text = urlopen(request, timeout=10).read()
text = text.decode('gbk')
except _network_error_classes:
pass
else:
reg = re.compile(r'\,(.*?)\:')
text = reg.sub(r',"\1":', text)
text = text.replace('"{symbol', '{"symbol')
text = text.replace('{symbol', '{"symbol"')
jstr = json.dumps(text)
js = json.loads(jstr)
df = pd.DataFrame(pd.read_json(js, dtype={'code': object}),
columns=ct.THE_FIELDS)
df = df[ct.FOR_CLASSIFY_B_COLS]
return df
raise IOError(ct.NETWORK_URL_ERROR_MSG)
def _parse_nav_history_data(code,
start,
end,
nums,
ismonetary=False,
retry_count=3,
pause=0.01,
timeout=10):
if nums == 0:
return None
for _ in range(retry_count):
time.sleep(pause)
# try:
ct._write_console()
if ismonetary:
request = Request(
ct.SINA_NAV_HISTROY_DATA_CUR_URL %
(ct.P_TYPE['http'], ct.DOMAINS['ssf'], code, start, end, nums))
else:
request = Request(
ct.SINA_NAV_HISTROY_DATA_URL %
(ct.P_TYPE['http'], ct.DOMAINS['ssf'], code, start, end, nums))
text = httputil.get(request, timeout=timeout)
text = text.decode('gbk')
org_js = json.loads(text)
status_code = int(org_js['result']['status']['code'])
if status_code != 0:
status = str(org_js['result']['status']['msg'])
raise Not200Error(status)
data = org_js['result']['data']['data']
if 'jjjz' in data[0].keys():
fund_df = pd.DataFrame(data, columns=ct.NAV_HIS_JJJZ)
fund_df['jjjz'] = fund_df['jjjz'].astype(float)
fund_df['ljjz'] = fund_df['ljjz'].astype(float)
fund_df.rename(columns=ct.DICT_NAV_EQUITY, inplace=True)
else:
fund_df = pd.DataFrame(data, columns=ct.NAV_HIS_NHSY)
fund_df['nhsyl'] = fund_df['nhsyl'].astype(float)
fund_df['dwsy'] = fund_df['dwsy'].astype(float)
fund_df.rename(columns=ct.DICT_NAV_MONETARY, inplace=True)
#fund_df.fillna(0, inplace=True)
if fund_df['date'].dtypes == np.object:
fund_df['date'] = pd.to_datetime(fund_df['date'])
fund_df = fund_df.set_index('date')
fund_df = fund_df.sort_index(ascending=False)
fund_df['pre_value'] = fund_df['value'].shift(-1)
fund_df['change'] = (fund_df['value'] / fund_df['pre_value'] - 1) * 100
fund_df = fund_df.drop('pre_value', axis=1)
return fund_df
raise IOError(ct.NETWORK_URL_ERROR_MSG)