def add_transactions():
df = pd.read_excel('data/transactions.xlsx')
r = []
for line in df.itertuples():
if line.target != 'cash':
r.append([
line.trade_date, line.account, 'cash', 1,
-1 * (line.price * line.num * 1 + line.fee_rate), 0
])
dft = pd.DataFrame(r, columns=df.columns)
df = pd.concat([df, dft], ignore_index=True)
# with open('data/transactions.txt') as f:
# ll = f.readlines()
# ll = [s.strip()[2:-2].split(', ') for s in ll]
# cols = 'trade_date account target price num fee_rate'.split()
# r = []
# for i in ll:
# # 买卖成本在买入时都算进去，卖出时不算成本
# t = [datetime.now(), i[3], i[0][:-1], float(i[2]), float(i[1]), 0.003]
# if t[2] == 'cash' or t[-2] < 0: # 卖出时不算成本
# t[-1] = 0
# r.append(t)
# if t[2] != 'cash':
# # 每增加一笔not cash，就要加1笔-cash
# r.append([t[0], t[1], 'cash', 1, -t[3]*t[4]*(1+t[5]), 0])
# df = pd.DataFrame(r, columns=cols)
print(df)
if not df.empty:
mongo = PyMongoWrapper()
mongo.insertDataframe(df, 'finance', 'transactions')
else:
print('no transactions!')
def get_last_day_market_val(target, tdate):
"""
获取最近一日市值
:param target:
:param tdate:
'20190908'
:return:
"""
if target == 'cash':
return datetime.now().date(), 1
if any([target.endswith('.SZ'), target.endswith('.SH')]):
tname = target
else:
tname = series.loc[target]
mongo = PyMongoWrapper()
db_name = 'finance_n'
tb_name = 'stocks_daily'
table = mongo.getCollection(db_name, tb_name)
r = list(
mongo.findAll(table, {
'ts_code': tname,
'trade_date': {
'$lte': datetime.strptime(tdate, '%Y%m%d')
}
},
fieldlist=['trade_date', 'close', 'pct_chg'],
sort=[('trade_date', -1)],
limit=1))[0]
return r['trade_date'].date(), r['close']
def get_stocks_dict_top_n(cls, n, t_date):
mongo = PyMongoWrapper()
table = mongo.getCollection('finance_n', 'stock_daily_basic')
df = mongo.findAll(table, {'trade_date': pd.to_datetime(t_date)}, fieldlist=['ts_code'],
sort=[('rank', 1)], limit=n, returnFmt='df')
stock_reverse_dict = cls.get_stocks_dict()[1]
return df.ts_code.map(lambda c: stock_reverse_dict[c]).tolist()
def get_stocks_dict(cls):
mongo = PyMongoWrapper()
table = mongo.getCollection('finance_n', 'stocks_info')
df = mongo.findAll(table, fieldlist=['ts_code', 'name'], returnFmt='df')
stock_names = df.name.tolist()
stock_ts_codes = df.ts_code.tolist()
return dict(zip(stock_names, stock_ts_codes)), dict(zip(stock_ts_codes, stock_names))
def r(o, s):
if o == 'cash':
return CE().date(), 1
if Cu([o.endswith('.SZ'), o.endswith('.SH')]):
b = o
else:
b = M.loc[o]
W = PyMongoWrapper()
m = W.getCollection('finance', b)
r = Cw(
W.findAll(m, {'trade_date': {
'$lte': Cn(s, '%Y%m%d')
}},
fieldlist=['trade_date', 'close', 'pct_chg'],
sort=[('trade_date', -1)],
limit=1))[0]
return r['trade_date'].date(), r['close']
def c():
df = X('data/transactions.xlsx')
r = []
for j in df.itertuples():
if j.target != 'cash':
r.append([
j.trade_date, j.account, 'cash', 1,
-1 * (j.price * j.num * 1 + j.fee_rate), 0
])
d = B(r, columns=df.columns)
df = CS([df, d], ignore_index=Cy)
CT(df)
if not df.empty:
W = PyMongoWrapper()
W.insertDataframe(df, 'finance', 'transactions')
else:
CT('no transactions!')
def add_transactions():
df = U('data/transactions.xlsx')
r = []
for I in df.itertuples():
if I.target != 'cash':
r.append([
I.trade_date, I.account, 'cash', 1,
-1 * (I.price * I.num * 1 + I.fee_rate), 0
])
A = w(r, columns=df.columns)
df = r([df, A], ignore_index=mV)
mE(df)
if not df.empty:
K = PyMongoWrapper()
K.insertDataframe(df, 'finance', 'transactions')
else:
mE('no transactions!')
def get_last_day_market_val(T, D):
if T == 'cash':
return e().date(), 1
if mF([T.endswith('.SZ'), T.endswith('.SH')]):
s = T
else:
s = i.loc[T]
K = PyMongoWrapper()
v = K.getCollection('finance', s)
r = X(
K.findAll(v, {'trade_date': {
'$lte': c(D, '%Y%m%d')
}},
fieldlist=['trade_date', 'close', 'pct_chg'],
sort=[('trade_date', -1)],
limit=1))[0]
return r['trade_date'].date(), r['close']
def get_hist(cls, name=None, ts_code=None, count=365):
if not ts_code:
stocks_dict = cls.get_stocks_dict()[0]
ts_code = stocks_dict[name]
# start_date = datetime.now() - timedelta(days=count)
mongo = PyMongoWrapper()
table = mongo.getCollection('finance_n', 'stocks_daily')
# df = mongo.findAll(table, {'trade_date': {'$gte': start_date}},
# fieldlist=['trade_date', 'pct_chg'], returnFmt='df')
df = mongo.findAll(table, {'ts_code': ts_code}, fieldlist=['trade_date', 'pct_chg'],
sort=[('trade_date', -1)], limit=count, returnFmt='df')
df.set_index('trade_date', inplace=True)
df.sort_index(inplace=True)
df['net'] = (1 + df['pct_chg'] / 100).cumprod()
del df['pct_chg']
# print(df.head())
# print(df.tail())
return df
def main():
tweepyWrapper = TweepyWrapper()
tweepyWrapper.print_user_information()
tweepyWrapper.print_all_my_followers()
tweepyWrapper.print_most_influential_followers()
tweepyWrapper.print_all_tweets_from_me()
tweepyWrapper.print_retweets_of_me()
#allFollowerTweetsWhichContainStick2Me = tweepyWrapper.get_follower_tweets_which_contain_Stick2Me()
#tweepyWrapper.print_all_tweets_which_contain_Stick2Me()
pyMongoWrapper = PyMongoWrapper()
#pyMongoWrapper.insert_all_tweets(allFollowerTweetsWhichContainStick2Me)
#pyMongoWrapper.print_all_collection_tweets()
collectionNames = pyMongoWrapper.get_all_collection_names()
"""
Basic data analytics
"""
for collectionName in collectionNames:
PrintHelper.print_header("Collection name: " + collectionName)
pyMongoWrapper.print_collection(collectionName)
""""
NLTK
"""
nltkWrapper = NLTKWrapper()
#targetCollectionName = "tweets.From_2017_02_08_To_2017_02_15"
for collectionName in collectionNames:
PrintHelper.print_header("Collection name: " + collectionName)
tweetTextSummaryFromCollection = pyMongoWrapper.get_tweet_text_summary_for_collection(
collectionName)
nltkWrapper.analize_text(collectionName,
tweetTextSummaryFromCollection)
"""
SNA
"""
targetCollectionName = "tweets.all"
for collectionName in collectionNames:
if collectionName in targetCollectionName:
PrintHelper.print_header("Collection name: " + collectionName)
vertexes = pyMongoWrapper.get_tweet_vertexes_for_collection(
collectionName)
SNAWrapper(collectionName, vertexes).create_sna_analysis_html()
# print(df_ac.groupby(['type'])['market_val'].sum())
# print(f'profit: {df_ac.profit.sum()}')
# s = df_ac.groupby(['type', 'target'])['market_val'].sum()
s = df_ac.groupby('account')['market_val'].sum()
# s = df_ac.groupby('type')['market_val'].sum()
s.loc['TOTAL'] = s.sum()
dfr = pd.DataFrame(s).T
dfr.index.name = 'date'
dfr.index = pd.to_datetime([tdate])
return dfr
if __name__ == '__main__':
# add_transactions()
transaction_list = []
mongo = PyMongoWrapper()
table = mongo.getCollection('finance', 'transactions')
cols = 'trade_date account target price num fee_rate'.split()
rs = mongo.findAll(table, fieldlist=cols, sort=[('trade_date', 1)])
trans_df = pd.DataFrame(rs)
trans_df.set_index('trade_date', inplace=True)
trans_df['cost'] = trans_df['price'] * trans_df['num'] * (
1 + trans_df.fee_rate)
dfc = pd.DataFrame()
dates = pd.date_range('20190921', '20190926', freq='B')
dfc = get_account_val(trans_df, '20190930')
# for d in dates:
# d = d.strftime('%Y%m%d')
# if dfc.empty:
sys.path.append(
'/Users/luoyonggui/PycharmProjects/mayiutils_n1/mayiutils/finance')
from stock_wrapper import get_tushare_pro
import argparse
from datetime import datetime, timedelta
import sys
sys.path.append(
'/Users/luoyonggui/PycharmProjects/mayiutils_n1/mayiutils/config')
from logging_utils import get_logger
log_path = os.path.join(LOG_PATH, 'output.log')
logger = get_logger(__file__, file_handler=True, log_path=log_path)
mongo = PyMongoWrapper()
dbname = 'finance'
table_name = 'stock_daily_basic1'
table = mongo.getCollection(dbname, table_name)
if not mongo.isExists(dbname, table_name):
mongo.setUniqueIndex(dbname, table_name, ['ts_code', 'trade_date'])
pro = get_tushare_pro()
f = 'ts_code,trade_date,close,turnover_rate,turnover_rate_f,volume_ratio,pe,pe_ttm,pb,ps,ps_ttm,total_share,float_share,free_share,total_mv,circ_mv'
# df = pro.daily_basic(ts_code='', trade_date=datetime.now().strftime('%Y%m%d'), fields=f)
df = pro.daily_basic(ts_code='', trade_date='20190926', fields=f)
# df = pro.daily_basic(ts_code='', trade_date='20190827', fields=f)
if not df.empty:
logger.info(f'请求到{len(df)}条数据！')
df.columns = [
'ts_code', 'trade_date', 'close', '换手率（%）', '换手率（自由流通股）', '量比',
'市盈率（总市值/净利润）', '市盈率（TTM）', '市净率（总市值/净资产）', '市销率', '市销率（TTM）', '总股本',
import sys
sys.path.append('/Users/luoyonggui/PycharmProjects/mayiutils_n1/mayiutils/db')
from pymongo_wrapper import PyMongoWrapper
mongo = PyMongoWrapper()
LOG_PATH = '/Users/luoyonggui/Documents/logs'
# 投资组合
portfolio = {'stock': [
('东方航空', 15500),
('中国核电', 8300),
('中信证券', 1500),
('兴业银行', 1800),
('中国石油', 1700),
],
'fund': [
('510300.SH', 7200),
('510500.SH', 3200),
('159938.SZ', 9500),
('159920.SZ', 1000),
('518880.SH', 3200),
('512580.SH', 15800),
('159915.SZ', 4100),
('512800.SH', 6000),
('512880.SH', 2000),
('513050.SH', 5000),
],
'otc_fund': [
('建信中证500指数增强A', 13699.71+9028.49),
('富国沪深300指数增强', 13704.73),
('富国中证红利指数增强', 21081.72+7133.1+11245.7),
('兴全可转债混合', 17831.75),
"""
import pandas as pd
import os
PROJ_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
from datetime import datetime
import sys
sys.path.append(
'/Users/luoyonggui/PycharmProjects/mayiutils_n1/mayiutils/finance')
from tushare_wrapper import TushareWrapper
sys.path.append('/Users/luoyonggui/PycharmProjects/mayiutils_n1/mayiutils/db')
from pymongo_wrapper import PyMongoWrapper
mongo = PyMongoWrapper()
ts = TushareWrapper()
# 查询当前所有正常上市交易的股票列表
# df = ts.get_stocks_info()
# print(df.head())
# mongo.insertDataframe(df, 'finance_n', 'stocks_info')
# 获取股票历史数据
# start_date = '20110101'
# end_date = datetime.now().strftime('%Y%m%d')
db_name = 'finance_n'
tb_name = 'stocks_daily'
# table = mongo.getCollection(db_name, tb_name)
# mongo.setUniqueIndex(db_name, tb_name, ['trade_date', 'ts_code'])
# count = 3611
# import time
def get_stocks_to_mongo(stocks, mode='stock'):
import pandas as pd
stock_series = pd.read_pickle(
os.path.join(PROJ_DIR, 'finance/data/stock_dict.pkl'))
mongo = PyMongoWrapper()
db = mongo.getDb('finance')
tables = db.list_collection_names()
start_date = '20010101'
end_date = datetime.now().strftime('%Y%m%d')
for name in stocks:
if mode == 'stock':
ts_code = stock_series.loc[name]
table = mongo.getCollection('finance', ts_code)
if ts_code not in tables:
logger.info(f'{ts_code}表在数据库中不存在，新建表插入！')
if mode in ['stock', 'index', 'fund', 'otc_fund']:
mongo.setUniqueIndex('finance', ts_code, ['trade_date'])
else:
logger.info(f'{ts_code}表在数据库中存在，update表数据！')
if mode in ['stock', 'index', 'fund']:
r = mongo.findAll(table,
fieldlist=['trade_date'],
sort=[('trade_date', -1)],
limit=1)
r = list(r)
if len(r) > 0:
d = r[0]['trade_date']
start_date = (d + timedelta(days=1)).strftime('%Y%m%d')
try:
if mode in ['stock', 'index', 'fund']:
df = daily(ts_code,
start_date=start_date,
end_date=end_date,
mode=mode)
elif mode in ['otc_fund']:
import requests
import json
import pandas as pd
from pyquery import PyQuery as pq
headers = {
'user-agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36 Maxthon/5.2.1.6000',
}
dd = dict()
r = requests.get(f'http://fund.eastmoney.com/{ts_code}.html')
r.encoding = 'utf8'
root = pq(r.text)
d1 = root(
'#body > div:nth-child(12) > div > div > div.fundDetail-main > div.fundInfoItem > div.dataOfFund > dl.dataItem02 > dd.dataNums > span.ui-font-middle.ui-num'
)
dd['pct_chg'] = float(d1.text()[:-1])
d1 = root(
'#body > div:nth-child(12) > div > div > div.fundDetail-main > div.fundInfoItem > div.dataOfFund > dl.dataItem02 > dd.dataNums > span.ui-font-large.ui-num'
)
dd['close'] = float(d1.text())
d1 = root(
'#body > div:nth-child(12) > div > div > div.fundDetail-main > div.fundInfoItem > div.dataOfFund > dl.dataItem02 > dt > p'
)
dd['trade_date'] = datetime.strptime(d1.text()[-11:-1],
'%Y-%m-%d')
df = pd.DataFrame(columns=['trade_date', 'close', 'pct_chg'])
df = df.append(dd, ignore_index=True)
df.set_index('trade_date', inplace=True)
print(df)
# print(d, dd, df)
logger.info(f'共请求到{df.shape[0]}条数据！')
if df.shape[0] > 0:
mongo.insertDataframe(df,
'finance',
ts_code,
df_index='trade_date')
else:
logger.warning('请求数据为空！')
except Exception as e:
logger.error(str(e))
mongo.close()
CR((Y.account == 'ZLT') & (Y.target != 'cash'), 'FUND',
CR((Y.account.isin(['TTJ', 'TTA'])) & (Y.target != 'cash'),
'OTC_FUND', 'cash')))
Y['market_val'] = Y.num * Y.price
Y['profit'] = Y['market_val'] - Y['cost']
s = Y.groupby('account')['market_val'].sum()
s.loc['TOTAL'] = s.sum()
l = B(s).T
l.index.name = 'date'
l.index = CI([s])
return l
if __name__ == '__main__':
q = []
W = PyMongoWrapper()
m = W.getCollection('finance', 'transactions')
A = 'trade_date account target price num fee_rate'.split()
rs = W.findAll(m, fieldlist=A, sort=[('trade_date', 1)])
J = B(rs)
J.set_index('trade_date', inplace=Cy)
J['cost'] = J['price'] * J['num'] * (1 + J.fee_rate)
e = B()
N = Co('20190921', '20190926', freq='B')
for d in N:
d = d.strftime('%Y%m%d')
if e.empty:
e = H(J, d)
else:
e = CS([e, H(J, d)])
CT(e)
import sys
sys.path.append('/Users/luoyonggui/PycharmProjects/mayiutils_n1/mayiutils/db')
from pymongo_wrapper import PyMongoWrapper
sys.path.append('/Users/luoyonggui/PycharmProjects/mayiutils_n1/mayiutils')
from email_ops import send_email
import html_ops
PROJ_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
stock = '山鹰纸业'
stock_series = pd.read_pickle(
os.path.join(PROJ_DIR, 'finance/data/stock_dict.pkl'))
fund_otc_series = pd.read_pickle(
os.path.join(PROJ_DIR, 'finance/data/fund_otc_series.pkl'))
mongo = PyMongoWrapper()
time_range = 'near_1_year'
start_date = datetime.now() - timedelta(days=365 * 5)
# 基本面 area、industry、sw_industry、市值、市值排名
table = mongo.getCollection('finance', 'stock_basic')
df_base = mongo.findAll(table, {'ts_code': stock_series.loc[stock]},
fieldlist=['name', 'area', 'industry', 'market'],
returnFmt='df')
# print(df)
table = mongo.getCollection('finance', 'swclass')
dft = mongo.findAll(
table,
{'股票代码': stock_series.loc[stock].map(lambda s: s.split('.')[0]).tolist()},
fieldlist=['股票名称', '行业名称'],
returnFmt='df')
l((Q.account == 'ZLT') & (Q.target != 'cash'), 'FUND',
l((Q.account.isin(['TTJ', 'TTA'])) & (Q.target != 'cash'),
'OTC_FUND', 'cash')))
Q['market_val'] = Q.num * Q.price
Q['profit'] = Q['market_val'] - Q['cost']
s = Q.groupby('account')['market_val'].sum()
s.loc['TOTAL'] = s.sum()
B = w(s).T
B.index.name = 'date'
B.index = R([D])
return B
if __name__ == '__main__':
f = []
K = PyMongoWrapper()
v = K.getCollection('finance', 'transactions')
n = 'trade_date account target price num fee_rate'.split()
rs = K.findAll(v, fieldlist=n, sort=[('trade_date', 1)])
J = w(rs)
J.set_index('trade_date', inplace=mV)
J['cost'] = J['price'] * J['num'] * (1 + J.fee_rate)
C = w()
P = N('20190921', '20190926', freq='B')
for d in P:
d = d.strftime('%Y%m%d')
if C.empty:
C = get_account_val(J, d)
else:
C = r([C, get_account_val(J, d)])
mE(C)