def update_profit(): cur_year = time.localtime().tm_year df_profit = pd.concat( [ts.profit_data(cur_year, 'all'), ts.profit_data(cur_year - 1, 'all')]) df_profit.set_index(['code', 'year'], inplace=True) df_profit.to_sql('profit', engine, if_exists='replace')
def load_to_csv_profit2014_2017(): df_profit2014 = ts.profit_data(year=2014, top=3000) df_profit2015 = ts.profit_data(year=2015, top=3000) df_profit2016 = ts.profit_data(year=2016, top=3000) df_profit2017 = ts.profit_data(year=2017, top=3000) df_profit2014.to_csv('CSV/profit2014.csv') df_profit2015.to_csv('CSV/profit2015.csv') df_profit2016.to_csv('CSV/profit2016.csv') df_profit2017.to_csv('CSV/profit2017.csv') df_profit1 = pd.merge(df_profit2014, df_profit2015, how='outer', on='code', suffixes=[2014, 2015]) df_profit2 = pd.merge(df_profit1, df_profit2016, how='outer', on='code', suffixes=["", ""]) df_profit = pd.merge(df_profit2, df_profit2017, how='outer', on='code', suffixes=["2016", 2017]) df_profit.to_csv('CSV/profit2014-2017.csv') return df_profit
def stockfh(): df = ts.profit_data(year=2016, top=1000) df.to_excel('d:/pythontest/stock/股票2016分红.xlsx') df = ts.profit_data(year=2017, top=1000) df.to_excel('d:/pythontest/stock/股票2017分红.xlsx') df = ts.profit_data(year=2018, top=1000) df.to_excel('d:/pythontest/stock/股票2018分红.xlsx') df = ts.profit_data(year=2019, top=1000) df.to_excel('d:/pythontest/stock/股票2019分红.xlsx')
def get_fpya(): df = ts.profit_data(top=3500) df.to_sql('fpya', engine, if_exists='replace', index=False, index_label='code')
def HighShares(t=50): df = ts.profit_data() if len(df)<2: return [u"高转送股票信息:",u"无!"] else: df=df.sort_values('shares',ascending=False) return [u"高转送股票信息:",df[:t]]
def getProfitData(cursor): for i in range(1992, 2017 + 1): try: df = ts.profit_data(top=1000, year=i) # 处理缺失值 df = df.fillna(0) df = df.stack().replace('--', '').unstack() dfLen = len(df) uuidList = [] # 添加uuid yearList = [] # 添加年份 for l in range(0, dfLen): uuidList.append(uuid.uuid1()) yearList.append(str(i)) df['uuid'] = uuidList df['year'] = yearList for k in range(0, dfLen): df2 = df[k:k + 1] cursor.execute( "insert into stock_profit_data(uuid, code, name, year, report_date, divi, shares) " "values('%s', '%s', '%s', '%s','%s', '%d', '%d')" % (str(list(df2['uuid'])[0]), str(list(df2['code'])[0]), str(list(df2['name'])[0]), str(list( df2['year'])[0]), str(list(df2['report_date'])[0]), round(float(df2['divi']), 4), round(float(df2['shares']), 4))) cursor.execute("commit") except Exception as e: pass print(e)
def update_db_dividend_data(cls): tbl_dividend_data = 'dividend_data' # get the start date result = cls.engine.execute("select max(year) from %s" % tbl_dividend_data) last_year = result.fetchone()[0] if last_year is None: start_year = 2005 else: start_year = last_year + 1 # get the end year end_year = datetime.datetime.now().year if start_year >= end_year: start_year = end_year print('start year:' + str(start_year) + ' ; end year:' + str(end_year)) # get the profit data for y in range(end_year, start_year, -1): dividend_data = ts.profit_data(year=y, top=4000) print("Dividend data at year:%s" % y) # insert data to database cls.insert_to_db_no_duplicate(dividend_data, tbl_dividend_data) print("Updated, done") # close the engine pool cls.engine.dispose() return "Update dividend, Done!"
def getProfit(self, year=0, top=10, shares=10, divi=1): df = ts.profit_data(year=year, top=top) # df.sort('shares', ascending=False) print (df.sort_values(by='divi', ascending=False)) df1 = ts.inst_detail() print (df1) return df[df.divi >= divi]
def get_data_date(year): cmd = ''' select * from %s ''' % (table_name) print cmd df = pd.read_sql(sql=cmd, con=conn) #从mysql中读取指定年份的数据 rs = ts.profit_data(year=year, top=3000) #从网上获取近3000条分配预案 last_df = pd.merge(rs, df, how='left', on=['code', 'report_date'], left_index=False, right_index=False, sort=False, suffixes=('_x', '_y'), copy=True) #进行左连接,股票代码和发布如期唯一确定一行 for i in range(len(last_df)): last_df.loc[i, 'flag'] = math.isnan(float( last_df.loc[i, 'divi_y'])) #新加一列flag last_df = last_df[last_df.flag] #剔除重复的行,插入数据库 #剔除不需要的列 del last_df['name_y'] del last_df['year_y'] del last_df['divi_y'] del last_df['shares_y'] del last_df['flag'] last_df.columns = [ 'code', 'name', 'year', 'report_date', 'divi', 'shares' ] #重新修改列名 return last_df
def get_investment_reference_data(self): df = ts.profit_data(year=2017, top=60, retry_count=3, pause=0) if(df is not None): return df else: return -1
def profit(mechanism1): list = [] df = ts.profit_data(top=10, year=datetime.now().year - 1).sort_values( 'shares', ascending=False) df.apply(lambda row: list.append(row.to_csv()) if is_today(row['report_date']) == True else None, axis=1) return list
def profit_data(year): profit_data = ts.profit_data(year=year, top=100) profit_data.sort('shares', ascending=False) if profit_data is not None: profit_data.to_sql('reference_profit_data', engine, flavor='mysql', if_exists='append')
def download_divi_data( self, year, top ): try: df_tmp = ts.profit_data( year, top ) except: ERROR( 'exception occurs when update divi data of year {0}'.format( year ) ) return pd.DataFrame() else: LOG( 'update divi data of year {0}'.format( year ) ) thread_queue.put( df_tmp )
def get_profit_report(): df = ts.profit_data(top=4000) df = df.sort_values('divi', ascending=False) #获取最新股价 df_quots = ts.get_today_all() df_quots['roe'] = df_quots['pb'] * 100 / df_quots['per'] #df_basic['peg'] = df_basic['pe']/df_basic['profit'] df_quots['close'] = map(swap_value, df_quots['trade'], df_quots['settlement']) df_quots.reset_index(inplace=True) df_quots = df_quots.drop([ 'index', 'name', 'changepercent', 'trade', 'open', 'high', 'low', 'settlement', 'volume', 'turnoverratio', 'amount', 'nmc' ], axis=1) df = pd.merge(df, df_quots, how='left', on=key) df['rate'] = df['divi'] / 10 * 100 / df['close'] df['valueprice'] = df['roe'] * (df['close'] / df['pb']) / 15 df = df.sort_values('rate', ascending=False) #df['value'] = df['esp']/0.08 #df['rate'] = (df['value']-df['close'])*100/df['close'] df = df[df['per'] > 0] df = df.sort_values('report_date', ascending=False) df = df.drop_duplicates(['name']) # 按现金股息率排行 df = df.sort_values('rate', ascending=False) export_report(df, title="stock_dividence") # 按roe排行 df = df.sort_values('roe', ascending=False) export_report(df, title="stock_roe") # 按市盈率 df = df.sort_values('per', ascending=True) export_report(df, title="stock_pe") # 按市净率 df = df.sort_values('pb', ascending=True) export_report(df, title="stock_pb") # 不排序 df = df.sort_values('mktcap', ascending=True) export_report(df, title="stock_value")
def get_red(): """分配预案 year : 预案公布的年份,默认为2014 top :取最新n条数据,默认取最近公布的25条 retry_count:当网络异常后重试次数,默认为3 pause:重试时停顿秒数,默认为0 """ df = ts.profit_data(year=2020, top=60) # df.sort('shares', ascending=False) print(df)
def _getRevenuData(self,year,top=100): res = ts.profit_data(year=year,top=top) return res
def getvrdata(isinit='no'): mylogger = getmylogger() curday = datetime.date.today() print(curday) curyear = curday.year curmonth = curday.month quarter = (curmonth - 1) // 3 + 1 # prdict = {'profitdivi': '分配预案', 'forecast': '业绩预告', 'xsg': '限售股解禁', 'fundholdings': '基金持股', 'newstocks': '新股'} prdict = {'fundholdings': '基金持股'} for dv in prdict: vrtbname = dv vrinfo = prdict[vrtbname] try: if dv == "profitdivi": df = ts.profit_data(year=curyear, top=100) if isinit == 'no' and df is not None: df = df[df['report_date'] >= str(curday)] elif dv == "forecast": df = ts.forecast_data(curyear, quarter) if isinit == 'no' and df is not None: # print(df[df['report_date'].isin(['2019-12-28'])]) df = df[df['report_date'] >= str(curday)] elif dv == "xsg": df = ts.xsg_data() if isinit == 'no' and df is not None: curym = datetime.date.today().strftime("%Y-%m") df = df[df['date'].str.contains(curym)] elif dv == "fundholdings": if quarter == 1: quarter = 4 curyear -= 1 else: quarter -= 1 df = ts.fund_holdings(curyear, quarter) if isinit == 'no' and df is not None: lastyq = str(curyear) + '-' + str(quarter) df = df[df['date'].str.contains(lastyq)] if df is not None: newcolumns = ['nums', 'count', 'nlast', 'name', 'amount', 'date', 'ratio', 'code', 'clast'] df.columns = newcolumns elif dv == "newstocks": df = ts.new_stocks() if isinit == 'no' and df is not None: df = df[df['ipo_date'] >= str(curday)] else: mylogger.info("没有执行命令。") if df is not None: tosql(df, vrtbname, "append", vrinfo, mylogger) else: mylogger.info("没有%s数据。" % vrinfo) except Exception: tracelog = traceback.format_exc() mylogger.info("获取数据异常。") mylogger.info(tracelog)
def updatedistribute(): distributedatalist=ts.profit_data('2013') distributedata=pd.DataFrame(distributedatalist) conn= ms.connect(host='localhost',port = 3306,user='******', passwd='123456',db ='investment',charset="utf8") cur = conn.cursor() values=[] for index,row in distributedata.iterrows(): values.append((row['code'],row['name'],row['year'],row['report_date'],row['divi'],row['shares'])) cur.executemany('insert into distribute (code,name,year,report_date,divi,shares) values(%s,%s,%s,%s,%s,%s)',values) conn.commit() cur.close() conn.close()
def ai_gzscx(self): #分配预案的次新股 df = ts.profit_data(top=1000, year=datetime.now().year - 1) #df = df[df.shares >= 1] df.sort_values(by='shares', ascending=False) df['code'] = df['code'].astype(int) dfCX = set(df['code'].values) pd = ts.get_industry_classified() pd['code'] = pd['code'].astype(int) # print(pd[pd.c_name == '次新股']) # print(set(pd['c_name'].values)) setCX = set(pd[pd.c_name == '次新股']['code'].values) print('\n') print(df[df.code.isin(setCX)])
def updaterPofitData(con, year: int = None, top: int = 10000): import share.model.dao.reference.ProfitData as Model if year is None: year = datetime.now().year logging.debug("Updating PofitData of year {} top {}".format(year, top)) df = ts.profit_data(year=year, top=top, retry_count=16) res = [] for _, row in df.iterrows(): obj = Model.rowToORM(row) if obj is not None: res.append(obj) Base.metadata.create_all(con.engine) con.save_all(res) return
def override_share_div(cls, start): # start = 1990 sd = pd.DataFrame() while True: try: print('\n', start) df = ts.profit_data(year=start, top='all') if df is None or df.empty: break df.sort_values('report_date') sd = pd.concat([sd, df]) start -= 1 except Exception as e: print(e) break DMGR.save(sd, 'macro', 'share_div')
def job_2(): try: print("I'm working......投资参考数据") # 分配预案 profit_data = ts.profit_data(year, top=1000) data = pd.DataFrame(profit_data) data.to_sql('profit_data',engine,index=True,if_exists='replace') print("分配预案......done") # 业绩预告 forecast_data = ts.forecast_data(year,1) data = pd.DataFrame(forecast_data) data.to_sql('forecast_data',engine,index=True,if_exists='replace') print("业绩预告......done") # 限售股解禁 xsg_data = ts.xsg_data() data = pd.DataFrame(xsg_data) data.to_sql('xsg_data',engine,index=True,if_exists='replace') print("限售股解禁......done") # 基金持股 fund_holdings = ts.fund_holdings(year, 1) data = pd.DataFrame(fund_holdings) data.to_sql('fund_holdings',engine,index=True,if_exists='replace') print("基金持股......done") # 新股数据 new_stocks = ts.new_stocks() data = pd.DataFrame(new_stocks) data.to_sql('new_stocks',engine,index=True,if_exists='replace') print("新股数据......done") # 融资融券(沪市) sh_margins = ts.sh_margins() data = pd.DataFrame(sh_margins) data.to_sql('sh_margins',engine,index=True,if_exists='replace') print("融资融券(沪市)......done") # 融资融券(深市) sz_margins = ts.sz_margins() data = pd.DataFrame(sz_margins) data.to_sql('sz_margins',engine,index=True,if_exists='replace') print("融资融券(深市)......done") except Exception as e: print(e)
def get_dividend_plan(year, top=3000): try: if os.path.exists(DIVIDEND_PLAN_DIR) is False: os.makedirs(DIVIDEND_PLAN_DIR) df = ts.profit_data(year, top) df.to_csv( os.path.join(DIVIDEND_PLAN_DIR, '{yyyy}.csv'.format(yyyy=year))) # df['createDate'] = '{yyyy}{mm}{dd}'.format(yyyy=str(now.year), # mm=str(now.strftime('%m')), # dd=str(now.strftime('%d'))) engine = connection.create_db_engine() df.to_sql('dividend_plan', engine, if_exists='append', index=False) except IntegrityError as error: print(error) else: print( 'Success: {yyyy} year dividend_plan downloaded.'.format(yyyy=year))
def profit(engine, year, quarter=None): tbl = "invest_profit" tsl.log(tbl + " start...") try: df = ts.profit_data(year, 'all') if quarter is not None: qd = tsu.get_quarter_date(year, quarter) df = df[df.report_date >= qd[0]] df = df[df.report_date <= qd[1]] df = df.set_index('code', drop='true') df.to_sql(tbl, engine, if_exists='append') print tsl.log(tbl + " done") except BaseException, e: print print e tsl.log(tbl + " error")
def get_ref_profit(year, top): """ 投资参考 系列 分配预案:股票的送转和分红预案 year : 预案公布的年份,默认为2014 top :取最新n条数据,默认取最近公布的25条 """ df = ts.profit_data(year, top) print(df) if df is not None: if len(df) > 0: if year is None: year = 2014 df.insert(0, 'year', year) res = df.to_sql(invest_stock_ref_profit, engine, if_exists='replace') msg = 'ok' if res is None else res print('获取股票的送转和分红预案数据: {0}: {1}'.format(year, msg) + '\n') else: print('获取股票的送转和分红预案数据: {0}: {1}'.format(year, 'None') + '\n')
def updatedistribute(): distributedatalist = ts.profit_data('2013') distributedata = pd.DataFrame(distributedatalist) conn = ms.connect(host='localhost', port=3306, user='******', passwd='123456', db='investment', charset="utf8") cur = conn.cursor() values = [] for index, row in distributedata.iterrows(): values.append((row['code'], row['name'], row['year'], row['report_date'], row['divi'], row['shares'])) cur.executemany( 'insert into distribute (code,name,year,report_date,divi,shares) values(%s,%s,%s,%s,%s,%s)', values) conn.commit() cur.close() conn.close()
def profit_data(request): try: year = request.POST.get('year', 2014) # 选填 top = request.POST.get('top', 25) # 选填 data = ts.profit_data(int(year), int(top)) res = {'columns': [ 'code', 'name', 'year', 'report_date', 'divi', 'shares' ], 'data': json.loads(json.dumps(data.values, cls=DateEncoder))} except(TypeError): return HttpResponse(TypeError) else: return HttpResponse(json.dumps(res),content_type="application/json")
def get_distribution_plans_data(year, top=25, retry_count=RETRY_COUNT, pause=PAUSE): """得到分配预案数据 Args: year : 预案公布的年份,默认为2014 top :取最新n条数据,默认取最近公布的25条 retry_count:当网络异常后重试次数,默认为3 pause:重试时停顿秒数,默认为0 Returns: data_dicts: 字典的列表, 具体的字段及值, 如果没有返回空列表, 如果接口出错返回None """ logger.info('Begin get distribution plans, the year is %s' % year) try: data_df = ts.profit_data(year=year, top=top, retry_count=retry_count, pause=pause) except Exception as e: logger.exception('Error get distribution plans, the year is %s' % year) return None else: data_dicts = [] if data_df is None or data_df.empty: logger.warn('Empty get distribution plans, the year is %s' % year) else: data_dicts = [{ 'year': year, 'code': row[0], 'name': row[1], 'report_date': row[3], 'divi': row[4], 'shares': row[5], 'insert_date': today_line } for row in data_df.values] logger.info('Success get distribution plans, the year is %s' % year) return data_dicts
def update_db_dividend_data(self): #create db engine engine = self.create_db_engine(self.str_db_investment_data) #set the table name table_dividend_data = self.table_creator.get_table_dividend_data() table_dividend_data.create(engine, checkfirst=True) #create table print("Create table:%s ok!" % (table_dividend_data.name)) #get the start date result = engine.execute( "select max(%s) from %s" % (table_dividend_data.c.year, table_dividend_data.name)) last_year = result.fetchone()[0] if last_year == None: start_year = 2005 else: start_year = last_year + 1 #get the end year end_year = datetime.datetime.now().year if (start_year >= end_year): start_year = end_year print('start year:' + str(start_year) + ' ; end year:' + str(end_year)) #get the profit data for n in range(start_year, end_year): dividend_data = ts.profit_data(year=n, top=4000) print("Dividend data at year:%s" % n) #print(dividend_data) #insert data to database self.insert_to_db_no_duplicate(dividend_data, table_dividend_data.name, engine) #close the engine pool engine.dispose()
def k_profit(_db): begin = get_micro_second() top = 1000 try: df = ts.profit_data(top=top) except Exception: log_error("warn:error: %s profit_data() exception!", top) return -4 # calc cost time log_info("profit_data [%s] costs %d us", top, get_micro_second()-begin) if df is None : log_error("warn: df is None, next") return -1 if df.empty: log_error("warn: df is empty, next") return -2 if len(df) <= 1: log_error("warn: df is empty, next") return -3 # pd.options.display.max_rows = 1000 log_debug(df) begin = get_micro_second() k_profit_to_db(df, _db) log_info("to_db costs %d us", get_micro_second() - begin) return
import tushare as ts import easyquotation import csv from datetime import datetime as dt source = easyquotation.use('sina') # current price cp = source.all # profit data year = dt.now().year previous_year = year - 1 year_before_previous_year = previous_year - 1 pd_ybpy = ts.profit_data(top='all', year=year_before_previous_year) pd_py = ts.profit_data(top='all', year=previous_year) pr_list = [] def _cal_profit_ratio(code, price, profit_data): try: pds = profit_data[profit_data['code'] == code] total_divis_per_share = 0 if len(pds) == 0: return 0, None for pd in pds.iterrows(): shares = pd[1]['shares'] divis = pd[1]['divi'] divis_per_share = divis / (10 + shares) total_divis_per_share += divis_per_share
def get_history_high_transfer(self): df = ts.profit_data(top=1500) df.index = df['code'] return df
# -*- coding: utf-8 -*- """ Created on Thu Oct 27 15:17:29 2016 @author: colec """ import tushare as ts import csv import codecs preDivi_csv = open('PreDiviResult.csv', 'wb+') spamwriter = csv.writer(preDivi_csv,dialect='excel') spamwriter.writerow(['stock', 'name', 'year', 'report_date', "divi", "shares"]) df = ts.profit_data(top=1000, year=2016) for code, stockRow in df.iterrows(): print("pre divi " + stockRow['code']) print(type(stockRow['shares'])) spamwriter.writerow([stockRow['code'], stockRow['name'], stockRow['year'], stockRow['report_date'], stockRow["divi"], stockRow["shares"]]) preDivi_csv.close() preEarning_csv = open('PreEarningResult.csv', 'wb+') spamwriter1 = csv.writer(preEarning_csv,dialect='excel') spamwriter1.writerow(['stock', 'name', 'type', 'report_date', "pre_eps", "range"])
""" import os import sys import tushare as ts from code.pub.common import DATA_PATH, PATH_SEP from tushare.fund import cons as ct from tushare.util import dateu as du reload(sys) sys.setdefaultencoding('utf-8') today = du.today() profit_data_file = DATA_PATH + PATH_SEP + 'profit_data' + today + '.csv' # 删除一周前同一天文件,如果天天运行,则只保留一周数据 del_file = DATA_PATH + PATH_SEP + 'profit_data' + du.day_last_week() + '.csv' if os.path.exists(del_file): os.remove(del_file) if not os.path.exists(profit_data_file): df = ts.profit_data(year=today[:4]) if df.loc[0, 'code'] == '000nan': df = ts.profit_data(year=int(today[:4]) - 1) df.to_csv(profit_data_file, sep=',', header=True, index=False, encoding='GBK') if __name__ == '__main__': pass
def profit_data(self, conns, top): self.base = Base() self.financial_data = conns['financial_data'] self.df = ts.profit_data(top) self.base.batchwri(self.df, 'fenpeiyuan', self.financial_data)
def year_profit_date(year): DTS=ts.profit_data(year=year,top=3000) return DTS
def profit_test(self): df = ts.profit_data(top=60) print(df.sort_value(by='shares', ascending=False))
# c_name df = ts.get_concept_classified() df.to_sql('concept_data',engine, if_exists='replace') # =================================================bonus data================================================= # import bonus data #code:股票代码 #name:股票名称 #year:分配年份 #report_date:公布日期 #divi:分红金额(每10股) #shares:转增和送股数(每10股) df = ts.profit_data(year=lastYear,top=1000) df.to_sql('bonus_data',engine, if_exists='replace') # =================================================company data================================================== # import basic data #code,代码 #name,名称 #industry,所属行业 #area,地区 #pe,市盈率 #outstanding,流通股本(亿) #totals,总股本(亿) #totalAssets,总资产(万) #liquidAssets,流动资产 #fixedAssets,固定资产
def Get_profit_data(self): # 获取分配预案 df = ts.profit_data(top=60) df = df.sort_values(by=['shares'], ascending=False) print(df)
#coding=utf-8 ''' Created on 2015年6月4日 @author: Administrator ''' import tushare as ts # 分配预案 #每到季报、年报公布的时段,就经常会有上市公司利润分配预案发布,而一些高送转高分红的股票往往会成为市场炒作的热点。 df = ts.profit_data(top=60) df.sort('shares',ascending=False) df[df.shares>=10]#选择每10股送转在10以上的 # 业绩预告 ts.forecast_data(2014,2)#获取2014年中报的业绩预告数据 # 限售股解禁 # 以月的形式返回限售股解禁情况,通过了解解禁股本的大小,判断股票上行的压力。可通过设定年份和月份参数获取不同时段的数据。 ts.xsg_data() # 新股数据 ts.new_stocks() # 融资融券 ts.sh_margins(start='2015-01-01', end='2015-04-19') ts.sz_margins(start='2015-01-01', end='2015-04-19') #如果不设symbol参数或者开始和结束日期时段设置过长,数据获取可能会比较慢,建议分段分步获取,比如一年为一个周期 ts.sh_margin_details(start='2015-01-01', end='2015-04-19', symbol='601989') ts.sz_margin_details('2015-04-20')
def reference_info(self, stock): data = ts.profit_data(top=60) data.sort('shares',ascending=False) print data[data.shares>=10]