def update_stockbase_Equ(self,fields):
col_name='stockbase'
stkif=StockInterface(self.token)
db=DatabaseInterface()
tickers=db.get_db_tickers()
for t in tickers:
#获取字典
Equ_dics=stkif._getEqu(fields,t)[0]
db.update_db(self.db_name,col_name,{"ticker":t},{"$set":Equ_dics})
print 'update stockbase data with ticker.no'+t+'....'
def update_stockbase_SecTips(self,tips):
col_name='stockbase'
fields='ticker'
stkif=StockInterface(self.token)
db=DatabaseInterface()
tickers_dic=stkif._getSecTips([fields],tips)
tickers=self.unpack_dic(fields,tickers_dic)
updates={"$set":{'tipsTypeCD':tips}}
for t in tickers:
print t+'today turn to'+tips
db.update_db(self.db_name,col_name,{'ticker':t},updates)
def write_revenuplan(self,year,top=100):
today=self.today_as_str()
file_name=str(year)+'revenu plan'+today+'.csv'
api=StockInterface(self.token)
res=api._getRevenuData(self,year=year,top=top)
self.write_df_csv(self,res,file_name)
def __init__(self,token):
self.token=token
self.db_name='stock'
self.stkif=StockInterface(token)
class StockGetAll(object):
def __init__(self,token):
self.token=token
self.db_name='stock'
self.stkif=StockInterface(token)
#将一个字典列表中，所有指定key的值导出
def unpack_dic(self,key,diclist):
#key，指定键值，str
#diclist，字典list，如[{dic1},{dic2}]
#为防万一，使用如果key不存在于某个字段会报错
try:
res=[i[key] for i in diclist]
except KeyError:
print '字典中key值不存在'
sys.exit()
#此种方法即使key不存在于list中某个字典，会跳过这个字典继续输出
#res=[i[key] for i in diclist if key in i]
return res
#获取当日日期的str格式
def today_as_str(self,date_format="%Y%m%d"):
now = datetime.datetime.now()
todate=now.strftime(date_format)
return todate
#获取所有当日ticker列表
#ticker满足以下条件：
#1.交易所范围在exchages=['XSHG','XSHE']，即沪深两市
#2.企业在上市状态
#3.包含停牌企业
#4.A股
def get_api_tickers(self):
key='ticker'
fields=[key]
res=self.stkif._getEqu(fields)
tickers=self.unpack_dic(key,res)
return tickers
#获取currentinfo表所需数据
def get_api_currentinfo(self):
res=self.get_api_tickers()
currentinfo={}
currentinfo['currentTickerList']=res
currentinfo['tickerDate']=None
currentinfo['stockbaseDate']=None
currentinfo['stockfundsDate']=None
currentinfo['stocktradeDate']=None
currentinfo['stocktradestep']=-1
currentinfo['stockbasestep']=-1
return [currentinfo]
#获取stockbase表全部数据
def get_api_stockbase_all(self,step):
#获取当前全部股票代码
db=DatabaseInterface()
tickers=db.get_db_tickers()
tickers=tickers[step+1:]
return self.get_api_stockbase(tickers)
#获取股票基本信息，登记stockbase
#涉及3个接口：
#1.getEqu，获取股票基本信息
#2.getEquIndustry，获取行业信息
#3.getSecTips，停牌复牌信息
def get_api_stockbase(self,tickers):
#定义接口的抓取fields
# exchangeCD交易市场
# ListSectorCD上市板块编码
# ListSector上市板块
# secShortName证券简称
# listDate上市日期
# equTypeCD股票分类编码
# equType股票类别
# totalShares总股本(最新)
# nonrestFloatShares公司无限售流通股份合计(最新)
# nonrestfloatA无限售流通股本(最新)。如果为A股，该列为最新无限售流通A股股本数量；如果为B股，该列为最新流通B股股本数量
# primeOperating主营业务范围
Equ_fields=['exchangeCD','ListSectorCD','ListSector','secShortName',
'listDate','equTypeCD','equType','totalShares',
'nonrestFloatShares','nonrestfloatA','primeOperating']
#使用申银万国行业标准
EquIndustry_fields=['industry','industryID1','industryID2',
'industryID3','industryID4','industryName1','industryName2',
'industryName3','industryName4']
#获取tickers列表
for t in tickers:
Equ_dics=self.stkif._getEqu(Equ_fields,t)[0]
EquIndustry_dics=self.stkif._getEquIndustry(t,EquIndustry_fields)[0]
hub_dics=Equ_dics.copy()
hub_dics.update(EquIndustry_dics)
hub_dics['tipsTypeCD']='R'
hub_dics['ticker']=t
print t+'ticker get.........'
yield hub_dics
#获取全部股票复权交易信息,ts接口
#1.ts复权数据接口
#2.数据库stockbase三级行业ID信息
def get_db_stocktradeadj(self,step,beginDate,endDate=''):
#如果没有输入截止日期，就取得今天的日期
if not endDate:
endDate=self.today_as_str()
#获取tickers列表
db=DatabaseInterface()
tickers=db.get_db_tickers()
#获取上次中断步数
tickers=tickers[step+1:]
print 'started from ticker: '+tickers[0]
col_name='stockbase'
for t in tickers:
ticker_trade={}
ticker_trade['ticker']=t
ticker_trade['tradedata']=self.stkif._getTradeDataAdj(t,beginDate=beginDate,endDate=endDate)
ind=db.getone_db(self.db_name,col_name,{"ticker":t},['industryID3'])
ticker_trade['industryID3']=ind['industryID3']
print 'get ticker:'+t+'.....'
yield ticker_trade
def get_db_fundstocksinfo(self,beginYear,endYear=''):
#如果没有输入截止年，就取得今年的年份
if not endYear:
endYear=int(self.today_as_str()[:4])
years=range(beginYear,endYear+1)
quaters=range(1,5)
#循环取出起始年到截止年的所有是个季度的数据dataframe
#如果季度数据还没有出来，就取得None
frames=[self.stkif._getFundStocksData(y,q) for y,q in its.product(years,quaters)]
#数据处理，处理成需要的数据库格式
#数据拼合
funddata=pd.concat(frames)
#防止row index出现重复的情况
m,n=funddata.shape
funddata.index=range(m)
print 'data combine finished......'
#按照股票代码分组
funddata_groups=funddata.groupby(by=funddata['code'])
#每一组转成字典格式
for code,df in funddata_groups:
sub_funddata={}
sub_funddata['ticker']=code
sub_funddata['secShortName']=df.iloc[0,1]
#获取每个日期的记录
data=df.iloc[:,2:]
sub_funddata['data']=json.loads(data.T.to_json()).values()
yield sub_funddata
#获取全部股票交易信息,通联接口
#涉及1个接口：
#1.getMktEqud，获取股票交易信息
#2.getEquIndustry，获取行业信息
#3.getSecTips，停牌复牌信息
def get_db_stocktrade(token):
now = datetime.datetime.now()
todate=now.strftime("%Y%m%d")
beginDate='20130101'
ticker_fields=['secShortName','ticker','exchangeCD']
industry_fields=['industry','industryID1','industryID2','industryID3','industryID4','industryName1','industryName2','industryName3','industryName4']
stock_fields=['tradeDate','closePrice','negMarketValue','MarketValue','turnoverVol','turnoverValue','dealAmount','PE']
ticker_gene=get_ticker_gene(MONGODB_DB,'stockbase',ticker_fields)
if ticker_gene.count()==0:
print '全部数据已经抓取完毕'
sys.exit()
else:
print ticker_gene.count()
connection = pymongo.MongoClient(
MONGODB_URI,MONGODB_PORT
)
db = connection[MONGODB_DB]
collection = db['stockbase']
try:
for t in ticker_gene:
try:
res_trade=_getMktEqud(token,t['ticker'],stock_fields,beginDate=beginDate,endDate=todate,tradeDate='')
res_industry=_getEquIndustry(token,t['ticker'],industry_fields)
except Exception, e:
raise e
collection.update({"_id":t['_id']},{"$set":{"isupdate":1}})
if res_trade is not None:
t.update(res_industry[0])
t['data']=res_trade
yield t
except Exception, e:
#traceback.print_exc()
raise e
