def catchError (code,date):
scanData = scanConfigDate['data']
scanData['lastScanStock'] = code
scanData['lastScanDate'] = date
scanData['currDate'] = dateStr
writeFile(scanFile, scanData, 'records', False)
sys.exit()
def stockIndustryClassified():
jsonFile = os.path.join(config.listsRootPath,
"stockIndustryClassified.json")
data = ts.get_industry_classified()
#data.to_json(jsonFile, orient='records', force_ascii =False)
writeFile(jsonFile, data, 'records', False)
def getHistData(stock=None):
if stock is None:
print "代码为空"
return {'errcode': -1, 'errmsg': "代码为空", 'data': ''}
else:
jsonDir = os.path.join(config.dataRootDailyTotal, stock)
if os.path.isdir(jsonDir):
pass
else:
os.mkdir(jsonDir)
filename = os.path.join(config.dataRootDailyTotal, stock,
"histroySummary3Y.json")
data = ts.get_hist_data(stock)
writeFile(filename, data, 'index', False)
print stock
def getHData(stock=None):
if stock is None:
print "代码为空"
return {'errcode': -1, 'errmsg': "代码为空", 'data': ''}
else:
jsonDir = os.path.join(config.dataRootDailyTotal, stock)
if os.path.isdir(jsonDir):
pass
else:
os.mkdir(jsonDir)
df = ts.get_stock_basics()
datestr = str(df.ix[stock]['timeToMarket']) # 上市日期YYYYMMDD
datelen = len(datestr)
timeToMarket = datestr[:4] + "-" + datestr[4:6] + "-" + datestr[6:]
print timeToMarket
filename = os.path.join(config.dataRootDailyTotal, stock,
"histroySummaryAllParts.json")
data = ts.get_h_data(stock, start='2010-01-08', end='2012-01-06')
writeFile(filename, data, 'index', False)
def getContent(self, filename, stock=None):
if os.path.isfile(filename):
reload(sys)
sys.setdefaultencoding('utf-8')
#soup = BeautifulSoup(open(filename), 'lxml') #按xml解析
soup = BeautifulSoup(open(filename), 'html') #按html格式解析
section = soup.select(".section")
gdrs = soup.select("#gdrs")
sdltgd = soup.select("#sdltgd")
sdgd = soup.select("#sdgd")
sdgdcgbd = soup.select("#sdgdcgbd")
jjcc = soup.select("#jjcc")
xsjj = soup.select("#xsjj")
#print soup
if stock is None:
print stock + " error"
return
##股东人数
if gdrs:
#gdrs = section[0]
table = gdrs[0].parent.select("table")
if len(table):
if table[0]:
rows = table[0].select("tr")
times = rows[0].select(".tips-dataL")
numbers = rows[1].select(".tips-dataL") #股东人数
pernumberchanges = rows[2].select(
".tips-dataL") #股东人数变化
pershares = rows[3].select(".tips-dataL") #人均流通持股
persharechanges = rows[4].select(
".tips-dataL") #人均流通股变化
scr = rows[5].select(
".tips-dataL") #筹码集中度 Stock Convergence Rate
prices = rows[6].select(".tips-dataL") #股价
perprice = rows[7].select(".tips-dataL") #人均持股金额
tenshare = rows[8].select(".tips-dataL") #十大股东
tencirshare = rows[9].select(".tips-dataL") #十大流通股东
data = dict()
data['gdrs'] = dict()
for i in range(1, len(times)):
idx = times[i].get_text().decode()
data['gdrs'][idx] = dict()
data['gdrs'][idx]['number'] = numbers[i].get_text(
).decode()
data['gdrs'][idx]['numcg'] = pernumberchanges[
i].get_text().decode()
data['gdrs'][idx]['pershare'] = pershares[
i].get_text().decode()
data['gdrs'][idx]['sharecg'] = persharechanges[
i].get_text().decode()
data['gdrs'][idx]['scr'] = scr[i].get_text(
).decode()
data['gdrs'][idx]['price'] = prices[i].get_text(
).decode()
data['gdrs'][idx]['perprice'] = perprice[
i].get_text().decode()
for i in tenshare:
t = i.get_text().decode()
if t is not None and t != "--":
data['tenshare'] = t
break
for i in tencirshare:
t = i.get_text().decode()
if t is not None and t != "--":
data['tencirshare'] = t
break
jsonFile = os.path.join(self.pageroot, stock)
jsonFile = os.path.join(jsonFile, "gdrs.json")
writeFile(jsonFile, data, 'records', False)
##十大流通股东
if sdltgd:
#sdltgd = section[1]
dateIndex = sdltgd[0].parent.select(".tab li")
tables = sdltgd[0].parent.select(".first table")
if len(dateIndex) == len(tables):
data = dict()
data['sdltgd'] = dict()
for i in range(0, len(dateIndex)):
idx = dateIndex[i].get_text().decode()
data['sdltgd'][idx] = dict()
table = tables[i]
tr = table.select("tr")
for t in range(1, len(tr)):
tds = tr[t].select("td")
data['sdltgd'][idx][t] = dict()
rank = tr[t].select("th")[0].get_text()
data['sdltgd'][idx][t]['rank'] = rank #名称
data['sdltgd'][idx][t]['name'] = tds[0].get_text(
).decode() #股东名称
data['sdltgd'][idx][t]['gdxz'] = tds[1].get_text(
).decode() #股东性质
data['sdltgd'][idx][t]['gflx'] = tds[2].get_text(
).decode() #股份类型
data['sdltgd'][idx][t]['cgs'] = tds[3].get_text(
).decode() #平均持股数
data['sdltgd'][idx][t]['cgbl'] = tds[4].get_text(
).decode() #持股比例
data['sdltgd'][idx][t]['zjgs'] = tds[5].get_text(
).decode() #增减股数
data['sdltgd'][idx][t]['bdbl'] = tds[6].get_text(
).decode() #变动比率
jsonFile = os.path.join(self.pageroot, stock)
jsonFile = os.path.join(jsonFile, "sdltgd.json")
writeFile(jsonFile, data, 'records', False)
##十大股东
if sdgd:
#sdgd = section[2]
dateIndex = sdgd[0].parent.select(".tab li")
tables = sdgd[0].parent.select(".first table")
if len(dateIndex) == len(tables):
data = dict()
data['sdgd'] = dict()
for i in range(0, len(dateIndex)):
idx = dateIndex[i].get_text().decode()
data['sdgd'][idx] = dict()
table = tables[i]
tr = table.select("tr")
for t in range(1, len(tr)):
tds = tr[t].select("td")
data['sdgd'][idx][t] = dict()
rank = tr[t].select("th")[0].get_text()
data['sdgd'][idx][t]['rank'] = rank # 排名
data['sdgd'][idx][t]['name'] = tds[0].get_text(
).decode() # 股东名称
data['sdgd'][idx][t]['gflx'] = tds[1].get_text(
).decode() # 股份类型
data['sdgd'][idx][t]['cgs'] = tds[2].get_text(
).decode() # 持股数
data['sdgd'][idx][t]['cgbl'] = tds[3].get_text(
).decode() # 占总股本持股比例
data['sdgd'][idx][t]['zjgs'] = tds[4].get_text(
).decode() # 增减股数
data['sdgd'][idx][t]['bdbl'] = tds[5].get_text(
).decode() # 变动比率
jsonFile = os.path.join(self.pageroot, stock)
jsonFile = os.path.join(jsonFile, "sdgd.json")
writeFile(jsonFile, data, 'records', False)
##十大股东持股变动
if sdgdcgbd:
#sdgdcgbd = section[3]
trs = sdgdcgbd[0].parent.select("tr")
data = dict()
data['sdgdcgbd'] = dict()
if len(trs) > 1:
for i in range(1, len(trs)):
rows = trs[i].select('.tips-dataL')
data['sdgdcgbd'][i] = dict()
data['sdgdcgbd'][i]['time'] = rows[0].get_text(
).decode()
data['sdgdcgbd'][i]['rank'] = rows[1].get_text(
).decode()
data['sdgdcgbd'][i]['gdmc'] = rows[2].get_text(
).decode()
data['sdgdcgbd'][i]['gflx'] = rows[3].get_text(
).decode()
data['sdgdcgbd'][i]['cgsl'] = rows[4].get_text(
).decode()
data['sdgdcgbd'][i]['zgbl'] = rows[5].get_text(
).decode()
data['sdgdcgbd'][i]['zjgs'] = rows[6].get_text(
).decode()
data['sdgdcgbd'][i]['zjzgbl'] = rows[7].get_text(
).decode()
data['sdgdcgbd'][i]['bdyy'] = rows[8].get_text(
).decode()
else:
data['sdgdcgbd']['errmsg'] = "没有数据"
jsonFile = os.path.join(self.pageroot, stock)
jsonFile = os.path.join(jsonFile, "sdgdcgbd.json")
writeFile(jsonFile, data, 'records', False)
##基金持仓
if jjcc:
#jjcc = section[4]
dateIndex = jjcc[0].parent.select(".tab li")
tables = jjcc[0].parent.select(".first table")
data = dict()
data['jjcc'] = dict()
if len(dateIndex) == len(tables):
for i in range(0, len(dateIndex)):
idx = dateIndex[i].get_text().decode()
data['jjcc'][idx] = dict()
table = tables[i]
tr = table.select("tr")
for t in range(1, len(tr)):
tds = tr[t].select("td")
data['jjcc'][idx][t] = dict()
if len(tds) > 7:
data['jjcc'][idx][t]['rank'] = tr[0].get_text(
).decode() # 排名
data['jjcc'][idx][t]['name'] = tds[1].get_text(
).decode() # 基金名称
data['jjcc'][idx][t]['cgs'] = tds[3].get_text(
).decode() # 持股数
data['jjcc'][idx][t]['cgsz'] = tds[4].get_text(
).decode() # 持仓市值
data['jjcc'][idx][t]['zgbl'] = tds[5].get_text(
).decode() # 占总股本比
data['jjcc'][idx][t]['ltbl'] = tds[6].get_text(
).decode() # 占流通比
data['jjcc'][idx][t]['jzbl'] = tds[7].get_text(
).decode() # 占净值比
else:
pass
else:
data['jjcc']['errmsg'] = "没有数据"
jsonFile = os.path.join(self.pageroot, stock)
jsonFile = os.path.join(jsonFile, "jjcc.json")
writeFile(jsonFile, data, 'records', False)
#限售解禁
if xsjj:
#xsjj = section[5]
trs = xsjj[0].parent.select("tr")
data = dict()
data['xsjj'] = dict()
if len(trs) > 5:
for i in range(1, len(trs)):
rows = trs[i].select('.tips-dataL')
data['xsjj'][i] = dict()
data['xsjj'][i]['time'] = rows[0].get_text().decode()
data['xsjj'][i]['jjsl'] = rows[1].get_text().decode()
data['xsjj'][i]['jjzgbl'] = rows[2].get_text().decode()
data['xsjj'][i]['jjltbl'] = rows[3].get_text().decode()
data['xsjj'][i]['type'] = rows[4].get_text().decode()
else:
data['xsjj']['errmsg'] = "没有数据"
jsonFile = os.path.join(self.pageroot, stock)
jsonFile = os.path.join(jsonFile, "xsjj.json")
writeFile(jsonFile, data, 'records', False)
else:
print "error"
pass
def lpr_ma_data(year=""):
jsonFile = os.path.join(config.listsRootPath, "lpr_ma_data.json")
data = ts.shibor_ma_data()
if str(type(data)) == "<class 'pandas.core.frame.DataFrame'>":
data.sort('date', ascending=False)
writeFile(jsonFile, data, 'records', False)
def getSuspended():
jsonFile = os.path.join(config.listsRootPath, "stockSuspended.json")
data = ts.get_suspended()
writeFile(jsonFile, data, 'records', False)
def getTerminated():
jsonFile = os.path.join(config.listsRootPath, "stockTerminated.json")
data = ts.get_terminated()
writeFile(jsonFile, data, 'records', False)