def crawlSecurityData_AtFront(daysCount):
securities = fd.get_all_securities()
min_date = dao.select("select min(date) min_date from security_data",
())[0]['min_date']
if min_date is not None:
endDate = fd.preOpenDate(min_date, 1)
else:
endDate = fd.getLastestOpenDate()
startDate = fd.preOpenDate(endDate, daysCount)
if startDate > endDate: return
for code in securities:
count = 0
df = ts.get_k_data(code, startDate, endDate)
arr_values = []
while count < df.index.__len__():
open = str(df['open'].values[count])
close = str(df['close'].values[count])
high = str(df['high'].values[count])
low = str(df['low'].values[count])
volume = str(df['volume'].values[count])
date = str(df['date'].values[count])
arr_values.append((code, date, open, close, high, low, volume))
count = count + 1
print("Date: " + date + " Code: " + code)
#dao.update("delete from security_data where code=%s", (code,))
dao.updatemany(
"insert into security_data(code, date, open, close, high, low, volume) values(%s,%s,%s,%s,%s,%s,%s)",
arr_values)
def get_zhangtingconcept_countMap(dayCount):
nowDate = fd.getLastestOpenDate()
#（1）获取所有concept的集合(大于1)
startDate = fd.preOpenDate(nowDate, dayCount)
rows = dao.select("select concept from zhangting_concept where date>=%s group by concept", (startDate))
concepts = []
for row in rows:
concept = row['concept']
concepts.append(concept)
#（2）迭代concept，获取count，在特定的date
# date = nowDate
# while date > startDate:
# rows = dao.select("select concept, count(0) count from zhangting_concept where date=%s group by concept", (date))
# for row in rows:
# concept = row['concept']
# count = row['count']
date = startDate
ret = {}
concept_count_rel = {}
for concept in concepts:
countsArr = []
_date = date
while _date <= nowDate:
row = dao.select("select count(0) count from zhangting_concept where date=%s and concept=%s", (_date, concept))
count = str(row[0]['count'])
countsArr.append(count)
_date = fd.nextOpenDate(_date, 1)
concept_count_rel.setdefault(concept, countsArr)
ret.setdefault('concept_count_rel', concept_count_rel)
ret.setdefault('concepts', concepts)
#（3）返回图标接受的数据model
# endDate = fd.getLastestOpenDate()
# startDate = fd.preOpenDate(endDate, dayCount)
# nowDate = startDate
# ret = {}
# while nowDate <= endDate:
# try:
# arr = dao.select("select count(0) count, concept from zhangting_concept where date = %s GROUP BY concept", (nowDate))
# except Exception as e:
# return "get_zhangtingconcept_countMap mysql error"
# map = {}
# for it in arr:
# count = it['count']
# concept = it['concept']
# map.setdefault(concept, count)
# ret[nowDate] = map
# nowDate = fd.nextOpenDate(nowDate, 1)
dates = []
_date = date
while _date <= nowDate:
dates.append(_date)
_date = fd.nextOpenDate(_date, 1)
ret.setdefault('dates', dates)
return ret
def getWencaiCodesForGettingDataAndSave2DB(date=fd.getLastestOpenDate()):
count = 0
while True:
if count < 151:
count = count + 1
ye_date = fds.preOpenDate(date, 1)
date = ye_date
continue
if count > DateCount:
break
ye_date = fds.preOpenDate(date, 1)
w = "非st；" + ye_date + "日均线角度>30；" + ye_date + "日涨跌幅>0；((" + date + "日开盘价-" + ye_date + "日收盘价)/" + ye_date + "日收盘价)<-0.03"
codes = hg.getCodesFromWencai(w)
print(w)
for code in codes:
df = ts.get_k_data(code, start=fds.preOpenDate(date, 20), end=date)
# 过滤问句因除权数据产生的杂音
open_chg = fds.get_open_chg(df, date)
if open_chg > -3:
continue
print("Date: " + str(date) + " ye_Date: "+ ye_date + " Code: " + str(code))
print("(1)getting df: ")
ye_chg = fds.get_ye_chg(df, date)
print("(2)getting ye_chg: " + str(ye_chg))
continuous_rise_day_count = fds.get_continuous_rise_day_count(df, date)
print("(3)getting continuous_rise_day_count: " + str(continuous_rise_day_count))
ye_qrr = fds.get_ye_qrr(df, date)
print("(4)getting ye_qrr: " + str(ye_qrr))
open_chg = fds.get_open_chg(df, date)
print("(5)getting open_chg: " + str(open_chg))
close_chg = fds.get_close_chg(df, date)
print("(6)getting close_chg: " + str(close_chg))
continuous_z_day_count = fds.get_continuous_z_day_count(df, date)
print("(7)getting continuous_z_day_count: " + str(continuous_z_day_count))
print("(8)storing 2 DB")
dao.appendRecord(df, date, ye_chg, continuous_rise_day_count, ye_qrr, 0, open_chg, close_chg, close_chg-open_chg)
print("-------------------------------------------------------")
count = count + 1
date = ye_date
#getWencaiCodesForGettingDataAndSave2DB("2018-07-20")
def getZhangTingCodeConceptAnd2DB(date=fd.getLastestOpenDate()):
soups = hg.getSoupsFromWencai(date + "日涨跌幅>=9.89；" + date +"涨停原因；按"+date+"日首次涨停时间排序")
for soup in soups:
eles_codes = soup.select('#resultWrap .static_con_outer .tbody_table tr td.item div.em')
index = 0
codes = []
while index < eles_codes.__len__():
o_str = eles_codes[index].text.strip()
if (o_str.isdigit()):
codes.append(o_str)
index = index + 1
eles_reason = soup.select('#resultWrap .scroll_tbody_con .tbody_table tr td[colnum="5"] div.em')
reasons = []
for elem in eles_reason:
o_str = elem.text.strip()
reasons.append(o_str)
count = 0
for code in codes:
r = reasons[count]
dao.update("delete from zhangting_concept where date=%s and code=%s", (date, code))
dao.update("insert into zhangting_concept(code, date, concept) values(%s,%s,%s)", (code, date, r))
count = count + 1
def crawlSecurityData_AtRear(dayCount):
securities = fd.get_all_securities()
max_date = dao.select("select max(date) max_date from security_data",
())[0]['max_date']
endDate = fd.getLastestOpenDate()
if max_date is not None:
startDate = fd.nextOpenDate(max_date, 1)
else:
startDate = fd.preOpenDate(endDate, dayCount)
isFromHist = False
if startDate > endDate: return
for code in securities:
count = 0
df = ts.get_k_data(code, startDate, endDate)
if df.index.__len__() == 0:
isFromHist = True
df = ts.get_hist_data(code, startDate, endDate)
arr_values = []
while count < df.index.__len__():
open = str(df['open'].values[count])
close = str(df['close'].values[count])
high = str(df['high'].values[count])
low = str(df['low'].values[count])
volume = str(df['volume'].values[count])
if isFromHist is True:
date = df.index[count]
else:
date = str(df['date'].values[count])
arr_values.append((code, date, open, close, high, low, volume))
count = count + 1
print("Date: " + date + " Code: " + code)
dao.update("delete from security_data where code=%s and date=%s",
(code, date))
dao.updatemany(
"insert into security_data(code, date, open, close, high, low, volume) values(%s,%s,%s,%s,%s,%s,%s)",
arr_values)