class DataCrawl:
def __init__(self):
self.dm = DBManager("wm_details")
self.headers = {
"User-Agent":
":Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36"
}
def start_crawl(self):
print("-start up-")
_year, _season = get_cur_season()
self.get_url(_year, _season)
def end_crawl(self):
self.dm.close_db()
def get_url(self, year, season):
code_list = self.dm.get_code_list()
for item in code_list:
key = item["code"][:6]
url = "http://quotes.money.163.com/trade/lsjysj_" + key + ".html?year=" + year + "&season=" + season
print(url)
max_try = 8
for tries in range(max_try):
try:
content = requests.get(url)
self.parse_pager(content.content, item["code"])
break
except Exception:
if tries < (max_try - 1):
sleep(2)
continue
else:
print("ERROR TO DOWNLODE DATE")
code_list.close()
def parse_pager(self, content, key):
try:
_result = self.dm.find_by_id(key)
timer_list = [x["cur_timer"] for x in _result["price_list"]]
soup = bs4.BeautifulSoup(content, "lxml")
parse_list = soup.select("div.inner_box tr")
for item in parse_list[1:]:
data = [x.string for x in item.select("td")]
price = {
"cur_timer": data[0],
"cur_open_price": data[1],
"cur_max_price": data[2],
"cur_min_price": data[3],
"cur_close_price": data[4],
"cur_price_range": data[6],
"cur_total_volume": data[7],
"cur_total_money": data[8]
}
if price["cur_timer"] not in timer_list:
self.dm.add_tk_item(key, price)
print(key, "success")
except Exception:
print(key, "faild")
def update_data(self):
print("-redis_start-")
dm = DBManager("wm_details")
code_list = dm.get_code_list()
for item in code_list:
try:
code = item["code"][:6]
_result = dm.find_by_id(item["code"])
sorted_result = sorted(_result["price_list"],
key=lambda x: cmp_datatime(x),
reverse=True)
self.set_data(code, sorted_result)
except Exception:
print("redis_error:501 " + str(item["code"]))
print("-redis_close-")
def update_data(self):
add_info_logs("redis_start", "-开始同步缓存-")
dm = DBManager("tk_details")
code_list = dm.get_code_list()
for item in code_list:
try:
code = item["code"][:6]
_result = dm.find_by_id(item["code"])
sorted_result = sorted(_result["price_list"],
key=lambda x: cmp_datatime_02(x),
reverse=True)
self.set_data(code, sorted_result)
except Exception:
add_error_logs("redis_error", "501", item["code"])
continue
add_info_logs("redis_close", "-结束同步缓存-")
class WmacdUtils:
def __init__(self):
self.db_manager_wm = DBManager("wm_details")
self.db_manager_tk = DBManager("tk_details")
# 初始化wmacd数据
def init_w_time(self):
# 初始化时间轴
date_list = date_range("2016-01-04", "2018-12-30")
# tk_details = self.dm.find_by_key({"code": code, "cur_timer": {"$in": cur_date_list}})
code_list = self.db_manager_wm.get_code_list()
for code_item in code_list:
code = code_item["code"]
print(code)
tk_result = self.db_manager_tk.find_by_key({"code": code})[0]
tk_details = sorted(tk_result["price_list"],
key=lambda x: cmp_datatime_02(x),
reverse=False)
for index in range(len(date_list)):
if datetime.datetime.strptime(date_list[index],
"%Y-%m-%d").weekday() == 0:
cur_date_list = date_list[index:index + 7]
# 从数据库中获取这个时间段内的数据
cur_tk_details = [
x for x in tk_details
if x["cur_timer"] in cur_date_list
]
try:
open_price_list = [
float(x["cur_open_price"]) for x in cur_tk_details
]
max_price_list = [
float(x["cur_max_price"]) for x in cur_tk_details
]
min_price_list = [
float(x["cur_min_price"]) for x in cur_tk_details
]
close_price_list = [
float(x["cur_close_price"]) for x in cur_tk_details
]
total_volume_list = [
int(x["cur_total_volume"].replace(",", ""))
for x in cur_tk_details
]
total_money_list = [
int(x["cur_total_money"].replace(",", ""))
for x in cur_tk_details
]
except Exception as e:
continue
if cur_tk_details:
wmacd_item = {
"frist_date": cur_date_list[0],
"date_list": cur_date_list,
"open_price": open_price_list[0],
"max_price": max(max_price_list),
"min_price": min(min_price_list),
"close_price": close_price_list[-1],
"total_volume": sum(total_volume_list),
"total_money": sum(total_money_list),
}
else:
wmacd_item = {
"frist_date": cur_date_list[0],
"date_list": cur_date_list,
"open_price": 0,
"max_price": 0,
"min_price": 0,
"close_price": 0,
"total_volume": 0,
"total_money": 0,
}
# 在数据库中添加一条记录
self.db_manager_wm.add_tk_item(code, wmacd_item)
def update_w_macd(self, cur_date=datetime.datetime.now().date()):
date_list = date_range("2016-01-04", "2018-12-30")
for index in range(len(date_list)):
# 匹配到当前时间所在的区间
if datetime.datetime.strptime(date_list[index],
"%Y-%m-%d").weekday() == 0:
cur_date_list = date_list[index:index + 7]
if str(cur_date) in cur_date_list:
ticker_list = self.db_manager_wm.get_code_list()
# 更新每支股票的数据
for tk_item in ticker_list:
code = tk_item["code"]
tk_result = self.db_manager_tk.find_by_key(
{"code": code})[0]
tk_details = sorted(tk_result["price_list"],
key=lambda x: cmp_datatime_02(x),
reverse=False)
# 从数据库中获取这个时间段内的数据
cur_tk_details = [
x for x in tk_details
if x["cur_timer"] in cur_date_list
]
open_price_list = [
float(x["cur_open_price"]) for x in cur_tk_details
]
max_price_list = [
float(x["cur_max_price"]) for x in cur_tk_details
]
min_price_list = [
float(x["cur_min_price"]) for x in cur_tk_details
]
close_price_list = [
float(x["cur_close_price"]) for x in cur_tk_details
]
total_volume_list = [
int(x["cur_total_volume"].replace(",", ""))
for x in cur_tk_details
]
total_money_list = [
int(x["cur_total_money"].replace(",", ""))
for x in cur_tk_details
]
if cur_tk_details:
wmacd_item = {
"frist_date": cur_date_list[0],
"date_list": cur_date_list,
"open_price": open_price_list[0],
"max_price": max(max_price_list),
"min_price": min(min_price_list),
"close_price": close_price_list[-1],
"total_volume": sum(total_volume_list),
"total_money": sum(total_money_list),
}
# 修改数据库中的数据
self.db_manager_wm.update_wm_price_list(
code, wmacd_item["frist_date"], wmacd_item)
class TsStrategy5f(object):
def __init__(self):
self.db_manager_wm = DBManager("fcr_w_details")
def get_result(self, ticker):
if isinstance(ticker,
tkWMacdBean) and len(ticker.get_wmacd_list()) > 30:
if ticker.get_wmacd_list()[-1] > 0 >= ticker.get_wmacd_list()[-2]:
if 0.1 > ticker.get_diff_list()[-1] > 0:
if np.mean(ticker.get_tur_list()
[-5:-1]) < ticker.get_tur_list()[-1]:
return 1
return -1
# 排除一些虚假信号
def get_result2(self, ticker):
if isinstance(ticker,
tkWMacdBean) and len(ticker.get_wmacd_list()) > 30:
if ticker.get_wmacd_list()[-1] > 0 >= ticker.get_wmacd_list()[-2]:
if 0.1 > ticker.get_diff_list()[-1] > 0:
if np.mean(ticker.get_tur_list()
[-5:-1]) < ticker.get_tur_list()[-1]:
if ticker.get_wmacd_list(
)[-3] < 0 and ticker.get_wmacd_list(
)[-4] < 0 and ticker.get_wmacd_list()[-5] < 0:
return 1
return -1
def get_result3(self, ticker):
if isinstance(ticker,
tkWMacdBean) and len(ticker.get_wmacd_list()) > 30:
if ticker.get_wmacd_list()[-1] > 0 >= ticker.get_wmacd_list()[-2]:
if 0.1 > ticker.get_diff_list()[-1] > 0:
if np.mean(ticker.get_tur_list()
[-5:-1]) < ticker.get_tur_list()[-1]:
if ticker.get_wmacd_list(
)[-3] < 0 and ticker.get_wmacd_list(
)[-4] < 0 and ticker.get_wmacd_list()[-5] < 0:
return 1
return -1
def get_buy_list(self, date):
code_list = self.db_manager_wm.get_code_list()
buy_list = list()
for item in code_list:
try:
code = item["code"]
price_list = list()
tur_list = list()
highest_list = list()
open_list = list()
date_list = list()
# 获取wmacd数据
tk_details = self.db_manager_wm.find_by_key({"code": code})[0]
for tk_item in [
x for x in tk_details["price_list"] if x["close"] != 0
]:
if time_cmp(str(date), tk_item["date"]):
date_list.append(tk_item["date"])
price_list.append(float(tk_item["close"]))
tur_list.append(float(tk_item["volume"]))
highest_list.append(float(tk_item["high"]))
open_list.append(float(tk_item["open"]))
wmacd_list, diff_list, dea_list = self.get_w_macd(
price_list[:])
# 创建wmacd实体
tk_bean = tkWMacdBean(code, price_list, wmacd_list, diff_list,
dea_list, tur_list, highest_list,
open_list)
if self.get_result2(tk_bean) == 1:
buy_list.append(code)
except Exception as e:
continue
return buy_list
# 计算当前时间段的wmacd值
def get_w_macd(self, price_list):
ema_12_list = list()
for index in range(len(price_list)):
if index == 0:
ema_12_list.append(price_list[0])
else:
ema_12_list.append(
round(
ema_12_list[index - 1] * 11 / 13 +
price_list[index] * 2 / 13, 4))
ema_26_list = list()
for index in range(len(price_list)):
if index == 0:
ema_26_list.append(price_list[0])
else:
ema_26_list.append(
round(
ema_26_list[index - 1] * 25 / 27 +
price_list[index] * 2 / 27, 4))
diff_list = list()
for index in range(len(ema_12_list)):
diff = ema_12_list[index] - ema_26_list[index]
diff_list.append(diff)
dea_list = list()
for index in range(len(diff_list)):
if index == 0:
dea_list.append(diff_list[0])
else:
dea_list.append(
round(dea_list[index - 1] * 0.8 + diff_list[index] * 0.2,
4))
wmacd_list = list()
for index in range(len(dea_list)):
bar = (diff_list[index] - dea_list[index]) * 3
wmacd_list.append(bar)
return wmacd_list, diff_list, dea_list
class TsStrategyExpma:
def __init__(self):
self.db_manager_wm = DBManager("wm_details")
def get_result(self, ema_12_list, ema_26_list):
if len(ema_12_list) > 30 and ema_12_list[-1] > ema_26_list[
-1] and ema_12_list[-2] < ema_26_list[-2]:
return 1
return -1
def get_buy_list(self, date):
code_list = self.db_manager_wm.get_code_list()
buy_list = list()
for item in code_list:
try:
code = item["code"]
price_list = list()
expma_list = list()
# 获取ema数据
tk_details = self.db_manager_wm.find_by_key({"code": code})[0]
for tk_item in [
x for x in tk_details["price_list"]
if x["close_price"] != 0
]:
if time_cmp(str(date), tk_item["date_list"][-2]):
price_list.append(tk_item["close_price"])
ema_12_list, ema_26_list = self.get_w_expma_2(price_list[:])
if self.get_result(ema_12_list, ema_26_list) == 1:
buy_list.append(code)
except Exception as e:
continue
return buy_list
# 将策略结果同步到redis
def update_redis(self, date):
buy_list = self.get_buy_list(date)
rm = RedisManager()
rm.set_data("ema_" + str(date), buy_list)
# 计算当前时间段ema值
def get_w_expma(self, price_list):
ema_12_list = list()
for index in range(len(price_list)):
if index == 0:
ema_12_list.append(price_list[0])
else:
ema_12_list.append(
round(
ema_12_list[index - 1] * 11 / 13 +
price_list[index] * 2 / 13, 4))
ema_26_list = list()
for index in range(len(price_list)):
if index == 0:
ema_26_list.append(price_list[0])
else:
ema_26_list.append(
round(
ema_26_list[index - 1] * 25 / 27 +
price_list[index] * 2 / 27, 4))
return ema_12_list, ema_26_list
# 计算当前时间段ema值
def get_w_expma_2(self, price_list):
ema_12_list = list()
for index in range(len(price_list)):
if index == 0:
ema_12_list.append(price_list[0])
else:
ema_12_list.append(
round(
ema_12_list[index - 1] * 4 / 6 +
price_list[index] * 2 / 6, 4))
ema_26_list = list()
for index in range(len(price_list)):
if index == 0:
ema_26_list.append(price_list[0])
else:
ema_26_list.append(
round(
ema_26_list[index - 1] * 11 / 13 +
price_list[index] * 2 / 13, 4))
return ema_12_list, ema_26_list
plt.subplot(111)
lable_x = np.arange(len(list1))
lable_y = [x * 0 for x in range(len(list1))]
plt.plot(lable_x, lable_y, color="#404040", linewidth=1.0, linestyle="-")
plt.bar(lable_x, list1, color="r", width=1.0)
plt.bar(lable_x, list2, color="g", width=1.0)
plt.xlim(lable_x.min(), lable_x.max() * 1.1)
plt.ylim(min(list2) * 1.1, max(list1) * 1.1)
plt.grid(True)
plt.show()
if __name__ == '__main__':
db_manager = DBManager("wm_details")
code_list = db_manager.get_code_list()
result_list = list()
for item in code_list:
code = item["code"]
print(code)
tk_data = db_manager.find_by_key({"code": code})[0]
close_list = [
float(x["cur_close_price"]) for x in tk_data["price_list"]
if x["cur_close_price"] != 0
]
len_list = len(close_list)
if close_list:
for index in range(expect, len(close_list)):
rate_1 = (close_list[index - period] -
close_list[index]) / close_list[index - period]
if rate_1 > rate_fail and len_list > index + expect:
lable_x = np.arange(len(list1))
lable_y = [x * 0 for x in range(len(list1))]
# 绘制中轴线
plt.plot(lable_x, lable_y, color="#404040", linewidth=1.0, linestyle="-")
plt.bar(lable_x, list1, color="r", width=1.0)
plt.bar(lable_x, list2, color="g", width=1.0)
plt.xlim(lable_x.min(), lable_x.max() * 1.1)
plt.ylim(min(list2) * 1.1, max(list1) * 1.1)
plt.grid(True)
plt.show()
if __name__ == "__main__":
# 获取数据
db_manager_wm = DBManager("fcr_details")
code_list = db_manager_wm.get_code_list()
result_list = list()
for item in code_list:
code = item["code"]
print(code)
tk_data = db_manager_wm.find_by_key({"code": code})[0]
colse_list = [
float(x["close"]) for x in tk_data["price_list"] if x["close"] != 0
]
len_list = len(colse_list)
if colse_list:
# 执行判断逻辑
for index in range(expect, len(colse_list)):
rate_1 = (colse_list[index - period] - colse_list[index]
) / colse_list[index - period] # 下跌幅度
if rate_1 > rate_fall and len_list > index + expect: