def check_diff(self, db_name='ss.db'):
self.logger.info("Check diff between stock list from db and pre-dump")
db = StockDb(db_name)
total_stocks = db.get_stock_list()
f = open(self.pre_dump_file, 'r')
recorded_stocks = []
for line in f.readlines():
item = line.replace('\n', '').split(',')
if item[0].startswith('60'):
stock_id = "sh%s" % (item[0])
else:
stock_id = "sz%s" % (item[0])
recorded_stocks.append(stock_id)
f.close()
self.logger.info(
'Total stock from db:%s,total stock from pre-dump:%s' %
(len(total_stocks), len(recorded_stocks)))
test = []
for s in total_stocks:
if s not in recorded_stocks:
test.append(s)
self.logger.info('===Stocks in total, but not in prelist====')
self.logger.info(test)
test = []
for s in recorded_stocks:
if s not in total_stocks:
test.append(s)
self.logger.info('===Stocks in prelist,but not in total===')
self.logger.info(test)
def update_db(self, db_name='ss.db'):
self.logger.info("Start store pre-dumped info to database...")
db = StockDb(db_name)
f = open(self.pre_dump_file, 'r')
last_trading_date = self.get_last_trading_date_live()
no_data_stocks = []
for line in f.readlines():
item = line.replace('\n', '').split(',')
if (item[1] == '0.000'):
self.logger.info("Stock %s has no data, skip it" % (item[0]))
continue
if item[0].startswith('60'):
stock_id = "sh%s" % (item[0])
else:
stock_id = "sz%s" % (item[0])
sql_cmd = "insert into tb_daily_info values ('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s')"\
%(last_trading_date,stock_id,item[1],item[2],item[3],item[4],item[5],item[6],item[7])
db.update_db(sql_cmd)
f.close()
self.logger.info("There are total %s stock which have no data..." %
(len(no_data_stocks)))
self.logger.info("Store pre-dumped info to database done...")
def get_live_mon_items(self, stock_id):
info = self.get_live_status(stock_id).split(',')
cur_price = float(info[3])
last_day_price = float(info[2])
open_price = float(info[1])
stock_name = info[0]
#print(len(stock_name))
#print(2*'aaa')
if len(stock_name) < 4:
#stock_name = "%s%s"%(''*(4-len(stock_name)),stock_name)
stock_name = "%s%s" % (' ', stock_name)
#print(len(stock_name))
aoi = round((cur_price - last_day_price) * 100 / last_day_price, 2)
aoi_open = round((open_price - last_day_price) * 100 / last_day_price,
2)
volume = round(float(info[8]) / 1000000, 2)
rmb = round(float(info[9]) / 100000000, 2)
db = StockDb()
float_shares = round(
db.get_float_shares_from_id(stock_id) / 100000000, 2)
score = round(self.get_weighted_score(stock_id), 2)
ret = "%s(%s) | %8s%% | %8s%% | %8s | %8s(万手) | %8s(亿) | %8s(亿) | %8s"\
%(stock_name,stock_id,aoi_open,aoi,info[3],volume,rmb,float_shares,score)
return ret
def __init__(self):
self.logger = Logger("StockUtil")
self.db = StockDb()
self.valid_stock_file = "valid_stock.csv"
pass
class StockUtil():
def __init__(self):
self.logger = Logger("StockUtil")
self.db = StockDb()
self.valid_stock_file = "valid_stock.csv"
pass
def is_list_sorted(self, lst):
if sorted(lst) == lst:
return 'asc'
elif sorted(lst, reverse=True) == lst:
return 'desc'
def is_bid_time(self):
t = datetime.datetime.now()
if t.hour == 9 and t.minute >= 15 and t.minute <= 25:
return True
return False
def is_trading_time(self):
t = datetime.datetime.now()
if t.hour < 9 or t.hour > 15 or t.hour == 12:
return False
if t.hour == 9 and t.minute < 30:
return False
if t.hour == 11 and t.minute > 30:
return False
return True
def get_new_stock_info(self, page_num):
url = "http://vip.stock.finance.sina.com.cn//corp/view/vRPD_NewStockIssue.php?page=%s" % (
page_num)
r = requests.get(url)
bs = BeautifulSoup(r.content, 'lxml')
trs = bs.table.find_all('tr', class_='tr_2')
ret = []
for tr in trs:
tds = tr.find_all('td')
td_list = []
for td in tds:
td_text = td.text.lstrip().rstrip().replace('\n', '')
td_list.append(td_text)
#print(','.join(td_list))
ret.append(td_list)
return ret[1:]
def get_stock_trading_dates(self, stock_id):
in_mkt_date = self.db.get_in_mkt_date_from_id(stock_id)
last_trading_date = self.db.get_last_trading_date()
d1 = datetime.datetime.strptime(in_mkt_date, "%Y-%m-%d")
d2 = datetime.datetime.strptime(last_trading_date, "%Y-%m-%d")
return (d2 - d1).days
def get_yesterday(self):
today = datetime.date.today()
oneday = datetime.timedelta(days=1)
yesterday = today - oneday
return yesterday.strftime('%Y_%m_%d')
def get_today(self):
return datetime.date.today().strftime('%Y_%m_%d')
def get_fp_stock_list(self):
fp_file = "output/fp_%s.csv" % (self.get_yesterday())
return self.get_stock_list_from_file(fp_file)
def get_valid_stocks(self):
return self.get_stock_list_from_file(self.valid_stock_file)
def get_weighted_score(self, stock_id):
score_list = self.db.get_stock_score_list(stock_id)
ret = 0
count = 1
for score in score_list:
ret = ret + score * count
count = count / 2
return ret
def get_stock_list_from_file(self, file_name):
'''
从csv文件中获取列表，返回一个数组
'''
if not os.path.exists(file_name):
self.logger.info(
"File %s does not exist, please check...[get_stock_list_from_file]"
% (file_name))
return []
with open(file_name, 'r') as f:
output = f.read()
return output.split(',')
def save_stock_list_to_file(self, stock_list, file_name):
#file_name = "./output/fp_%s.csv"%(self.last_trading_date.replace('-','_'))
s_list_str = ','.join(stock_list)
with open(file_name, 'w') as f:
f.write(s_list_str)
def get_summary_status(self, stock_list):
ret = []
title = " 股票名称（股票ID）| 开盘涨幅 | 当前涨幅 | 当前价格 | 成交量 | 成交金额 | 流通股 | 得分"
ret.append(title)
#print(stock_list)
for s in stock_list:
try:
status = self.get_live_mon_items(s)
ret.append(status)
except:
self.logger.info("Exception on stock:%s" % (s))
return ret
def get_live_status(self, stock_id):
ret = ""
url = "http://hq.sinajs.cn/list=%s" % (stock_id)
#self.logger.info(url)
r = requests.get(url)
if r.status_code != 200:
return ret
re_info = re.compile(r'="(.*)"')
ret = re_info.findall(r.text)[0]
return ret
def get_live_mon_items(self, stock_id):
info = self.get_live_status(stock_id).split(',')
cur_price = float(info[3])
last_day_price = float(info[2])
open_price = float(info[1])
stock_name = info[0]
#print(len(stock_name))
#print(2*'aaa')
if len(stock_name) < 4:
#stock_name = "%s%s"%(''*(4-len(stock_name)),stock_name)
stock_name = "%s%s" % (' ', stock_name)
#print(len(stock_name))
aoi = round((cur_price - last_day_price) * 100 / last_day_price, 2)
aoi_open = round((open_price - last_day_price) * 100 / last_day_price,
2)
volume = round(float(info[8]) / 1000000, 2)
rmb = round(float(info[9]) / 100000000, 2)
db = StockDb()
float_shares = round(
db.get_float_shares_from_id(stock_id) / 100000000, 2)
score = round(self.get_weighted_score(stock_id), 2)
ret = "%s(%s) | %8s%% | %8s%% | %8s | %8s(万手) | %8s(亿) | %8s(亿) | %8s"\
%(stock_name,stock_id,aoi_open,aoi,info[3],volume,rmb,float_shares,score)
return ret
def get_live_mon_items_bid(self, stock_id):
info = self.get_live_status(stock_id).split(',')
self.logger.info(info)
cur_price = float(info[11])
last_day_price = float(info[2])
#open_price = float(info[1])
aoi = round((cur_price - last_day_price) * 100 / last_day_price, 2)
#aoi_open = (open_price-last_day_price)*100/last_day_price
ret = "%s(%s) | %s | %s | %s" % (info[0], stock_id, aoi,
round(cur_price, 2),
round(float(info[10]) / 10000, 0))
return ret
def get_live_price(self, stock_id):
info = self.get_live_status(stock_id).split(',')
return info[3]
def get_live_aoi_bid(self, stock_id):
info = self.get_live_status(stock_id).split(',')
cur_price = float(info[11])
last_day_price = float(info[2])
return round((cur_price - last_day_price) * 100 / last_day_price, 2)
def get_live_aoi(self, stock_id):
info = self.get_live_status(stock_id).split(',')
cur_price = float(info[3])
last_day_price = float(info[2])
return round((cur_price - last_day_price) * 100 / last_day_price, 2)
def get_delta(self, stock_id, day_num):
if day_num <= 0 or day_num > 10:
self.logger.info("Please specify a daynum which between 1~10...")
return 0
sql_cmd = "select sum(pchg) from (select * from tb_daily_info where stock_id='%s' order by date desc limit %s)" % (
stock_id, day_num)
return float(self.db.query_db(sql_cmd)[0][0])
def get_lift_in_one_day(self, stock_id, day_num):
pass
def get_increase_amount(self, stock_id, day_num):
pass
def get_volume_sum(self, stock_id, day_num):
ret = 0
for d in range(day_num):
ret = ret + self.get_volume(stock_id, d)
return round(ret, 2)
def get_volume(self, stock_id, day_num):
pass
def update_last_trading_date(self, last_trading_date):
self.logger.info("Update last trading date to db")
db = StockDb()
sql_cmd = "update tb_configuration set value='%s' where name='last_trading_date'" % (
last_trading_date)
db.update_db(sql_cmd)
def __init__(self):
self.logger = Logger("StockFilter")
self.db = StockDb()
self.util = StockUtil()
class StockFilter():
def __init__(self):
self.logger = Logger("StockFilter")
self.db = StockDb()
self.util = StockUtil()
def check_if_time_available(self):
pass
def get_top_increase(self, stock_list, n, day_num):
d = {}
for s in stock_list:
d[s] = self.util.get_delta(s, day_num)
sorted_list = sorted(d.items(), key=lambda d: d[1], reverse=True)
ret = []
for i in range(1000):
stock_id = sorted_list[i][0]
ret.append(stock_id)
if len(ret) == n:
break
return ret
def filter_new_stocks(self, stock_list):
new_stocks = self.db.get_new_stocks()
return list(set(stock_list) - set(new_stocks))
def get_volume_within_days(self, stock_list, day_num, volume_critiria):
'''
过滤stock_list，返回day_num天前到现在，换手率超过volume_critiria的股票
例如，get_volume_within_days(stock_list, 3, 20)表示返回3天内总换手超过20%的股票
'''
ret = []
for s in stock_list:
volume = self.util.get_volume_sum(s, day_num)
if volume >= volume_critiria:
#self.logger.info("%s:%s"%(self.util.get_stock_name_from_id(s),volume))
ret.append(s)
self.logger.info(
"Found %s stocks after get volume sum within %s days" %
(len(ret), day_num))
return ret
def get_delta_within_days(self, stock_list, day_num, delta_criteria):
'''
过滤stock_list，返回delta_day内涨幅>delta_critira的股票列表
例如，get_delta_within_days(stock_list,3,20)表示拿到3天内涨幅>20%的股票列表
'''
self.logger.info(
"======Start, get delta>%s stocks within %s days...======" %
(delta_criteria, day_num))
ret = []
for s in stock_list:
delta = self.db.get_sum_n_pchg(s, day_num)
#self.logger.info("%s:%s"%(s,delta))
if (delta > delta_criteria and s not in ret):
ret.append(s)
#filtered_list=list(set(stock_list)-set(ret))
#self.logger.info("Filtered %s stocks"%(len(filtered_list)))
#self.logger.info(filtered_list)
self.logger.info(
"======End, found %s stocks after calling get_delta_within_days======"
% (len(ret)))
return ret
def get_yd(self, amp_criteria=10):
last_trading_date = self.db.get_last_trading_date()
sql_cmd = "select * from (select *,(high-low)*100/low as lift,(high-close)*100/close as lift2 \
from tb_daily_info where pchg>3 and date='%s') where lift>%s" % (
last_trading_date, amp_criteria)
ret = self.db.query_db(sql_cmd)
return DataFrame(ret)[1].values.tolist()
def get_big_increase_within_days(self,
stock_list,
day_num,
increase_criteria=9):
self.logger.info("======Start, Get big increase within %s days======" %
(day_num))
day_list = self.db.get_last_n_dates(day_num)
sql_cmd = "select distinct stock_id from tb_daily_info where pchg>%s and date>='%s' and pchg!=''" % (
increase_criteria, day_list[-1])
ret = self.db.query_db(sql_cmd)
match_list = DataFrame(ret)[0].values.tolist()
ret = list(set(stock_list).intersection(set(match_list)))
self.logger.info(
"======End, found %s stocks after calling get_big_increase_within_days======"
% (len(ret)))
return ret
def get_big_turnover_within_days(self,
stock_list,
day_num,
turnover_criteria=5):
self.logger.info("======Start, get big turnover within %s days======" %
(day_num))
day_list = self.db.get_last_n_dates(day_num)
sql_cmd = "select distinct stock_id from tb_daily_info where turnover>%s and date>='%s' and turnover!=''" % (
turnover_criteria, day_list[-1])
ret = self.db.query_db(sql_cmd)
match_list = DataFrame(ret)[0].values.tolist()
ret = list(set(stock_list).intersection(set(match_list)))
self.logger.info(
"======End, found %s stocks after calling get_big_turnover_within_days======"
% (len(ret)))
return ret
def get_big_lift_within_days(self, stock_list, day_num, lift_criteria):
pass
def get_high_score_list(self, stock_list, score_criteria=40):
self.logger.info("======Start, get high score list======")
ret = []
for s in stock_list:
score = self.util.get_weighted_score(s)
if score > score_criteria:
ret.append(s)
self.logger.info(
"======End, found %s stocks after calling get_high_score_list======"
% (len(ret)))
return ret
def filter_low_score_today(self, stock_list):
ret = []
last_trading_date = self.db.get_last_trading_date()
sql_cmd = "select stock_id from (select date,stock_id,turnover,pchg,t1,t2,t3,pchg+t1+t2+t3 as score \
from (select *,(close-high)*100/close as t1,(close-open)*100/open as t2, (close-low)*100/low as t3, \
(high-low)*100/low as t4 from tb_daily_info where date='%s')) where score<0" % (
last_trading_date)
ret = self.db.query_db(sql_cmd)
s_list = DataFrame(ret)[0].values.tolist()
print(s_list)
for s in stock_list:
if s not in s_list:
ret.append(s)
return ret
def filter_big_lift_within_days(self, stock_list, day_num, lift_criteria):
'''
过滤stock_list,如果在day_num内，出现大阴线形态2（高点和收盘差超过lift_critiria），则剔除这些股票。
返回一个stock列表，去掉了所有n日内有大阴线形态2的股票。
'''
ret = []
self.logger.info("======Start, filter big lift within %s days======" %
(day_num))
for s in stock_list:
sum_lift = self.db.get_sum_n_lift(s, day_num)
if sum_lift > lift_criteria:
ret.append(s)
filtered_list = list(set(stock_list) - set(ret))
self.logger.info("Filtered %s stocks" % (len(filtered_list)))
self.logger.info(filtered_list)
self.logger.info(
"======End, found %s stocks after filtering big lift within %s days======"
% (len(ret), day_num))
return ret
def get_mkt_share_below_limit(self, stock_list, mkt_share_limit=100):
#Get market share below 100E...
ret = []
self.logger.info("Get mkt share which is below %sE..." %
(mkt_share_limit))
for s in stock_list:
float_shares = round(
self.db.get_float_shares_from_id(s) / 100000000, 2)
cur_price = float(self.util.get_live_status(s).split(',')[3])
mkt_share = float_shares * cur_price
if mkt_share < mkt_share_limit:
#self.logger.info("Add stock %s which mkt share<%s"%(s,mkt_share_limit))
ret.append(s)
return ret
def get_float_shares_below_limit(self, stock_list, float_shares_limit=10):
ret = []
match_list = self.db.get_stock_list_by_float_shares(float_shares_limit)
ret = list(set(stock_list).intersection(set(match_list)))
return ret
def get_increase_rate_increase(self,
stock_list,
day_num,
increase_criteria=1):
self.logger.info(
"======Start, get increase rate increase within %s days======" %
(day_num))
ret = []
for s in stock_list:
try:
increase_list = self.db.get_last_n_pchg(s, day_num)
if increase_list != [] and self.util.is_list_sorted(
increase_list) == 'desc':
#self.logger.info("Function get_increase_rate_increase, add stock %s"%(s))
ret.append(s)
except:
self.logger.info("Exception on stock:%s" % (s))
self.logger.info(
"======End, Found %s stocks after filtering get_increase_rate_increase within %s days======"
% (len(ret), day_num))
return ret
def get_turnover_increase(self, stock_list, day_num, increase_criteria=1):
self.logger.info("Get volume rate increase within %s days" % (day_num))
ret = []
for s in stock_list:
turnover_list = self.db.get_last_n_turnover(s, day_num)
if self.util.is_list_sorted(turnover_list) == 'desc':
self.logger.info("Add stock %s" % (s))
ret.append(s)
self.logger.info(
"Found %s stocks after filtering get_turnover_increase within %s days"
% (len(ret), day_num))
return ret
def get_turnover_burst(self, stock_list, day_num, increase_criteria=1):
self.logger.info("Get volume rate increase within %s days" % (day_num))
ret = []
for s in stock_list:
turnover_list = self.db.get_last_n_turnover(s, day_num)
turnover_list.reverse()
while len(turnover_list) > 1:
turnover = turnover_list.pop()
average = sum(turnover_list) / len(turnover_list)
#print("%s-%s:%s"%(s,turnover,average))
if turnover / average > 2:
ret.append(s)
self.logger.info("Add stock %s" % (s))
break
self.logger.info(
"Found %s stocks after filtering get_turnover_burst within %s days"
% (len(ret), day_num))
return ret
def is_trading(self):
db = StockDb()
return db.get_last_trading_date() == self.last_trading_date()
def last_trading_date(self):
db = StockDb()
status = db.get_stock_status('tb_daily_info', self.stock_id, 0)
return status[0]
def stock_name(self):
db = StockDb()
basic_info = db.get_stock_basic(self.stock_id)
return basic_info[1]