def start_collect_data(start_date, end_date):
try:
all_stocks = StockInfo.objects()
except Exception as e:
logging.error('Error when query StockInfo:' + str(e))
raise e
stocks_count = all_stocks.count()
skip = 0
while skip < stocks_count:
try:
stocks = StockInfo.objects().skip(skip).limit(query_step)
except Exception as e:
logging.error('Error when query skip %s StockInfo:%s' % (skip, e))
stocks = []
for i in stocks:
try:
collect_stock_data(i.stock_number, start_date, end_date)
except Exception as e:
logging.error(
'Error when collect datayes weekly trading data %s: %s' %
(i.stock_number, e))
skip += query_step
def start_quant_analysis(**kwargs):
if not SDT.objects(date=kwargs['date']):
print 'Not a Trading Day'
return
stock_count = StockInfo.objects().count()
skip = 0
while skip < stock_count:
try:
stocks = StockInfo.objects().skip(skip).limit(step)
except Exception, e:
logging.error('Error when query StockInfo:' + str(e))
stocks = []
for s in stocks:
if s.account_firm and u'瑞华会计师' in s.account_firm:
# 过滤瑞华的客户
continue
try:
quant_stock(s.stock_number, s.stock_name, **kwargs)
except Exception, e:
logging.error('Error when macd quant %s:%s' %
(s.stock_number, e))
def start_collect_detail():
try:
all_stocks = StockInfo.objects()
except Exception as e:
logging.error('Error when query StockInfo:' + str(e))
raise e
stocks_count = len(all_stocks)
skip = 0
while skip < stocks_count:
try:
stocks = StockInfo.objects().skip(skip).limit(query_step)
except Exception as e:
logging.error('Error when query skip %s StockInfo:%s' % (skip, e))
stocks = []
for i in stocks:
try:
collect_company_survey(i)
except Exception as e:
logging.error('Error when collect %s data: %s' %
(i.stock_number, e))
time.sleep(random.random())
skip += query_step
def collect_stock_info():
"""
存储股票最基本的信息
"""
url = eastmoney_stock_api
data = request_and_handle_data(url)
stock_data = data['rank']
for i in stock_data:
stock = i.split(',')
stock_number = stock[1]
stock_name = stock[2]
total_value = 0 if stock[-2] == '-' else stock[-2]
cursor = StockInfo.objects(stock_number=stock_number)
if cursor:
stock_info = cursor[0]
stock_info.total_value = int(total_value)
stock_info.update_time = datetime.datetime.now()
else:
stock_info = StockInfo(stock_number=stock_number,
stock_name=stock_name,
update_time=datetime.datetime.now(),
total_value=int(total_value))
try:
stock_info.save()
except Exception as e:
logging.error('Saving %s data failed: %s' %
(stock_info.stock_number, e))
def start_quant_analysis(**kwargs):
"""
:param kwargs:{
qr_date: 运行策略时间
}
:return:
"""
if not kwargs.get('qr_date'):
print('no qr_date')
return
if not kwargs.get('quant_stock'):
print('not quant_stock funtion')
return
if not SDT.objects(date=kwargs['qr_date']) and not kwargs.get('real_time'):
print('Not a Trading Date')
return
try:
all_stocks = StockInfo.objects()
except Exception as e:
logging.error('Error when query StockInfo:' + str(e))
raise e
stocks_count = all_stocks.count()
skip = 0
quant_res = []
while skip < stocks_count:
try:
stocks = StockInfo.objects().skip(skip).limit(query_step)
except Exception as e:
logging.error('Error when query skip %s StockInfo:%s' % (skip, e))
stocks = []
for i in stocks:
if i.account_firm and u'瑞华会计师' in i.account_firm:
# 过滤瑞华的客户
continue
if not kwargs.get('real_time') and\
not SDT.objects(Q(date=kwargs['qr_date']) & Q(stock_number=i.stock_number)):
continue
qr = ''
kwargs['industry_involved'] = i.industry_involved
try:
qr = kwargs['quant_stock'](i.stock_number, i.stock_name,
**kwargs)
except Exception as e:
logging.error('Error when quant %s ma strategy: %s' %
(i.stock_number, e))
if isinstance(qr, QR):
quant_res.append(qr)
skip += query_step
return quant_res
def start_mining_notice():
try:
all_stocks = StockInfo.objects()
except Exception as e:
logging.error('Error when query StockInfo:' + str(e))
raise e
stocks_count = len(all_stocks)
skip = 0
notice_data = []
while skip < stocks_count:
try:
stocks = StockInfo.objects().skip(skip).limit(query_step)
except Exception as e:
logging.error('Error when query skip %s StockInfo:%s' % (skip, e))
stocks = []
for i in stocks:
if i.account_firm and u'瑞华会计师' in i.account_firm:
# 过滤瑞华的客户
continue
sdt = SDT.objects(
Q(stock_number=i.stock_number) & Q(date=datetime.date.today()))
if not sdt:
# 过滤仍在停牌的票
continue
try:
notice = collect_event_notice(i.stock_number)
except Exception as e:
logging.error('Error when collect %s notice: %s' %
(i.stock_number, e))
if notice:
notice_data += notice
skip += query_step
if not notice_data:
return
df = DataFrame(notice_data).sort_values(by=['stock_number', 'date'], ascending=[True, True])\
.set_index(['stock_number', 'date'])
pd.set_option('display.width', 400)
pd.set_option('display.max_colwidth', 150)
pd.set_option('display.max_rows', 800)
notice_path = '/root/healing-ward/notice_mining/%s.txt' % datetime.date.today(
)
notice_data = df.to_string()
if exists(notice_path):
print(notice_data)
else:
with codecs.open(notice_path, 'w+', 'utf-8') as fd:
fd.write(notice_data)
pd.set_option('display.width', None)
pd.set_option('display.max_rows', None)
def collect_stock_info():
"""
存储股票最基本的信息
"""
stock_data = get_pro_client().stock_basic(
exchange='',
list_status='L',
fields='ts_code,symbol,name,area,industry,list_date')
for i in range(len(stock_data)):
stock = stock_data.iloc[i]
cursor = StockInfo.objects(stock_number=stock.symbol)
if cursor:
stock_info = cursor[0]
else:
stock_info = StockInfo(stock_number=stock.symbol)
stock_info.stock_name = stock['name']
stock_info.area = stock.area
stock_info.industry = stock.industry
stock_info.list_date = stock.list_date
try:
stock_info.save()
except Exception as e:
logging.error("save base stock_info error, e = %s" % e)
def pre_sdt_check(stock_number, **kwargs):
"""
依据量价进行预先筛选
:param stock_number:
:param qr_date:
:return:
"""
qr_date = kwargs.get('qr_date')
rate_value = 0
cursor = SDT.objects(Q(stock_number=stock_number) & Q(today_closing_price__ne=0.0) & Q(date__lte=qr_date))\
.order_by('-date')
if not cursor:
return False
min_total_value = 3000000000
stock_info = StockInfo.objects(stock_number=stock_number)
if stock_info and stock_info[
0].total_value and stock_info[0].total_value < min_total_value:
return False
max_trade_amount = 5000
avg_trade_amount = 3000
amount_avg_num = 5
amount_sdt = cursor[:amount_avg_num]
if cal_turnover_ma(cursor, amount_avg_num) >= avg_trade_amount or\
max([i.turnover_amount for i in amount_sdt]) >= max_trade_amount:
rate_value += 1
if rate_value:
return True
else:
return False
def start_calculate(start_date, end_date, reverse=False):
if not isinstance(start_date, datetime.datetime) or not isinstance(
end_date, datetime.datetime):
return
stock_info = StockInfo.objects()
price_volatility = list()
for i in stock_info:
if is_new_stock(i.stock_number, end_date):
continue
start_sdt = fetch_stock_price(i.stock_number, start_date)
end_sdt = fetch_stock_price(i.stock_number, end_date)
if not start_sdt or not end_sdt:
continue
start_price = start_sdt.today_closing_price
end_price = end_sdt.today_closing_price
if start_sdt.total_stock != end_sdt.total_stock:
if not start_sdt.total_stock or not end_sdt.total_stock:
continue
start_price = start_price * start_sdt.total_stock / end_sdt.total_stock
start_price = round(start_price, 2)
price_diff = end_price - start_price
if reverse:
if price_diff < 0:
continue
else:
if price_diff > 0:
continue
price_volatility.append({
'stock_number':
i.stock_number,
'stock_name':
i.stock_name,
'start_price':
start_price,
'end_price':
end_price,
'increase_rate':
round(price_diff / start_price, 4)
})
if not price_volatility:
print('No Output')
return
price_volatility = sorted(price_volatility,
key=lambda x: x['increase_rate'],
reverse=reverse)
count = len(price_volatility)
print('------------------%s--------------------' % count)
df = DataFrame(price_volatility[:display_count]).set_index(['stock_number'])\
.reindex(columns=['stock_name', 'start_price', 'end_price', 'increase_rate'])
pd.set_option('display.max_rows', display_count + 10)
print(df)
pd.reset_option('display.max_rows')
def main():
setup_logging(__file__)
try:
all_stocks = StockInfo.objects().timeout(False)
except Exception, e:
logging.error("Error when query StockInfo:" + str(e))
raise e
def collect_stock_info():
"""
存储股票最基本的信息
"""
url = eastmoney_stock_api
data = request_and_handle_data(url)
stock_data = data['rank']
for i in stock_data:
stock = i.split(',')
stock_number = stock[1]
stock_info = StockInfo(stock_number=stock_number, update_time=datetime.datetime.now())
if not check_duplicate(stock_info):
try:
stock_info.save()
except Exception, e:
logging.error('Saving %s data failed:' % (stock_info.stock_number, e))
def check_duplicate(stock_info):
"""
检查存储的数据是否重复
"""
try:
cursor = StockInfo.objects(stock_number=stock_info.stock_number)
except Exception, e:
logging.error('Query %s data failed: %s' % (stock_info.stock_number, e))
raise e
def total_market_value():
cursor = StockInfo.objects()
total_value = 0.0
for i in cursor:
if u'退市' not in i.stock_name and isinstance(i.total_value, int):
total_value += float(i.total_value)
trillion = 10.0 ** 12
return round(total_value/trillion, 4)
def begin_collect_his():
stock_info = StockInfo.objects()
stock_count = stock_info.count()
skip = 0
while skip < stock_count:
try:
stocks = StockInfo.objects().skip(skip).limit(query_step)
except Exception, e:
logging.error("Error when query skip %s StockInfo:%s" % (skip, e))
stocks = []
for i in stocks:
try:
collect_his_trading(i.stock_number, i.stock_name)
except Exception, e:
logging.error("Collect %s his data failed:%s" % (i.stock_number, e))
finally:
def begin_collect_his():
stock_info = StockInfo.objects()
stock_count = stock_info.count()
skip = 0
while skip < stock_count:
try:
stocks = StockInfo.objects().skip(skip).limit(query_step)
except Exception, e:
logging.error('Error when query skip %s StockInfo:%s' % (skip, e))
stocks = []
for i in stocks:
try:
collect_his_trading(i.stock_number, i.stock_name)
except Exception, e:
logging.error('Collect %s his data failed:%s' % (i.stock_number, e))
finally:
def start_quant_analysis(short_ma_num, long_ma_num, qr_date):
if not SDT.objects(date=qr_date):
print 'Not a Trading Date'
return
try:
all_stocks = StockInfo.objects()
except Exception, e:
logging.error('Error when query StockInfo:' + str(e))
raise e
def check_duplicate(stock_info):
"""
检查存储的数据是否重复
"""
try:
cursor = StockInfo.objects(stock_number=stock_info.stock_number)
except Exception, e:
logging.error('Query %s data failed: %s' %
(stock_info.stock_number, e))
raise e
def start_quant_analysis(short_ma_num, long_ma_num, qr_date, half_year=False):
if not SDT.objects(date=qr_date).count():
print 'Not a Trading Date'
return
try:
all_stocks = StockInfo.objects()
except Exception, e:
logging.error('Error when query StockInfo:' + str(e))
raise e
def collect_stock_info():
"""
存储股票最基本的信息
"""
url = eastmoney_stock_api
data = request_and_handle_data(url)
stock_data = data['rank']
for i in stock_data:
stock = i.split(',')
stock_number = stock[1]
stock_name = stock[2]
stock_info = StockInfo(stock_number=stock_number, stock_name=stock_name, update_time=datetime.datetime.now())
if not check_duplicate(stock_info):
try:
stock_info.save()
except Exception as e:
logging.error('Saving %s data failed:' % (stock_info.stock_number, e))
def total_market_value():
cursor = StockInfo.objects()
total_value = 0.0
for i in cursor:
if i.stock_name and u'退市' not in i.stock_name and isinstance(
i.total_value, int):
total_value += float(i.total_value)
trillion = 10.0**12
return round(total_value / trillion, 4)
def start_quant_analysis(**kwargs):
if not SDT.objects(date=kwargs['date']):
print 'Not a Trading Day'
return
stock_count = StockInfo.objects().count()
skip = 0
while skip < stock_count:
try:
stocks = StockInfo.objects().skip(skip).limit(step)
except Exception, e:
logging.error('Error when query StockInfo:' + str(e))
stocks = []
for s in stocks:
if s.account_firm and u'瑞华会计师' in s.account_firm:
# 过滤瑞华的客户
continue
try:
quant_stock(s.stock_number, s.stock_name, **kwargs)
except Exception, e:
logging.error('Error when macd quant %s:%s' % (s.stock_number, e))
def start_quant_analysis(**kwargs):
if not kwargs.get('qr_date'):
print 'no qr_date'
return
if not kwargs.get('quant_stock'):
print 'not quant_stock funtion'
return
if not SDT.objects(date=kwargs['qr_date']) and not kwargs.get('real_time'):
print 'Not a Trading Date'
return
try:
all_stocks = StockInfo.objects()
except Exception, e:
logging.error('Error when query StockInfo:' + str(e))
raise e
def query_market_plate_stock(market_plate, filter_ruihua=True):
"""
查询某个板块的个股
"""
stock_info = StockInfo.objects().timeout(False)
stocks = []
filtered_firm = u'瑞华会计师事务所'
for i in stock_info:
if i.market_plate:
if market_plate in i.market_plate:
if filter_ruihua and filtered_firm not in i.account_firm:
# 由于老爸是瑞华会计师事务所的，我不能买卖这家事务所出具报告的上市公司股票
stocks.append(i)
elif not filter_ruihua:
stocks.append(i)
return stocks
def query_market_plate_stock(market_plate, filter_ruihua=True):
"""
查询某个板块的个股
"""
stock_info = StockInfo.objects().timeout(False)
stocks = []
filtered_firm = u'瑞华会计师事务所'
for i in stock_info:
if i.market_plate:
if market_plate in i.market_plate:
if filter_ruihua and filtered_firm not in i.account_firm:
# 由于老爸是瑞华会计师事务所的，我不能买卖这家事务所出具报告的上市公司股票
stocks.append(i)
elif not filter_ruihua:
stocks.append(i)
return stocks
def check_duplicate(stock_info):
"""
检查存储的数据是否重复
"""
try:
cursor = StockInfo.objects(stock_number=stock_info.stock_number)
if cursor:
data = cursor.next()
if data.stock_name != stock_info.stock_name:
data.stock_name = stock_info.stock_name
data.save()
return True
except Exception as e:
logging.error('Query %s data failed: %s' %
(stock_info.stock_number, e))
raise e
return False
stock_info.save()
def start_collect_detail():
try:
all_stocks = StockInfo.objects()
except Exception, e:
logging.error('Error when query StockInfo:' + str(e))
raise e
stocks_count = len(all_stocks)
skip = 0
while skip < stocks_count:
try:
stocks = StockInfo.objects().skip(skip).limit(query_step)
except Exception, e:
logging.error('Error when query skip %s StockInfo:%s' % (skip, e))
stocks = []
for i in stocks:
try:
collect_company_survey(i)
except Exception, e:
logging.error('Error when collect %s data: %s' % (i.stock_number, e))
time.sleep(random.random())
skip += query_step
if __name__ == '__main__':
setup_logging(__file__, logging.WARNING)
def start_collect_detail():
try:
all_stocks = StockInfo.objects()
except Exception, e:
logging.error('Error when query StockInfo:' + str(e))
raise e
def start_collect_fundamentals():
df_all_stock = ts.get_stock_basics()
for i in range(len(df_all_stock)):
df_stock = df_all_stock.iloc[i]
if not int(df_stock['timeToMarket']):
continue
try:
cursor = StockInfo.objects(stock_number=df_stock.name)
if cursor:
stock_info = cursor[0]
else:
stock_info = StockInfo()
stock_info.update_time = datetime.datetime.now()
stock_info.industry = df_stock['industry']
stock_info.pe = float(df_stock['pe'])
stock_info.liquid_assets = int(df_stock['liquidAssets'])
stock_info.fixed_assets = int(df_stock['fixedAssets'])
stock_info.reserved = int(df_stock['reserved'])
stock_info.reserved_per_share = float(df_stock['reservedPerShare'])
stock_info.esp = float(df_stock['esp'])
stock_info.bvps = float(df_stock['bvps'])
stock_info.pb = float(df_stock['pb'])
stock_info.time_to_market = datetime.datetime.strptime(
str(df_stock['timeToMarket']), '%Y%m%d')
stock_info.undp = float(df_stock['undp'])
stock_info.perundp = float(df_stock['perundp'])
stock_info.rev = float(df_stock['rev'])
stock_info.profit = float(df_stock['profit'])
stock_info.gpr = float(df_stock['gpr'])
stock_info.npr = float(df_stock['npr'])
stock_info.holders = int(df_stock['holders'])
stock_info.save()
except Exception as e:
logging.error(
'Error when collect stock %s fundamentals from tushare:%s' %
(df_stock.name, str(e)))
def start_mining_notice():
try:
all_stocks = StockInfo.objects()
except Exception, e:
logging.error('Error when query StockInfo:' + str(e))
raise e
stocks_count = len(all_stocks)
skip = 0
notice_data = []
while skip < stocks_count:
try:
stocks = StockInfo.objects().skip(skip).limit(query_step)
except Exception, e:
logging.error('Error when query skip %s StockInfo:%s' % (skip, e))
stocks = []
for i in stocks:
if i.account_firm and u'瑞华会计师' in i.account_firm:
# 过滤瑞华的客户
continue
sdt = SDT.objects(
Q(stock_number=i.stock_number) & Q(date=datetime.date.today()))
if not sdt:
# 过滤仍在停牌的票
continue
def start_quant_analysis(short_ma_num=1, long_ma_num=30):
try:
all_stocks = StockInfo.objects()
except Exception, e:
logging.error('Error when query StockInfo:' + str(e))
raise e
def start_mining_notice():
try:
all_stocks = StockInfo.objects()
except Exception, e:
logging.error('Error when query StockInfo:' + str(e))
raise e
def start_quant_analysis(short_ma_num=1, long_ma_num=30):
try:
all_stocks = StockInfo.objects()
except Exception, e:
logging.error('Error when query StockInfo:' + str(e))
raise e
def start_collect_detail():
try:
all_stocks = StockInfo.objects()
except Exception, e:
logging.error('Error when query StockInfo:' + str(e))
raise e
def start_mining_notice():
try:
all_stocks = StockInfo.objects()
except Exception, e:
logging.error('Error when query StockInfo:' + str(e))
raise e