def run(self):
self.df_today_all = self.get_today_market()
# 存储每天 涨幅排行 榜,避免每次读取耗时过长
filename = self.today + '_all_.xls'
# 放在data文件夹下
full_filename = os.path.join(self.path, filename)
if self.df_today_all is not None:
# 保留小数点的后两位数
self.df_today_all['turnoverratio'] = self.df_today_all[
'turnoverratio'].map(lambda x: round(x, 2))
self.df_today_all['per'] = self.df_today_all['per'].map(
lambda x: round(x, 2))
self.df_today_all['pb'] = self.df_today_all['pb'].map(
lambda x: round(x, 2))
try:
self.df_today_all.to_excel(full_filename)
except Exception as e:
notify(title='写excel出错', desp=f'{self.__class__}')
self.logger.error(e)
try:
self.df_today_all.to_sql(self.today,
self.engine,
if_exists='fail')
except Exception as e:
notify(title='mysql出错', desp=f'{self.__class__}')
self.logger.error(e)
def predict_price(self):
session = requests.Session()
session.get(url=home_page, headers=home_headers)
r = session.get(url=xueqiu_url, headers=headers)
js_data = r.json()
quote = js_data.get('data', {}).get('quote')
quote['crawltime'] = datetime.datetime.now()
doc.insert_one(quote)
percent = quote.get('percent')
jsl_qdii, est_val_dt = self.qdii_info()
if jsl_qdii:
predict_v = round((1 + percent * 0.95 * 0.01) * jsl_qdii, 3)
self.logger.info(f'最新估值{predict_v}')
d = {'日期': today, '估值': predict_v}
client['db_stock']['huabaoyouqi_predict'].insert_one(d)
title = f'华宝估值{predict_v} 净值日期{est_val_dt[5:]}'
send_from_aliyun(title, '')
else:
notify(title='华宝油气获取估值失败')
def yesterday(self, df, post_fix, engine):
df = df.set_index('序号')
formula = lambda x: round(x * 100, 3)
df['最大涨幅'] = df['最大涨幅'].map(formula)
df['最大跌幅'] = df['最大跌幅'].map(formula)
df['今日开盘涨幅'] = df['今日开盘涨幅'].map(formula)
df['昨日涨停强度'] = df['昨日涨停强度'].map(lambda x: round(x, 0))
df['今日涨停强度'] = df['今日涨停强度'].map(lambda x: round(x, 0))
try:
df.to_sql(self.today + post_fix, engine, if_exists='fail')
except Exception as e:
notify(f'{self.__class__} 出错')
self.logger.info(e)
avg = round(df['今日涨幅'].mean(), 2)
median = round(df['今日涨幅'].median(), 2)
min_v = round(df['今日涨幅'].min(), 2)
min_index = df['今日涨幅'].argmin()
min_percent_name = df.iloc[min_index]['名称']
current = datetime.datetime.now().strftime('%Y-%m-%d')
title = '昨涨停今天{}平均涨{}\n'.format(current, avg)
content = '<p>昨天涨停今天<font color="red">{}</font></p>' \
'<p>平均涨幅 <font color="red">{}</font></p>' \
'<p>涨幅中位数 <font color="red">{}</font></p>' \
'<p>涨幅最小 <font color="red">{}</font></p>' \
'<p>涨幅最小股 <font color="red">{}</font></p>'.format(current, avg, median, min_v, min_percent_name)
try:
send_from_aliyun(title, content, types='html')
except Exception as e:
self.logger.error(e)
def get_basic_info(self, retry=5):
'''
保存全市场数据
:param retry:
:return:
'''
# 需要添加异常处理 重试次数
count = 0
while count < retry:
try:
df = ts.get_stock_basics()
except Exception as e:
self.logger.info(e)
time.sleep(10)
count+=1
continue
else:
break
if count==retry:
notify(title='获取股市市场全景数据失败',desp=f'{self.__class__}')
exit(0)
if df is not None:
df = df.reset_index()
df['更新日期'] = datetime.datetime.now()
engine = DBSelector().get_engine('db_stock','qq')
try:
df.to_sql('tb_basic_info', engine, if_exists='replace',index='index',index_label='id')
except Exception as e:
self.logger.error(e)
notify(title='mysql入库出错',desp=f'{self.__class__}')
def save_mysql(self, df):
conn2 = DB.get_engine('double_low_full', 'qq')
try:
df.to_sql(f'double_low_{self.today}',
con=conn2,
if_exists='replace')
except:
notify(title='mysql入库出差', desp=f'{self.__class__}')
def start(self):
buy, sell = self.run()
sub = '{}: 美元汇率{}'.format(
datetime.datetime.now().strftime('%Y-%m-%d %H:%M'), buy)
self.logger.info(sub)
conn = DBSelector().get_mysql_conn('db_stock', 'qq')
cmd = 'insert into `usd_ratio` (`price`,`date`) VALUES ({},{!r})'.format(
buy,
datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
self.execute(cmd, (), conn)
notify(title=sub, desp='')
def run(self):
content = self.crawl()
if content is None:
self.logger.error('爬取内容为空')
return
rows = content.get('rows')
row_list = list(map(lambda x: x.get('cell'), rows))
if not self.save_mongo(row_list):
self.logger.info('保存失败')
notify(title='jsl分级入库出错', desp=f'{self.__class__}')
def main():
# current='20191016'
current = datetime.datetime.now().strftime('%Y%m%d')
app = PlotYesterdayZT()
api = ts.get_apis()
for plot_type in ['zrzt', 'zdt']:
try:
app.plot_yesterday_zt(api, plot_type, current=current)
except Exception as e:
notify(title='zdt_plot 出错', desp=f'{__name__}')
continue
ts.close_apis(conn=api)
def parse_json(self, types):
if types == 'stock':
url = self.stock_url
mongo_doc = self.jsl_stock_lof
else:
url = self.index_lof_url
mongo_doc = self.jsl_index_lof
return_js = self.get(url=url)
rows = return_js.get('rows')
for item in rows:
cell = item.get('cell')
try:
mongo_doc.insert_one(cell)
except Exception as e:
self.logger.error(e)
notify(title='入mongo出错', desp=f'{self.__class__} 写入mongodb出错')
def main(current=None):
# current='20191016'
if current is None:
current = datetime.datetime.now().strftime('%Y%m%d')
if isinstance(current, int):
current = str(current)
app = PlotYesterdayZT()
api = ts.get_apis()
for plot_type in ['zrzt', 'zdt']:
try:
app.plot_yesterday_zt(api, plot_type, current=current)
except Exception as e:
print(plot_type, 'error')
print(e)
notify(title='zdt_plot 出错', desp=f'{__name__}')
continue
ts.close_apis(conn=api)
def yesterday_zt(self, df, post_fix, engine):
df = df.set_index('序号')
formula = lambda x: round(x * 100, 3)
df['最大涨幅'] = df['最大涨幅'].map(formula)
df['最大跌幅'] = df['最大跌幅'].map(formula)
df['今日开盘涨幅'] = df['今日开盘涨幅'].map(formula)
df['昨日涨停强度'] = df['昨日涨停强度'].map(lambda x: round(x, 0))
df['今日涨停强度'] = df['今日涨停强度'].map(lambda x: round(x, 0))
try:
df.to_sql(self.today + post_fix, engine, if_exists='fail')
except Exception as e:
notify(f'{self.__class__} 出错')
self.logger.info(e)
title, content = self.generate_html(df)
try:
send_from_aliyun(title, content, types='html')
except Exception as e:
self.logger.error(e)
def convert_json(self, content):
p = re.compile(r'"Data":(.*)};', re.S)
if len(content) <= 0:
self.logger.info('Content\'s length is 0')
exit(0)
result = p.findall(content)
if result:
try:
t1 = result[0]
t2 = re.sub('[\\r\\n]', '', t1)
t2 = re.sub(',,', ',0,0', t2)
t2 = re.sub('Infinity', '-1', t2)
t2 = re.sub('NaN', '-1', t2)
t2 = list(eval(t2))
return t2
except Exception as e:
notify(title='获取涨跌停数据出错', desp=f'{self.__class__}')
self.logger.info(e)
return None
else:
return None
def download(self, url, headers, retry=5):
for i in range(retry):
try:
resp = requests.get(url=url, headers=headers)
content = resp.text
md_check = re.findall('summary|lasttradedate', content)
if content and len(md_check) > 0:
return content
else:
time.sleep(60)
self.logger.info(
'failed to get content, retry: {}'.format(i))
continue
except Exception as e:
notify(title='获取涨跌停数据出错', desp=f'{self.__class__}')
self.logger.error(e)
time.sleep(60)
continue
return None
def get_basic_info(self, retry=5):
'''
保存全市场数据
:param retry:
:return:
'''
# 需要添加异常处理 重试次数
count = 0
while count < retry:
try:
df = self.pro.stock_basic(exchange='',
list_status='',
fields='')
except Exception as e:
self.logger.info(e)
time.sleep(10)
count += 1
continue
else:
break
if count == retry:
notify(title='获取股市市场全景数据失败', desp=f'{self.__class__.__name__}')
exit(0)
if df is not None:
df = df.reset_index(drop=True)
df.rename(columns={'symbol': 'code'}, inplace=True)
df['更新日期'] = datetime.datetime.now()
engine = DBSelector().get_engine('db_stock', 'qq')
try:
df.to_sql('tb_basic_info', engine, if_exists='replace')
except Exception as e:
self.logger.error(e)
notify(title='mysql入库出错', desp=f'{self.__class__}')
def notify(self, title='', desp=''):
notify(title, desp)
