def run(self): self.df_today_all = self.get_today_market() # 存储每天 涨幅排行 榜,避免每次读取耗时过长 filename = self.today + '_all_.xls' # 放在data文件夹下 full_filename = os.path.join(self.path, filename) if self.df_today_all is not None: # 保留小数点的后两位数 self.df_today_all['turnoverratio'] = self.df_today_all[ 'turnoverratio'].map(lambda x: round(x, 2)) self.df_today_all['per'] = self.df_today_all['per'].map( lambda x: round(x, 2)) self.df_today_all['pb'] = self.df_today_all['pb'].map( lambda x: round(x, 2)) try: self.df_today_all.to_excel(full_filename) except Exception as e: notify(title='写excel出错', desp=f'{self.__class__}') self.logger.error(e) try: self.df_today_all.to_sql(self.today, self.engine, if_exists='fail') except Exception as e: notify(title='mysql出错', desp=f'{self.__class__}') self.logger.error(e)
def predict_price(self): session = requests.Session() session.get(url=home_page, headers=home_headers) r = session.get(url=xueqiu_url, headers=headers) js_data = r.json() quote = js_data.get('data', {}).get('quote') quote['crawltime'] = datetime.datetime.now() doc.insert_one(quote) percent = quote.get('percent') jsl_qdii, est_val_dt = self.qdii_info() if jsl_qdii: predict_v = round((1 + percent * 0.95 * 0.01) * jsl_qdii, 3) self.logger.info(f'最新估值{predict_v}') d = {'日期': today, '估值': predict_v} client['db_stock']['huabaoyouqi_predict'].insert_one(d) title = f'华宝估值{predict_v} 净值日期{est_val_dt[5:]}' send_from_aliyun(title, '') else: notify(title='华宝油气获取估值失败')
def yesterday(self, df, post_fix, engine): df = df.set_index('序号') formula = lambda x: round(x * 100, 3) df['最大涨幅'] = df['最大涨幅'].map(formula) df['最大跌幅'] = df['最大跌幅'].map(formula) df['今日开盘涨幅'] = df['今日开盘涨幅'].map(formula) df['昨日涨停强度'] = df['昨日涨停强度'].map(lambda x: round(x, 0)) df['今日涨停强度'] = df['今日涨停强度'].map(lambda x: round(x, 0)) try: df.to_sql(self.today + post_fix, engine, if_exists='fail') except Exception as e: notify(f'{self.__class__} 出错') self.logger.info(e) avg = round(df['今日涨幅'].mean(), 2) median = round(df['今日涨幅'].median(), 2) min_v = round(df['今日涨幅'].min(), 2) min_index = df['今日涨幅'].argmin() min_percent_name = df.iloc[min_index]['名称'] current = datetime.datetime.now().strftime('%Y-%m-%d') title = '昨涨停今天{}平均涨{}\n'.format(current, avg) content = '<p>昨天涨停今天<font color="red">{}</font></p>' \ '<p>平均涨幅 <font color="red">{}</font></p>' \ '<p>涨幅中位数 <font color="red">{}</font></p>' \ '<p>涨幅最小 <font color="red">{}</font></p>' \ '<p>涨幅最小股 <font color="red">{}</font></p>'.format(current, avg, median, min_v, min_percent_name) try: send_from_aliyun(title, content, types='html') except Exception as e: self.logger.error(e)
def get_basic_info(self, retry=5): ''' 保存全市场数据 :param retry: :return: ''' # 需要添加异常处理 重试次数 count = 0 while count < retry: try: df = ts.get_stock_basics() except Exception as e: self.logger.info(e) time.sleep(10) count+=1 continue else: break if count==retry: notify(title='获取股市市场全景数据失败',desp=f'{self.__class__}') exit(0) if df is not None: df = df.reset_index() df['更新日期'] = datetime.datetime.now() engine = DBSelector().get_engine('db_stock','qq') try: df.to_sql('tb_basic_info', engine, if_exists='replace',index='index',index_label='id') except Exception as e: self.logger.error(e) notify(title='mysql入库出错',desp=f'{self.__class__}')
def save_mysql(self, df): conn2 = DB.get_engine('double_low_full', 'qq') try: df.to_sql(f'double_low_{self.today}', con=conn2, if_exists='replace') except: notify(title='mysql入库出差', desp=f'{self.__class__}')
def start(self): buy, sell = self.run() sub = '{}: 美元汇率{}'.format( datetime.datetime.now().strftime('%Y-%m-%d %H:%M'), buy) self.logger.info(sub) conn = DBSelector().get_mysql_conn('db_stock', 'qq') cmd = 'insert into `usd_ratio` (`price`,`date`) VALUES ({},{!r})'.format( buy, datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')) self.execute(cmd, (), conn) notify(title=sub, desp='')
def run(self): content = self.crawl() if content is None: self.logger.error('爬取内容为空') return rows = content.get('rows') row_list = list(map(lambda x: x.get('cell'), rows)) if not self.save_mongo(row_list): self.logger.info('保存失败') notify(title='jsl分级入库出错', desp=f'{self.__class__}')
def main(): # current='20191016' current = datetime.datetime.now().strftime('%Y%m%d') app = PlotYesterdayZT() api = ts.get_apis() for plot_type in ['zrzt', 'zdt']: try: app.plot_yesterday_zt(api, plot_type, current=current) except Exception as e: notify(title='zdt_plot 出错', desp=f'{__name__}') continue ts.close_apis(conn=api)
def parse_json(self, types): if types == 'stock': url = self.stock_url mongo_doc = self.jsl_stock_lof else: url = self.index_lof_url mongo_doc = self.jsl_index_lof return_js = self.get(url=url) rows = return_js.get('rows') for item in rows: cell = item.get('cell') try: mongo_doc.insert_one(cell) except Exception as e: self.logger.error(e) notify(title='入mongo出错', desp=f'{self.__class__} 写入mongodb出错')
def main(current=None): # current='20191016' if current is None: current = datetime.datetime.now().strftime('%Y%m%d') if isinstance(current, int): current = str(current) app = PlotYesterdayZT() api = ts.get_apis() for plot_type in ['zrzt', 'zdt']: try: app.plot_yesterday_zt(api, plot_type, current=current) except Exception as e: print(plot_type, 'error') print(e) notify(title='zdt_plot 出错', desp=f'{__name__}') continue ts.close_apis(conn=api)
def yesterday_zt(self, df, post_fix, engine): df = df.set_index('序号') formula = lambda x: round(x * 100, 3) df['最大涨幅'] = df['最大涨幅'].map(formula) df['最大跌幅'] = df['最大跌幅'].map(formula) df['今日开盘涨幅'] = df['今日开盘涨幅'].map(formula) df['昨日涨停强度'] = df['昨日涨停强度'].map(lambda x: round(x, 0)) df['今日涨停强度'] = df['今日涨停强度'].map(lambda x: round(x, 0)) try: df.to_sql(self.today + post_fix, engine, if_exists='fail') except Exception as e: notify(f'{self.__class__} 出错') self.logger.info(e) title, content = self.generate_html(df) try: send_from_aliyun(title, content, types='html') except Exception as e: self.logger.error(e)
def convert_json(self, content): p = re.compile(r'"Data":(.*)};', re.S) if len(content) <= 0: self.logger.info('Content\'s length is 0') exit(0) result = p.findall(content) if result: try: t1 = result[0] t2 = re.sub('[\\r\\n]', '', t1) t2 = re.sub(',,', ',0,0', t2) t2 = re.sub('Infinity', '-1', t2) t2 = re.sub('NaN', '-1', t2) t2 = list(eval(t2)) return t2 except Exception as e: notify(title='获取涨跌停数据出错', desp=f'{self.__class__}') self.logger.info(e) return None else: return None
def download(self, url, headers, retry=5): for i in range(retry): try: resp = requests.get(url=url, headers=headers) content = resp.text md_check = re.findall('summary|lasttradedate', content) if content and len(md_check) > 0: return content else: time.sleep(60) self.logger.info( 'failed to get content, retry: {}'.format(i)) continue except Exception as e: notify(title='获取涨跌停数据出错', desp=f'{self.__class__}') self.logger.error(e) time.sleep(60) continue return None
def get_basic_info(self, retry=5): ''' 保存全市场数据 :param retry: :return: ''' # 需要添加异常处理 重试次数 count = 0 while count < retry: try: df = self.pro.stock_basic(exchange='', list_status='', fields='') except Exception as e: self.logger.info(e) time.sleep(10) count += 1 continue else: break if count == retry: notify(title='获取股市市场全景数据失败', desp=f'{self.__class__.__name__}') exit(0) if df is not None: df = df.reset_index(drop=True) df.rename(columns={'symbol': 'code'}, inplace=True) df['更新日期'] = datetime.datetime.now() engine = DBSelector().get_engine('db_stock', 'qq') try: df.to_sql('tb_basic_info', engine, if_exists='replace') except Exception as e: self.logger.error(e) notify(title='mysql入库出错', desp=f'{self.__class__}')
def notify(self, title='', desp=''): notify(title, desp)