def __init__(self, send=True): super(BreakMonitor, self).__init__() self.send = send self.DB = DBSelector() self.engine = self.DB.get_engine('db_stock', 'qq') self.bases = pd.read_sql('tb_basic_info', self.engine, index_col='index')
def main(): filename = os.path.join(DATA_PATH, 'blacklist.csv') # 本地更新 db_name = 'db_stock' DB = DBSelector() conn = DB.get_mysql_conn(db_name, 'qq') create_tb(conn) update_data(filename,conn) # 远程更新 # db_name = 'db_stock' remote_conn = DB.get_mysql_conn('qdm225205669_db', 'qq') create_tb(remote_conn) update_data(filename,remote_conn)
def __init__(self): self.today = datetime.datetime.now().strftime('%Y-%m-%d') # self.TODAY = '2020-02-07' self.path = DATA_PATH if not os.path.exists(self.path): try: os.mkdir(self.path) except Exception as e: print(e) self.df_today_all = pd.DataFrame() self.TIMEOUT = 10 self.DB = DBSelector()
def process(self, data, history=False): ''' 数据存储 ''' # print(data) data_, date_ = data[0], data[1] data = eval(data_) date = eval(date_) # print(data) # print(date) df = pd.DataFrame(data) origin_columns = list(df.columns) df['date'] = date origin_columns.insert(0, 'date') # print(df.head(5)) # print(origin_columns) df = df.reindex(columns=origin_columns) # print(df.head(5)) if history == True: ''' 历史数据 ''' engine = DBSelector().get_engine('db_stock', 'qq') try: df.to_sql('bond_overview', con=engine, index_label='id', if_exists='replace', dtype={'date': DATE}) except Exception as e: print(e) else: print('pass') else: ''' 当天数据 ''' last_data = df.iloc[-1] # current_date = datetime.date if last_data['date'] == datetime.date.today().strftime('%Y-%m-%d'): # pass insert_data = list(last_data.values) insert_data = self.convert(insert_data) join_str = ','.join(['%s']*21) conn = DBSelector().get_mysql_conn('db_stock', 'qq') cursor = conn.cursor() cursor.execute( 'SELECT id FROM db_stock.bond_overview order by id desc limit 1') idx = cursor.fetchone() idx = idx[0] # insert_sql = f'''insert into `bond_overview` (`id`,`date`, `price`,`volume`,`amount`,`count`,`avg_price`,`mid_price`,`avg_premium_rt`, `avg_ytm_rt`, `increase_val`, `increase_rt`, `turnover_rt`, `price_90`, `price_90_100`, `price_100_110`, `price_110_120`, `price_120_130`, `price_130`, `idx_price`, `idx_increase_rt`)value ({join_str})''' insert_sql = f'''insert into `bond_overview` values ({join_str})''' # print(insert_sql) insert_data.insert(0, idx+1) cursor.execute(insert_sql, insert_data) conn.commit() conn.close()
def basic_info(self, retry=5): engine = DBSelector().get_engine('db_stock') # 需要添加异常处理 重试次数 count = 0 while count < retry: try: df = ts.get_stock_basics() except Exception as e: logger.info(e) time.sleep(10) count += 1 continue else: if df is not None: df = df.reset_index() df['更新日期'] = datetime.datetime.now() df.to_sql('tb_basic_info', engine, if_exists='replace') logger.info('入库成功') break else: count += 1 time.sleep(10) continue
def get_basic_info(self, retry=5): ''' 保存全市场数据 :param retry: :return: ''' engine = DBSelector().get_engine('db_stock') # 需要添加异常处理 重试次数 count = 0 while count < retry: try: df = ts.get_stock_basics() except Exception as e: self.logger.info(e) time.sleep(10) count+=1 continue else: if df is not None: df=df.reset_index() df['更新日期']=datetime.datetime.now() df.to_sql('tb_basic_info',engine,if_exists='replace') break else: count+=1 time.sleep(10) continue if count==retry: self.notify(title='获取股市市场全景数据失败',desp=f'{self.__class__}')
def __init__(self): work_space = DATA_PATH if os.path.exists(work_space) == False: os.mkdir(work_space) os.chdir(work_space) self.today = datetime.datetime.now().strftime("%Y-%m-%d") self.daily_engine = DBSelector().get_engine('db_daily', 'qq')
def __init__(self,logpath='log/collect_data.log'): super(BasicMarket, self).__init__(logpath) work_space=DATA_PATH if os.path.exists(work_space) ==False: os.mkdir(work_space) os.chdir(work_space) self.today = datetime.datetime.now().strftime("%Y-%m-%d") self.daily_engine = DBSelector().get_engine('db_daily','qq')
def __init__(self, today): ''' today 格式 20200701 :param today: ''' self.user_agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/64.0.3282.167 Chrome/64.0.3282.167 Safari/537.36" if today is None: self.today = time.strftime("%Y%m%d") else: self.today = today self.path = DATA_PATH self.zdt_url = 'http://home.flashdata2.jrj.com.cn/limitStatistic/ztForce/' + \ self.today + ".js" self.zrzt_url = 'http://hqdata.jrj.com.cn/zrztjrbx/limitup.js' self.host = "home.flashdata2.jrj.com.cn" self.reference = "http://stock.jrj.com.cn/tzzs/zdtwdj/zdforce.shtml" self.header_zdt = { "User-Agent": self.user_agent, "Host": self.host, "Referer": self.reference } self.zdt_indexx = [ '代码', '名称', '最新价格', '涨跌幅', '封成比', '封流比', '封单金额', '最后一次涨停时间', '第一次涨停时间', '打开次数', '振幅', '涨停强度' ] self.zrzt_indexx = [ '序号', '代码', '名称', '昨日涨停时间', '最新价格', '今日涨幅', '最大涨幅', '最大跌幅', '是否连板', '连续涨停次数', '昨日涨停强度', '今日涨停强度', '是否停牌', '昨天的日期', '昨日涨停价', '今日开盘价格', '今日开盘涨幅' ] self.header_zrzt = { "User-Agent": self.user_agent, "Host": "hqdata.jrj.com.cn", "Referer": "http://stock.jrj.com.cn/tzzs/zrztjrbx.shtml" } self.DB = DBSelector()
def __init__(self): today = datetime.datetime.now().strftime('%Y-%m-%d') # today = '2019-10-18' if ts.is_holiday(today): logger.info('{}假期 >>>>>'.format(today)) exit() self.DB = DBSelector() self.db_stock_engine = self.DB.get_engine('db_stock', 'qq') self.jisilu_df = self.get_code() self.code_name_dict = dict( zip(list(self.jisilu_df['可转债代码'].values), list(self.jisilu_df['可转债名称'].values))) basic_info = _json_data['mongo']['qq'] host = basic_info['host'] port = basic_info['port'] user = basic_info['user'] password = basic_info['password'] # connect_uri = f'mongodb://{user}:{password}@{host}:{port}' connect_uri = f'mongodb://{host}:{port}' self.db = pymongo.MongoClient(connect_uri)
def notice(self): buy,sell=self.run() sub = '{}: 美元汇率{}'.format(datetime.datetime.now().strftime('%Y-%m-%d %H:%M'),buy) logger.info(sub) # sendmail('',sub) conn=DBSelector().get_mysql_conn('db_stock','qq') cursor = conn.cursor() cmd = 'insert into `usd_ratio` (`price`,`date`) VALUES ({},{!r})'.format(buy,datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')) try: cursor.execute(cmd) conn.commit() except Exception as e: logger.error(e) conn.rollback() conn.close()
# -*- coding: utf-8 -*- # website: http://30daydo.com # @Time : 2020/8/3 19:01 # @File : closed_end_fund.py # 抓取封闭式基金数据 import datetime import requests from settings import llogger, DBSelector, _json_data import pymongo logger = llogger('log/fund.log') RETRY = 0 db = DBSelector() class CloseEndFundCls(): def __init__(self): self.url = 'https://www.jisilu.cn/data/cf/cf_list/' self.headers = { 'User-Agent': 'User-Agent:Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36', 'X-Requested-With': 'XMLHttpRequest' } host = _json_data['mongo']['qq']['host'] port = _json_data['mongo']['qq']['port'] user = _json_data['mongo']['qq']['user'] password = _json_data['mongo']['qq']['password'] connect_uri = f'mongodb://{user}:{password}@{host}:{port}'
class BigDeal(object): def __init__(self): today = datetime.datetime.now().strftime('%Y-%m-%d') # today = '2019-10-18' if ts.is_holiday(today): logger.info('{}假期 >>>>>'.format(today)) exit() self.DB = DBSelector() self.db_stock_engine = self.DB.get_engine('db_stock', 'qq') self.jisilu_df = self.get_code() self.code_name_dict = dict( zip(list(self.jisilu_df['可转债代码'].values), list(self.jisilu_df['可转债名称'].values))) basic_info = _json_data['mongo']['qq'] host = basic_info['host'] port = basic_info['port'] user = basic_info['user'] password = basic_info['password'] # connect_uri = f'mongodb://{user}:{password}@{host}:{port}' connect_uri = f'mongodb://{host}:{port}' self.db = pymongo.MongoClient(connect_uri) def get_ticks(self, code, date): df = ts.get_tick_data(code, date=date, src='tt') df['time'] = df['time'].map(lambda x: date + ' ' + x) return df def get_volume_distribition(self, code, date, types='min', big_deal=BIG_DEAL): ''' # 从mongo获取数据 默认分钟 1000张 :param code: :param date: :param types: :param big_deal: :return: ''' # code='110030' # date='2019-04-02' # big_deal = 1000 # 1000张 100w date_d = datetime.datetime.strptime(date, '%Y-%m-%d') next_day = date_d + datetime.timedelta(days=1) doc = self.db['cb_deal'][code] d = [] for item in doc.find({'time': { '$gte': date_d, '$lt': next_day }}, {'_id': 0}): d.append(item) if len(d) == 0: return (code, -1) df = pd.DataFrame(d) df = df.set_index('time', drop=True) min_df = df.resample(types).sum()[['price', 'volume']] count = min_df[min_df['volume'] > big_deal]['volume'].count() return (code, count) def get_tick(self, code, date): fs_df = None for i in range(10): try: fs_df = ts.get_tick_data(code, date=date, src='tt') except Exception as e: logger.error('获取tick失败>>>>code={},date'.format(code, date)) logger.error(e) time.sleep(random.randint(2, 5)) else: if fs_df is not None and len(fs_df) > 0: break else: logger.error('>>>>code={},date={} 获取行情重试 {}次'.format( code, date, i)) return fs_df def loop_code(self, date): code_list = self.jisilu_df['可转债代码'].values for code in code_list: fs_df = self.get_tick(code, date) if fs_df is None: continue fs_df['time'] = fs_df['time'].map(lambda x: date + ' ' + x) fs_df['time'] = pd.to_datetime(fs_df['time'], format='%Y-%m-%d %H:%M:%S') ret = self.store(code, fs_df) if ret.get('status') == -1: logger.error('保存失败 >>>> code={}, date={}'.format(code, date)) else: logger.info('保存成功 >>>> code={}, date={}'.format(code, date)) def store(self, code, df): df['code'] = code js = json.loads(df.T.to_json()).values() for row in js: row['time'] = datetime.datetime.utcfromtimestamp(row['time'] / 1000) try: self.db['cb_deal'][code].insert_many(js) except Exception as e: logger.error(e) logger.error('插入数据失败') return {'status': -1, 'code': code} else: return {'status': 0, 'code': code} def get_code(self): df = pd.read_sql('tb_bond_jisilu', con=self.db_stock_engine) return df def loop_date(self, today=True): ''' # 获取大单数据 # 获取当天数据,18点之后 :param today: :return: ''' if today: if ts.is_holiday(current_day): logger.info('holiday,skip>>>>{}'.format(current_day)) else: logger.info('going>>>>{}'.format(current_day)) self.loop_code(current_day) # 获取一周的数据看看 else: delta = DELTA_DAY for i in range(1, delta + 1): d = (datetime.date.today() + datetime.timedelta(days=i * -1)).strftime('%Y-%m-%d') if ts.is_holiday(d): print('holiday,skip>>>>{}'.format(d)) else: print('going>>>>{}'.format(d)) self.loop_code(d) # 发送大单数据到手机 def analysis(self, date=None, head=300): if date is None: date = datetime.date.today().strftime('%Y-%m-%d') # date='2019-12-11' kzz_big_deal_count = [] for code in self.jisilu_df['可转债代码'].values: kzz_big_deal_count.append(self.get_volume_distribition(code, date)) kzz_big_deal_order = list( sorted(kzz_big_deal_count, key=lambda x: x[1], reverse=True)) # print(kzz_big_deal_order) send_content = [] for item in kzz_big_deal_order[:head]: logger.info('{} ::: 大单出现次数 {}'.format( self.code_name_dict.get(item[0]), item[1])) send_content.append('{} ::: 大单出现次数 {}'.format( self.code_name_dict.get(item[0]), item[1])) # 入库的 big_deal_doc = self.db['db_stock']['big_deal_logger'] for item in kzz_big_deal_order: d = { 'Date': date, 'name': self.code_name_dict.get(item[0]), 'times': int(item[1]) } try: big_deal_doc.insert_one(d) # 写入mongo except Exception as e: logger.error(e) logger.error(d) # send_content.append('{} ::: 大单出现次数 {}'.format(self.code_name_dict.get(item[0]),item[1])) content = '\n'.join(send_content) title = '{}-大单监控'.format(date) try: send_aliyun(title, content, QQ_MAIL) except Exception as e: logger.error(e) else: logger.info('发送成功')
class ReachTarget(): def __init__(self): self.DB = DBSelector() self.engine = self.DB.get_engine('db_stock', 'qq') self.api = ts.get_apis() # python3 这个返回的不是list,需要手工转换 # self.kzz_code_list = list(self.stocks.keys()) # pool_code,pool_name=self.stock_pool() # self.pool_dict = dict(zip(pool_code,pool_name)) # self.pool_list= list(self.pool_dict.keys()) # 添加一部分持仓数据 或者 监测仓 # self.df = pd.read_table(file, encoding='gbk', dtype={'证券代码': np.str}) # try: # del self.df['Unnamed: 15'] # except Exception as e: # logger.error(e) # logger.error('删除多余列失败') # # code_list = list(self.df['证券代码'].values) # # # 移除非法证券代码 中签 # t = [code_list.remove(i) for i in code_list.copy() if i.startswith('7') or i[:2] == '07'] # # self.code_lists=code_list def all_bond_market(self): self.kzz_code, self.kzz_name, self.zg_code, self.name, self.yjl = self.zg_bond( ) self.kzz_stocks = dict(zip(self.kzz_code, self.kzz_name)) self.zg_stocks = dict(zip(self.zg_code, self.name)) self.kzz_stocks_yjl = dict(zip(self.kzz_code, self.yjl)) self.zg_stocks_yjl = dict(zip(self.zg_code, self.yjl)) return (self.kzz_stocks, self.zg_stocks, self.kzz_stocks_yjl, self.zg_stocks_yjl) # 数据库获取模拟股,这个要废弃 def stock_pool(self): pool_table = 'tb_current_hold' pool_df = pd.read_sql(pool_table, self.engine, index_col='index') return list(pool_df['代码'].values), list(pool_df['名字'].values) # 判断市场 def identify_market(self, x): if x.startswith('3') or x.startswith('6') or x.startswith('0'): return False else: return True # 获取当前持仓个股 def get_current_position(self): engine = self.DB.get_engine('db_position', 'qq') df = pd.read_sql('tb_position_2019-06-17', con=engine) # 只关注可转债 df = df[df['证券代码'].map(self.identify_market)] kzz_stocks = dict(zip(list(df['证券代码'].values), list(df['证券名称'].values))) cons = get_mysql_conn('db_stock', 'local') cursor = cons.cursor() query_cmd = 'select 正股代码,正股名称,溢价率 from tb_bond_jisilu where 可转债代码=%s' zg_stocks = {} kzz_yjl = {} zg_yjl = {} for code in kzz_stocks: cursor.execute(query_cmd, (code)) ret = cursor.fetchone() if ret: zg_stocks[ret[0]] = ret[1] kzz_yjl[code] = ret[2] zg_yjl[ret[0]] = ret[2] # 可转债代码 # dict,dict,dict,dict return (kzz_stocks, zg_stocks, kzz_yjl, zg_yjl) # 获取市场所有可转债数据个股代码 正股 def zg_bond(self): bond_table = 'tb_bond_jisilu' try: jsl_df = pd.read_sql(bond_table, self.engine) except Exception as e: logger.info(e) return [], [], [], [], [] else: return list(jsl_df['可转债代码']), list(jsl_df['可转债名称']), list(jsl_df['正股代码'].values), \ list(jsl_df['正股名称'].values), list(jsl_df['溢价率'].values) # 可转债的监测 def monitor(self, total_market=True): ''' total_market 默认监控全市场 total_market = True ''' if total_market: (kzz_stocks, zg_stocks, kzz_yjl, zg_yjl) = self.all_bond_market() else: (kzz_stocks, zg_stocks, kzz_yjl, zg_yjl) = self.get_current_position() zg_code = list(zg_stocks.keys()) kzz_code = list(kzz_stocks.keys()) self.has_sent_kzz = dict( zip(kzz_code, [datetime.datetime.now()] * len(kzz_code))) self.has_sent_diff = dict( zip(kzz_code, [datetime.datetime.now()] * len(kzz_code))) self.has_sent_zg = dict( zip(zg_code, [datetime.datetime.now()] * len(zg_code))) while 1: current = trading_time() if current == MARKET_OPENING: self.get_realtime_info(kzz_code, self.has_sent_kzz, '转债', kzz_stocks, kzz_yjl, ZZ_ALERT_PERCENT) self.get_realtime_info(zg_code, self.has_sent_zg, '正股', zg_stocks, zg_yjl, ZG_ALERT_PERCENT) self.get_price_diff(codes=kzz_code, has_sent_=self.has_sent_diff, types='差价', kzz_stocks=kzz_stocks, kzz_stocks_yjl=kzz_yjl) time.sleep(LOOP_TIME) elif current == -1: time.sleep(LOOP_TIME) elif current == 1: try: ts.close_apis(self.api) except Exception as e: logger.info('fail to stop monitor {}'.format( datetime.datetime.now())) logger.info(e) exit(0) # 获取实时报价 def get_realtime_info(self, codes, has_sent, types, stock, yjl, percent): try: price_df = ts.quotes(codes, conn=self.api) except Exception as e: logger.error('获取可转债异常 >>>> {}'.format(e)) try: self.api = ts.get_apis() except Exception as e: logger.error('异常中存在异常{}'.format(e)) time.sleep(EXECEPTION_TIME) else: if len(price_df) != 0: price_df = price_df[price_df['cur_vol'] != 0] price_df['percent'] = ( price_df['price'] - price_df['last_close']) / price_df['last_close'] * 100 price_df['percent'] = price_df['percent'].map( lambda x: round(x, 2)) ret_dt = \ price_df[ (price_df['percent'] > percent) | (price_df['percent'] < -1 * percent)][ ['code', 'price', 'percent']] if len(ret_dt) > 0: # 提醒一次后,下一次的间隔为DELTA_TIME分钟后 # sent_list = [] for i in ret_dt['code']: if has_sent[i] <= datetime.datetime.now(): name_list = [] yjl_list = [] name_list.append(stock[i]) yjl_list.append(yjl[i]) has_sent[i] = datetime.datetime.now( ) + datetime.timedelta(minutes=DELTA_TIME) ret_dt1 = ret_dt[ret_dt['code'] == i] ret_dt1['名称'] = name_list ret_dt1['溢价率'] = yjl_list name = ret_dt1['名称'].values[0] price = ret_dt1['price'].values[0] percent = ret_dt1['percent'].values[0] yjl_v = ret_dt1['溢价率'].values[0] now = datetime.datetime.now().strftime( '%Y-%m-%d %H:%M:%S') content0 = '{t}\n{name}:价格:{price} 涨幅:{percent},溢价率:{yjl}'.format( name=name, price=price, percent=percent, yjl=yjl_v, t=now) logger.info(content0) try: wechat.send_content(content0) except Exception as e: logger.info('发送微信失败') logger.info(e) # 获取差价 可转债 def get_price_diff(self, codes, has_sent_, types, kzz_stocks, kzz_stocks_yjl): # 针对可转债 try: df = ts.quotes(codes, conn=self.api) except Exception as e: logger.error('获取可转债异常 >>>> {}'.format(e)) try: self.api = ts.get_apis() except Exception as e: logger.error('异常中存在异常{}'.format(e)) time.sleep(EXECEPTION_TIME) else: df['bid1'] = df['bid1'].astype(float) df['ask1'] = df['ask1'].astype(float) df['diff'] = np.abs(df['bid1'] - df['ask1']) result = df[df['diff'] >= DIFF_V] if result.empty: # continue return else: for j in result['code']: if has_sent_[j] <= datetime.datetime.now(): has_sent_[j] = datetime.datetime.now( ) + datetime.timedelta(minutes=DIFF_DELTA_TIME) name_list = [] yjl_list = [] name_list.append(kzz_stocks[j]) yjl_list.append(kzz_stocks_yjl[j]) ret_dt1 = result[result['code'] == j] ret_dt1['名称'] = name_list ret_dt1['溢价率'] = yjl_list # ret_dt1 = ret_dt1.set_index('code', drop=True) code = j name = ret_dt1['名称'].values[0] price = ret_dt1['price'].values[0] bid = ret_dt1['bid1'].values[0] ask = ret_dt1['ask1'].values[0] diff = round(ret_dt1['diff'].values[0], 2) now = datetime.datetime.now().strftime( '%Y-%m-%d %H:%M:%S') content0 = '{t}\n{code}::{name}:价格:{price} 买1:{bid} 卖1:{ask}差价:{diff}'.format( code=code, name=name, price=price, bid=bid, ask=ask, diff=diff, t=now) logger.info(content0) try: wechat.send_content(content0) except Exception as e: logger.info('发送微信失败') logger.info(e)
# pd.set_option('expand_frame_repr', False) pd.set_option('display.max_rows', None) # 条件 OUTSTANDING_MAX = 70 # 流通 x亿 TOTAL_MAX = 500 # 总市值 YIJIALU = 10 # 溢价率 ZZ_PRICE = 120 # 转债价格 REMAIN_SHARE = 5 # 转股剩余比例 logger = llogger('log/filter_bond.log') today = datetime.datetime.now().strftime('%Y-%m-%d') # today='2020-03-27' DB = DBSelector() # host=_json_data['mongo']['qq']['host'] # port=_json_data['mongo']['qq']['port'] try: engine_daily = DB.get_engine('db_daily', 'qq') engine = DB.get_engine('db_stock', 'qq') jsl_df = pd.read_sql('tb_bond_jisilu', con=engine) basic_df = pd.read_sql('tb_basic_info', con=engine, index_col='index') price_df = pd.read_sql(today, con=engine_daily) INFO = _json_data['mongo']['arm'] host = INFO['host'] port = INFO['port'] user = INFO['user'] password = INFO['password'] connect_uri = f'mongodb://{user}:{password}@{host}:{port}' db = pymongo.MongoClient(connect_uri)
# -*-coding=utf-8-*- # @Time : 2020/1/1 0:08 # @File : trend.py # 统计发帖趋势 import datetime import numpy as np import pandas as pd from settings import send_from_aliyun, llogger, DBSelector last_time = -10 # 多少周之前 logger = llogger('log/trend_.log') db = DBSelector().mongo() doc = db['db_parker']['jsl'] total_list = [] date = datetime.datetime.now() + datetime.timedelta(days=-365) # 一年内的数据 def main(send_mail=True): for item in doc.find({'last_resp_date': { '$gt': date }}, { 'html': 0, 'resp': 0, 'content': 0 }): del item['_id'] total_list.append(item) df = pd.DataFrame(total_list)
# @Time : 2019/7/1 16:50 # @File : bond_daily.py # 收集可转债的市场全景图 from settings import DBSelector,send_from_aliyun,llogger import pandas as pd import datetime from config import token import tushare as ts today = datetime.datetime.now().strftime('%Y-%m-%d') today_fmt = datetime.datetime.now().strftime('%Y%m%d') # today='2020-02-06' # today_fmt='20200206' cons=ts.get_apis() logger=llogger('log/bond_daily.log') # ts.set_token(token) DB = DBSelector() # pro = ts.pro_api() conn=DB.get_mysql_conn('db_bond_daily','qq') def creat_table(day): ''' 建表 :param day: :return: ''' tb_name = 'bond_{}'.format(day) create_cmd = 'create table if not exists `{tb_name}` (`date` varchar(20),`code` varchar(10) primary key,`name` varchar(16),`open` float ,' \ '`close` float,`high` float,`low` float,`vol` float,`amount` float) '.format(tb_name=tb_name) cursor = conn.cursor() try: cursor.execute(create_cmd)
class GetZDT(BaseService): def __init__(self, today=None, logpath='log/zdt.log'): ''' TODAY 格式 20200701 :param today: ''' super(GetZDT, self).__init__(logpath) if today: self.today = today else: self.today = time.strftime("%Y%m%d") self.user_agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/64.0.3282.167 Chrome/64.0.3282.167 Safari/537.36" self.path = DATA_PATH self.zdt_url = 'http://home.flashdata2.jrj.com.cn/limitStatistic/ztForce/' + \ self.today + ".js" self.zrzt_url = 'http://hqdata.jrj.com.cn/zrztjrbx/limitup.js' self.host = "home.flashdata2.jrj.com.cn" self.reference = "http://stock.jrj.com.cn/tzzs/zdtwdj/zdforce.shtml" self.header_zdt = {"User-Agent": self.user_agent, "Host": self.host, "Referer": self.reference} self.zdt_indexx = ['代码', '名称', '最新价格', '涨跌幅', '封成比', '封流比', '封单金额', '最后一次涨停时间', '第一次涨停时间', '打开次数', '振幅', '涨停强度'] self.zrzt_indexx = ['序号', '代码', '名称', '昨日涨停时间', '最新价格', '今日涨幅', '最大涨幅', '最大跌幅', '是否连板', '连续涨停次数', '昨日涨停强度', '今日涨停强度', '是否停牌', '昨天的日期', '昨日涨停价', '今日开盘价格', '今日开盘涨幅'] self.header_zrzt = {"User-Agent": self.user_agent, "Host": "hqdata.jrj.com.cn", "Referer": "http://stock.jrj.com.cn/tzzs/zrztjrbx.shtml" } self.DB = DBSelector() def getdata(self, url, headers, retry=5): for i in range(retry): try: resp = requests.get(url=url, headers=headers) content = resp.text md_check = re.findall('summary|lasttradedate', content) if content and len(md_check) > 0: return content else: time.sleep(60) self.logger.info('failed to get content, retry: {}'.format(i)) continue except Exception as e: self.notify(title='获取涨跌停数据出错', desp=f'{self.__class__}') self.logger.error(e) time.sleep(60) continue return None def convert_json(self, content): p = re.compile(r'"Data":(.*)};', re.S) if len(content) <= 0: self.logger.info('Content\'s length is 0') exit(0) result = p.findall(content) if result: try: t1 = result[0] t2 = re.sub('[\\r\\n]', '', t1) t2 = re.sub(',,', ',0,0', t2) t2 = re.sub('Infinity', '-1', t2) t2 = re.sub('NaN', '-1', t2) t2 = list(eval(t2)) return t2 except Exception as e: self.notify(title='获取涨跌停数据出错', desp=f'{self.__class__}') self.logger.info(e) return None else: return None def save_to_dataframe(self, data, index, choice, post_fix): engine = self.DB.get_engine('db_zdt', 'qq') data_len = len(data) filename = os.path.join( self.path, self.today + "_" + post_fix + ".xls") if choice == 1: for i in range(data_len): data[i][choice] = data[i][choice] df = pd.DataFrame(data, columns=index) # 今日涨停 if choice == 1: df['今天的日期'] = self.today df.to_excel(filename, encoding='gbk') try: df.to_sql(self.today + post_fix, engine, if_exists='fail') except Exception as e: self.logger.info(e) # 昨日涨停 if choice == 2: df = df.set_index('序号') formula = lambda x: round(x * 100, 3) df['最大涨幅'] = df['最大涨幅'].map(formula) df['最大跌幅'] = df['最大跌幅'].map(formula) df['今日开盘涨幅'] = df['今日开盘涨幅'].map(formula) df['昨日涨停强度'] = df['昨日涨停强度'].map(lambda x: round(x, 0)) df['今日涨停强度'] = df['今日涨停强度'].map(lambda x: round(x, 0)) try: df.to_sql(self.today + post_fix, engine, if_exists='fail') except Exception as e: self.notify(f'{self.__class__} 出错') self.logger.info(e) avg = round(df['今日涨幅'].mean(), 2) median = round(df['今日涨幅'].median(), 2) min_v = round(df['今日涨幅'].min(), 2) min_index = df['今日涨幅'].argmin() min_percent_name = df.iloc[min_index]['名称'] current = datetime.datetime.now().strftime('%Y-%m-%d') title = '昨涨停今天{}平均涨{}\n'.format(current, avg) content = '<p>昨天涨停今天<font color="red">{}</font></p>' \ '<p>平均涨幅 <font color="red">{}</font></p>' \ '<p>涨幅中位数 <font color="red">{}</font></p>' \ '<p>涨幅最小 <font color="red">{}</font></p>' \ '<p>涨幅最小股 <font color="red">{}</font></p>'.format(current, avg, median, min_v,min_percent_name) try: send_from_aliyun(title, content, types='html') except Exception as e: print(e) # 昨日涨停今日的状态,今日涨停 def storedata(self): zdt_content = self.getdata(self.zdt_url, headers=self.header_zdt) zdt_js = self.convert_json(zdt_content) self.save_to_dataframe(zdt_js, self.zdt_indexx, 1, 'zdt') # 昨日涨停数据会如果不是当天获取会失效 zrzt_content = self.getdata(self.zrzt_url, headers=self.header_zrzt) zrzt_js = self.convert_json(zrzt_content) self.save_to_dataframe(zrzt_js, self.zrzt_indexx, 2, 'zrzt')
def __init__(self): self.DB = DBSelector() self.engine = self.DB.get_engine('db_stock', 'qq') self.api = ts.get_apis()
# @Time : 2020/1/14 0:05 # @File : fd_money.py # 涨停封单数据 from settings import DBSelector import datetime import matplotlib.pyplot as plt DB = DBSelector() conn = DB.get_mysql_conn('db_zdt', 'qq') cursor = conn.cursor() diff_day = 20 dataset = [] date = [] for d in range(diff_day): day = datetime.datetime.now() + datetime.timedelta(days=-1 * d) # if ts.is_holiday(day.strftime('%Y-%m-%d')): # continue sql = 'select 封单金额 as total_money from `{}zdt`'.format( day.strftime('%Y%m%d')) # sql = '''select sum(封单金额) as total_money from `20200113zdt`''' # print(sql) try: cursor.execute(sql) ret = cursor.fetchone() # print(ret[0]) dataset.append(int(ret[0] / 10000)) date.append(day.strftime('%Y%m%d'))
class GetZDT(object): def __init__(self, today): ''' today 格式 20200701 :param today: ''' self.user_agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/64.0.3282.167 Chrome/64.0.3282.167 Safari/537.36" if today is None: self.today = time.strftime("%Y%m%d") else: self.today = today self.path = DATA_PATH self.zdt_url = 'http://home.flashdata2.jrj.com.cn/limitStatistic/ztForce/' + \ self.today + ".js" self.zrzt_url = 'http://hqdata.jrj.com.cn/zrztjrbx/limitup.js' self.host = "home.flashdata2.jrj.com.cn" self.reference = "http://stock.jrj.com.cn/tzzs/zdtwdj/zdforce.shtml" self.header_zdt = { "User-Agent": self.user_agent, "Host": self.host, "Referer": self.reference } self.zdt_indexx = [ '代码', '名称', '最新价格', '涨跌幅', '封成比', '封流比', '封单金额', '最后一次涨停时间', '第一次涨停时间', '打开次数', '振幅', '涨停强度' ] self.zrzt_indexx = [ '序号', '代码', '名称', '昨日涨停时间', '最新价格', '今日涨幅', '最大涨幅', '最大跌幅', '是否连板', '连续涨停次数', '昨日涨停强度', '今日涨停强度', '是否停牌', '昨天的日期', '昨日涨停价', '今日开盘价格', '今日开盘涨幅' ] self.header_zrzt = { "User-Agent": self.user_agent, "Host": "hqdata.jrj.com.cn", "Referer": "http://stock.jrj.com.cn/tzzs/zrztjrbx.shtml" } self.DB = DBSelector() def getdata(self, url, headers, retry=5): for i in range(retry): try: resp = requests.get(url=url, headers=headers) content = resp.text md_check = re.findall('summary|lasttradedate', content) if content and len(md_check) > 0: return content else: time.sleep(60) logger.info('failed to get content, retry: {}'.format(i)) continue except Exception as e: sender_139('获取涨跌停数据出错', '{}'.format(e)) logger.error(e) time.sleep(60) continue return None def convert_json(self, content): p = re.compile(r'"Data":(.*)};', re.S) if len(content) <= 0: logger.info('Content\'s length is 0') exit(0) result = p.findall(content) if result: try: # print(result) t1 = result[0] t2 = re.sub('[\\r\\n]', '', t1) t2 = re.sub(',,', ',0,0', t2) t2 = re.sub('Infinity', '-1', t2) t2 = re.sub('NaN', '-1', t2) t2 = list(eval(t2)) return t2 except Exception as e: sender_139('获取涨跌停数据出错', 'e{}'.format(e)) logger.info(e) return None else: return None # 2016-12-27 to do this def save_excel(self, date, data): # data is list type w = xlwt.Workbook(encoding='gbk') ws = w.add_sheet(date) excel_filename = date + ".xls" # sheet=open_workbook(excel_filenme) # table=wb.sheets()[0] xf = 0 ctype = 1 rows = len(data) point_x = 1 point_y = 0 ws.write(0, 0, '代码') ws.write(0, 1, '名称') ws.write(0, 2, '最新价格') ws.write(0, 3, '涨跌幅') ws.write(0, 4, '封成比') ws.write(0, 5, '封流比') ws.write(0, 6, '封单金额') ws.write(0, 7, '第一次涨停时间') ws.write(0, 8, '最后一次涨停时间') ws.write(0, 9, '打开次数') ws.write(0, 10, '振幅') ws.write(0, 11, '涨停强度') print("Rows:%d" % rows) for row in data: rows = len(data) cols = len(row) point_y = 0 for col in row: # print(col) # table.put_cell(row,col,) # print(col) ws.write(point_x, point_y, col) # print("[%d,%d]" % (point_x, point_y)) point_y = point_y + 1 point_x = point_x + 1 w.save(excel_filename) def save_to_dataframe(self, data, indexx, choice, post_fix): engine = self.DB.get_engine('db_zdt', 'qq') if not data: exit() data_len = len(data) if choice == 1: for i in range(data_len): data[i][choice] = data[i][choice] df = pd.DataFrame(data, columns=indexx) filename = os.path.join(self.path, self.today + "_" + post_fix + ".xls") # 今日涨停 if choice == 1: df['今天的日期'] = self.today df.to_excel(filename, encoding='gbk') try: df.to_sql(self.today + post_fix, engine, if_exists='fail') except Exception as e: logger.info(e) # 昨日涨停 if choice == 2: df = df.set_index('序号') df['最大涨幅'] = df['最大涨幅'].map(lambda x: round(x * 100, 3)) df['最大跌幅'] = df['最大跌幅'].map(lambda x: round(x * 100, 3)) df['今日开盘涨幅'] = df['今日开盘涨幅'].map(lambda x: round(x * 100, 3)) df['昨日涨停强度'] = df['昨日涨停强度'].map(lambda x: round(x, 0)) df['今日涨停强度'] = df['今日涨停强度'].map(lambda x: round(x, 0)) try: df.to_sql(self.today + post_fix, engine, if_exists='fail') except Exception as e: logger.info(e) avg = round(df['今日涨幅'].mean(), 2) median = round(df['今日涨幅'].median(), 2) min_v = round(df['今日涨幅'].min(), 2) current = datetime.datetime.now().strftime('%Y-%m-%d') title = '昨天涨停个股今天{}\n的平均涨幅{}\n'.format(current, avg) content = '昨天涨停个股今天{}\n的平均涨幅{}\n涨幅中位数{}\n涨幅最小{}\n'.format( current, avg, median, min_v) # try: # (title, content) # except Exception as e: # print(e) # 昨日涨停今日的状态,今日涨停 def storedata(self): zdt_content = self.getdata(self.zdt_url, headers=self.header_zdt) # logger.info('zdt Content' + zdt_content) zdt_js = self.convert_json(zdt_content) self.save_to_dataframe(zdt_js, self.zdt_indexx, 1, 'zdt') # 昨日涨停数据会如果不是当天获取会失效 time.sleep(0.5) zrzt_content = self.getdata(self.zrzt_url, headers=self.header_zrzt) logger.info('zrzt Content' + zrzt_content) zrzt_js = self.convert_json(zrzt_content) self.save_to_dataframe(zrzt_js, self.zrzt_indexx, 2, 'zrzt')
class FetchDaily(object): def __init__(self): self.today = datetime.datetime.now().strftime('%Y-%m-%d') # self.TODAY = '2020-02-07' self.path = DATA_PATH if not os.path.exists(self.path): try: os.mkdir(self.path) except Exception as e: print(e) self.df_today_all = pd.DataFrame() self.TIMEOUT = 10 self.DB = DBSelector() def gettodaymarket(self, re_try=10): while re_try > 0: try: df = ts.get_today_all() if df is None: continue if len(df) == 0: continue except Exception as e: logger.error(e) re_try = re_try - 1 time.sleep(self.TIMEOUT) # import tushare as ts else: return df return None def store(self): self.df_today_all = self.gettodaymarket() # 存储每天 涨幅排行 榜,避免每次读取耗时过长 filename = self.today + '_all_.xls' # 放在data文件夹下 full_filename = os.path.join(self.path, filename) if self.df_today_all is not None: # 保留小数点的后两位数 self.df_today_all['turnoverratio'] = self.df_today_all[ 'turnoverratio'].map(lambda x: round(x, 2)) self.df_today_all['per'] = self.df_today_all['per'].map( lambda x: round(x, 2)) self.df_today_all['pb'] = self.df_today_all['pb'].map( lambda x: round(x, 2)) try: self.df_today_all.to_excel(full_filename) except Exception as e: logger.error(e) engine = self.DB.get_engine('db_daily', 'qq') # print(self.df_today_all) try: self.df_today_all.to_sql(self.today, engine, if_exists='fail') except Exception as e: # print(e) logger.error(e) else: logger.error('today_all df is None') def store_new(self): self.df_today_all = self.gettodaymarket() filename = self.today + '_all_.xls' full_filename = os.path.join(self.path, filename) if not os.path.exists(full_filename): if self.df_today_all is not None: try: self.save_to_excel(self.df_today_all, full_filename) except Exception as e: print(e) engine = self.DB.get_engine('db_daily', 'qq') try: self.df_today_all.to_sql(self.today, engine) except Exception as e: print(e) pass def save_to_excel(self, df, filename, encoding='gbk'): try: df.to_csv('temp.csv', encoding=encoding, index=False) df = pd.read_csv('temp.csv', encoding=encoding, dtype={'code': str}) df.to_excel(filename, encoding=encoding) return True except Exception as e: print("Save to excel faile") print(e) return None
# website: http://30daydo.com # @Time : 2019/3/19 23:21 # @File : auto_trader.py import datetime import logging import time import pymongo import easyquotation import easytrader import pandas as pd from config import PROGRAM_PATH, MONGO_PORT, MONGO_HOST from settings import DBSelector SELL = 7 # 配置为8%个点卖 DB = DBSelector() class AutoTrader(): def __init__(self): self.today = datetime.date.today().strftime('%Y-%m-%d') # self.stock_candidates = self.get_candidates() # self.stock_candidates = self.get_candidates() self.logger = self.llogger('log/auto_trader_{}'.format(self.today)) self.logger.info('程序启动') self.user = easytrader.use('gj_client') # self.user = easytrader.use('ths') self.user.prepare('user.json') # self.user.connect(PROGRAM_PATH)
# -*-coding=utf-8-*- # 获取 不同形态的k线 import random import time import tushare as ts import pandas as pd import os, datetime, math import numpy as np import logging from settings import DBSelector, MYSQL_HOST, MYSQL_PORT, MYSQL_USER, MYSQL_PASSWORD, REDIS_HOST import redis from threading import Thread from BaseService import BaseService DB = DBSelector() engine = DB.get_engine('history', 'qq') conn = ts.get_apis() MYSQL_DB = 'history' cursor = DB.get_mysql_conn(MYSQL_DB, 'qq').cursor() # pd.set_option('display.max_rows', None) class Kline(BaseService): def __init__(self): super(Kline, self).__init__('log/kline.log') path = os.path.join(os.getcwd(), 'data') self.today_date = datetime.datetime.now().strftime('%Y-%m-%d') if not os.path.exists(path):
NOTIFY_HOUR = 13 MAX_PAGE = 114 headers = { 'Connection': 'keep-alive', 'Pragma': 'no-cache', 'Cache-Control': 'no-cache', 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36', 'Accept': '*/*', # 'Referer': 'http://stockapp.finance.qq.com/mstats/?id=fund_close', 'Accept-Encoding': 'gzip, deflate', 'Accept-Language': 'zh,en;q=0.9,en-US;q=0.8', } DB = DBSelector() conn = DB.get_mysql_conn('db_fund', 'qq') cursor = conn.cursor() base_dir = os.path.dirname(os.path.abspath(__file__)) log_path = os.path.join(base_dir, '..', 'log') logger = llogger(os.path.join(log_path, 'fund_info.log')) class FundSpider(object): def __init__(self): self.create_tb() def create_tb(self): create_table = 'create table if not EXISTS `{}` (`基金代码` varchar(20) PRIMARY KEY,`基金简称` varchar(100),`最新规模-万` float,' \ '`实时价格` float,`涨跌幅` float,`成交额-万` float,`净值日期` VARCHAR(10),`单位净值` float,`累计净值` float,`折溢价率` float ,`申购状态` VARCHAR(20),`申赎状态` varchar(20),`基金经理` VARCHAR(200),' \ '`成立日期` VARCHAR(20), `管理人名称` VARCHAR(200),`实时估值` INT,`QDII` INT ,`更新时间` VARCHAR(20));'.format(
''' http://30daydo.com Contact: [email protected] ''' from settings import DBSelector import pandas as pd import pymongo pd.set_option('expand_frame_repr', False) client = pymongo.MongoClient('raspberrypi') db = client['stock'] doc = db['industry'] today = '2018-05-08' # TODAY = datetime.datetime.now().strftime('%Y-%m-%d') daily_engine = DBSelector().get_engine('db_daily', 'qq') daily_df = pd.read_sql(today, daily_engine, index_col='index') # 保存到mongo def save_industry(): try: doc.drop() except Exception as e: print(e) engine = get_engine('db_stock') basic_df = pd.read_sql('tb_basic_info', engine, index_col='index') # print(basic_df) for name, group in basic_df.groupby('industry'): # print(name, group)
# 同步两个mongodb的数据 import pymongo from settings import DBSelector from loguru import logger logger.add('syncup.log') db = DBSelector() client = db.mongo('qq') remote = client['db_parker']['jsl'] local = pymongo.MongoClient()['db_parker']['jsl'] remote_data = remote.find() # 更新本地数据 def update(item, question_id, update=False): del item['_id'] if update: local.update_one( {'question_id': question_id}, {'$set': { 'resp': item['resp'], 'resp_no': item['resp_no'] }}) else: local.insert_one(item) remote.delete_one({'question_id': question_id}) def remove(item): remote.delete_one({'_id': item['_id']})
import sys __author__ = 'Rocky' ''' http://30daydo.com Contact: [email protected] ''' # 交割单处理 保存交割单到数据库 import os import datetime import pandas as pd import numpy as np import matplotlib.pyplot as plt from settings import DBSelector import fire DB = DBSelector() engine = DB.get_engine('db_stock', 'qq') conn = DB.get_mysql_conn('db_stock', 'qq') pd.set_option('display.max_rows', None) class DeliveryOrder(): def __init__(self): self.gj_table = 'tb_delivery_gj_django' self.hb_table = 'tb_delivery_hb_django' def setpath(self, path): path = os.path.join(os.getcwd(), path) if os.path.exists(path) == False: os.mkdir(path) os.chdir(path)
from mpl_finance import candlestick2_ochl, volume_overlay import matplotlib.pyplot as plt from settings import DBSelector from settings import llogger import sys if sys.platform == 'linux': # centos的配置, 根据自定义拷贝的字体 mpl.rcParams['font.sans-serif'] = ['Microsoft YaHei'] else: mpl.rcParams['font.sans-serif'] = ['simhei'] mpl.rcParams['axes.unicode_minus'] = False logger = llogger('log/plot_line.log') DB = DBSelector() engine = DB.get_engine('db_stock', 'qq') base_info = pd.read_sql('tb_basic_info', engine, index_col='index') def plot_stock_line(api, code, name, table_name, current, start='2019-10-01', save=False): title = '{}_{}_{}_{}'.format(current, code, name, table_name) title = title.replace('*', '_') if os.path.exists(title + '.png'):
class BreakMonitor(BaseService): def __init__(self, send=True): super(BreakMonitor, self).__init__() self.send = send self.DB = DBSelector() self.engine = self.DB.get_engine('db_stock', 'qq') self.bases = pd.read_sql('tb_basic_info', self.engine, index_col='index') def read_stock_list(self, file=None): if file: with open(file, 'r') as f: monitor_list = f.readlines() monitor_list = list(map(lambda x: x.strip(), monitor_list)) else: monitor_list = ['300100'] return monitor_list def percent(self, current, close): return (current - close) * 1.0 / close * 100 # 开板提示 def break_ceil(self, code): print(threading.current_thread().name) while 1: # 交易时间 if trading_time() != 0: break try: df = ts.get_realtime_quotes(code) except Exception as e: self.logger.error(e) time.sleep(5) continue v = float(df['b1_v'].values[0]) if self.percent(float(df.iloc[0]['price']), float(df.iloc[0]['pre_close'])) < 9: if self.send == True: title = f'{code}已经板了' self.notify(title) break if v <= 1000: print(datetime.datetime.now().strftime("%H:%M:%S")) print(u"小于万手,小心!跑") print(self.bases[self.bases['code'] == code]['name'].values[0]) if self.send == True: title = f'{code}开板了' self.notify(title) time.sleep(10) def monitor_break(self): thread_num = len(self.read_stock_list()) # 线程数和股票输一样 thread_list = [] for i in range(thread_num): t = threading.Thread(target=self.break_ceil, args=(self.read_stock_list()[i], )) thread_list.append(t) for j in thread_list: j.start() for k in thread_list: k.join()