def get_futunn_live(self):
lasttime = DateUtil.string_toDatetime(self.mongodbutil_live.getLastLivetime())
for i in range(0,-1,-1):
p = int(1000*time.mktime(time.localtime())) + i
url = 'https://news.futunn.com/main/live-list?page={0}page_size=50&_=1556778263374'.format(i,p)
logger.info("address current url {0}...".format(url))
arr = []
header = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}
try:
res = requests.get(url, headers=header, timeout=60)
res.raise_for_status()
if res.status_code == 200:
data = res.text
js = json.loads(data)
list = js['data']['list']
for elem in list:
itemTime = DateUtil.string_toDatetime(elem['time'])
if itemTime > lasttime:
arr.append( elem )
logger.info(elem)
else:
continue
if len(arr) > 0 :
self.mongodbutil_live.insertItems(arr)
logger.info("store items to mongodb ...")
else:
logger.info("still have no new live message")
except Exception as err:
#time.sleep(4 * random.random())
logger.warning(err)
except requests.exceptions.ConnectTimeout as err:
logger.warning(err)
ret_code = -1
ret_data = err
except requests.exceptions.ReadTimeout as err:
logger.warning(err)
ret_code = -1
ret_data = err
except requests.exceptions.Timeout as err:
logger.warning(err)
ret_code = -1
ret_data = err
except:
logger.warning('Unfortunitely -- An Unknow Error Happened, Please wait 3 seconds')
time.sleep(random.random())
ret_code = -1
ret_data = ''
finally:
res.close()
return 1, 'ok'
def job_once_global_tick(ts):
"""
获取全部商品分笔数据
:param ts:
:return:
"""
global is_closing
df = ts.get_stock_basics()
if df is None:
return
cont = False
#lastsymbol = '300598'
for symbol in df['name'].index.values:
# if symbol == lastsymbol:
# cont = True
# if not cont:
# continue
for ndays in range(1, 2, 1):
if not is_closing:
day = DateUtil.getDatetimePastStr(DateUtil.getDatetimeToday(),
ndays)
if not is_holiday(day):
ts.get_tick_data(symbol, day)
time.sleep(1)
else:
logging.info('is holiday')
if not is_closing:
time.sleep(2)
def job_realtime_custom(ts):
ts.get_today_ticks(AppConfig.custom_stocks)
ts.get_realtime_quotes(AppConfig.custom_stocks)
ts.get_realtime_quotes(AppConfig.custom_indexes)
for symbol in AppConfig.custom_stocks:
ts.get_tick_data(
symbol,
DateUtil.getDatetimeYesterdayStr(DateUtil.getDatetimeToday()))
def get_latest_news(self,top=5, show_content=True):
'''
功能：
即时新闻
参数说明：
top:int，显示最新消息的条数，默认为80条
show_content:boolean,是否显示新闻内容，默认False
结果返回的数据属性说明如下：
classify :新闻类别
title :新闻标题
time :发布时间
url :新闻链接
content:新闻内容（在show_content为True的情况下出现）
'''
df = None
try:
if time.time() - AppConfig.latest_news_pulltime > 1000:
top = 80
df = ts.get_latest_news(top, show_content)
if df is None:
logging.info('df is None')
return
table = 'ts2_latest_news'
latest_pulltime = None
pulltime = None
dropindex = -1
df.sort_values(by="time", ascending=False)
for i in range(0,len(df)):
pulltime = df.iloc[i]['time']
pulltime = DateUtil.string_toTimestamp(DateUtil.format_date(pulltime))
if i == 0:
latest_pulltime = pulltime
if pulltime <= AppConfig.latest_news_pulltime:
#remove
dropindex = i
print('dropindex:',dropindex)
break
if dropindex != -1:
df = df.drop(range(dropindex,len(df),1))
if len(df) > 0 :
self.storeservice.insert_many(table, df)
AppConfig.write_news_pulltime(latest_pulltime,True)
except IOError as err:
logging.error("OS|error: {0}".format(err))
except OperationalError as err:
logging.error("OS|error: {0}".format(err))
else:
pass
def job_realtime_global(ts):
"""
交易日里读取指数信息和全盘Snap(20秒左右)
:param ts:
:return:
"""
global is_closing
today = DateUtil.getDatetimeToday()
while not is_closing and not is_holiday(DateUtil.date_toString(today)):
ts.get_index()
time.sleep(3)
ts.get_today_all()
time.sleep(5)
def append_one_stock(worker, code, dtype, ktype, listing_date):
'''
:param worker:
:param code:
:param dtype:
:param last_fetchdate:
:param listing_date:
:return:
'''
global is_closing
todayStr = DateUtil.getTodayStr()
# last_fetchdate = DateUtil.string_toDate( DateUtil.getDatetimePastStr( DateUtil.string_toDate(todayStr),30) )
last_fetchdate = DateUtil.string_toDate('2018-08-02')
start = None
ld, tindex = worker.storeservice.find_lastdate_and_tindex(code, dtype)
lastdate = last_fetchdate if ld == None else ld
if lastdate is not None and lastdate.date() > listing_date:
start = DateUtil.getDatetimeFutureStr(lastdate.date(), 1)
else:
# if listing_date.year == 1970:
# listing_date = listing_date.replace(year=1997)
# start = DateUtil.date_toString(listing_date)
start = DateUtil.date_toString(last_fetchdate)
end = todayStr
gen = DateUtil.getNextHalfYear(DateUtil.string_toDate(start),
DateUtil.string_toDate(end))
while True:
try:
end = next(gen)
if is_closing is True:
break
b2 = time.time()
_, _, lastest_date = worker.get_history_kline(
code, tindex, start, end, ktype)
if lastest_date is not None:
worker.storeservice.update_lastdate(
code, dtype, DateUtil.string_toDatetime(lastest_date))
e2 = time.time()
logging.info(
"fetching {} dtype {} ktype {} listing_date: {} start: {} end:{} cost time {}"
.format(code, dtype, ktype, listing_date, start, end, e2 - b2))
start = DateUtil.getDatetimeFutureStr(DateUtil.string_toDate(end),
1)
except StopIteration as e:
print(e)
break
def get_index(self):
'''
功能：
获取大盘指数行情
返回值说明：
code:指数代码
name:指数名称
change:涨跌幅
open:开盘点位
preclose:昨日收盘点位
close:收盘点位
high:最高点位
low:最低点位
volume:成交量(手)
amount:成交金额（亿元）
'''
try:
df = ts.get_index()
if df is None:
return
df['date'] = DateUtil.getTodayStr()
df = df.reset_index(level=[0])
del df['index']
table = 'ts2_index'
self.storeservice.insert_many(table, df, 'replace')
except IOError as err:
logging.error("OS|error: {0}".format(err))
except OperationalError as err:
logging.error("OS|error: {0}".format(err))
else:
pass
def job_once_global_day_analysis(*_args):
'''
线程工作：低频数据接口
:return:
'''
global is_closing
global jump_pd
worker = _args[0][0]
arr = _args[0][1]
while not is_closing:
begin = time.time()
ret_arr = arr
total = len(ret_arr)
curr = 0
todayStr = DateUtil.getTodayStr()
for code, listing_date in ret_arr:
curr += 1
logging.info("current fetching progress {}/{} code:{} ".format(curr, total, code))
if curr < 1:
continue
b1 = time.time()
ret_list = worker.storeservice.find_history_kline(code, 'hk', listing_date, todayStr)
if len(ret_list) <= 0:
logging.info(
"fetching {} K_LINE, but no data".format(code))
continue
col_list = ['code', 'time_key', 'open', 'close', 'high', 'low', 'pe_ratio', 'turnover_rate', 'volume',
'turnover', 'change_rate', 'last_close']
data_pd = pd.DataFrame(ret_list, columns=col_list)
jump_threshold = data_pd.close.median() * 0.03
if abs(jump_threshold) > 0.01:
for index in np.arange(0,data_pd.shape[0]):
today = data_pd.ix[index]
judge_jump(today,jump_threshold)
e1 = time.time()
logging.info(
"fetching {} K_LINE listing_date:{} start: {} end:{} cost time {}".format(code, listing_date, listing_date,
todayStr, e1 - b1))
if is_closing is True:
break
if len(jump_pd) > 0:
table = 'ft_stat_jump'
worker.storeservice.insert_many(table, jump_pd, 'append')
end = time.time()
logging.info("fetching for one period , cost time: {}".format((end - begin)))
break
def job_realtime_custom(ts):
"""
1.获取定制商品的实时数据，tick, quote
2.TODO 全部商品需要分布式环境，有调度任务获取
:param ts:
:return:
"""
global is_closing
while not is_closing:
ts.get_today_ticks(AppConfig.custom_stocks)
ts.get_realtime_quotes(AppConfig.custom_stocks)
ts.get_realtime_quotes(AppConfig.custom_indexes)
for symbol in AppConfig.custom_stocks:
ts.get_tick_data(
symbol,
DateUtil.getDatetimeYesterdayStr(DateUtil.getDatetimeToday()))
time.sleep(1)
time.sleep(5)
def file_size_control():
if os.path.getsize(logname) > 100000000:
filehandler.close()
os.rename(
logname,
logname.replace('log.txt',
"log-{}.txt".format(DateUtil.getDatetimeToday4())))
file = open(logname, "w", encoding='utf-8', errors='ignore')
file.seek(0)
file.truncate()
def _gen_warp_df(self, warp_self, symbol):
"""
封装通用的数据解析规范及流程
:param warp_self: 被封装类init中使用的self对象
:param symbol: 请求的symbol str对象
:return:
"""
# 规范原始init函数中必须为类添加了如下属性
must_col = [
'code', 'open', 'close', 'high', 'low', 'volume', 'time_key'
]
# 检测所有的属性都有
all_has = all([hasattr(warp_self, col) for col in must_col])
if all_has:
# 将时间序列转换为pd时间
dates_pd = pd.to_datetime(warp_self.time_key)
# 构建df，index使用dates_pd
warp_self.df = pd.DataFrame(index=dates_pd)
for col in must_col:
# 所以必须有的类属性序列设置给df的列
warp_self.df[col] = getattr(warp_self, col)
# 从收盘价格序列shift出昨收价格序列
warp_self.df['last_close'] = warp_self.df['close'].shift(1)
warp_self.df.drop(
warp_self.df[warp_self.df.last_close.isnull()].index,
inplace=True)
# 添加日期int列
warp_self.df['time_key'] = warp_self.df['time_key'].apply(
lambda x: DateUtil.string_toDate(x))
# 添加周几列date_week，值为0-4，分别代表周一到周五
# warp_self.df['date_week'] = warp_self.df['date'].apply(
# lambda x: week_of_date(str(x), '%Y%m%d'))
# 类型转换
warp_self.df['close'] = warp_self.df['close'].astype(float)
warp_self.df['high'] = warp_self.df['high'].astype(float)
warp_self.df['low'] = warp_self.df['low'].astype(float)
warp_self.df['open'] = warp_self.df['open'].astype(float)
warp_self.df['volume'] = warp_self.df['volume'].astype(float)
warp_self.df['volume'] = warp_self.df['volume'].astype(np.int64)
warp_self.df['last_close'] = warp_self.df['last_close'].astype(
float)
# noinspection PyTypeChecker
warp_self.df['change_rate'] = np.where(
warp_self.df['last_close'] == 0, 0,
(warp_self.df['close'] - warp_self.df['last_close']) /
warp_self.df['last_close'] * 100)
warp_self.df['change_rate'] = warp_self.df['change_rate'].apply(
lambda x: round(x, 3))
# 给df加上name
warp_self.df.name = symbol
def job_global_remedy_task(ts):
"""
定时作业，收盘数据：分笔数据，5日数据和日线数据
:param ts:
:return:
"""
global is_closing
df = ts.get_stock_basics()
if df is None:
return
ndays = 1
startDate = '2018-07-09'
endDate = DateUtil.getDatetimeFutureStr(DateUtil.string_toDate(startDate),
ndays)
days = []
days.append(startDate)
for day in range(1, ndays, 1):
continueDate = DateUtil.getDatetimeFutureStr(
DateUtil.string_toDate(startDate), day)
days.append(continueDate)
cont = False
#lastsymbol = '603895'
for symbol in df['name'].index.values:
# if symbol == lastsymbol:
# cont = True
# if not cont:
# continue
for day in days:
if not is_closing:
if not is_holiday(day):
ts.get_tick_data(symbol, day)
time.sleep(1)
else:
logging.info('is holiday')
ts.get_hist_data(symbol, startDate, endDate, '5')
time.sleep(2)
ts.get_hist_data(symbol, startDate, endDate)
time.sleep(2)
def find_history_kline(self, code, dtype, start, end):
tindex = self.find_tindex(code, dtype)
cls = getClassByIndex(tindex)
ret = []
#for item in self.mysqlStore.session.query(FTHistoryKline6).filter(cls.code == code).filter(and_(text('time_key>:start'),text('time_key<:end')).params(start=start,end=end)).all():
#,cls.code,cls.time_key,cls.open,cls.close,cls.high,cls.low,cls.pe_ratio,cls.turnover_rate,cls.volume,cls.turnover,cls.change_rate,cls.last_close
#(code, time_key, open, close, high, low, pe_ratio, turnover_rate, volume, turnover, change_rate, last_close)
for code, time_key, open, close, high, low, pe_ratio, turnover_rate, volume, turnover, change_rate, last_close in self.mysqlStore.session.query(
cls.code, cls.time_key, cls.open, cls.close, cls.high, cls.low,
cls.pe_ratio, cls.turnover_rate, cls.volume, cls.turnover,
cls.change_rate,
cls.last_close).filter(cls.code == code).filter(
and_(cls.time_key >= start,
cls.time_key <= end)).order_by(cls.time_key).all():
item = (code, DateUtil.datetime_toString(time_key), open, close,
high, low, pe_ratio, turnover_rate, volume, turnover,
change_rate, last_close)
ret.append(item)
return ret
def get_today_ticks(self,code,retry_count=3,pause=0):
'''
功能：
获取当前交易日（交易进行中使用）已经产生的分笔明细数据。
参数说明：
code：股票代码，即6位数字代码
retry_count : int, 默认3,如遇网络等问题重复执行的次数
pause : int, 默认 0,重复请求数据过程中暂停的秒数，防止请求间隔时间太短出现的问题
返回值说明：
time：时间
price：当前价格
pchange:涨跌幅
change：价格变动
volume：成交手
amount：成交金额(元)
type：买卖类型【买盘、卖盘、中性盘】
'''
try:
df = ts.get_today_ticks(code,retry_count,pause)
if df is None:
return
df = df.replace('--', 0)
df['pchange'] = df['pchange'].astype(float)
df['date'] = DateUtil.getTodayStr()
df['code'] = code
df = df.reset_index(level=[0])
del df['index']
table = 'ts2_today_ticks'
self.storeservice.insert_many(table, df)
except IOError as err:
logging.error("OS|error: {0}".format(err))
except OperationalError as err:
logging.error("OS|error: {0}".format(err))
else:
pass
def get_tick_data(self,code, date=None, retry_count=3, pause=1):
'''
功能：
获取个股以往交易历史的分笔数据明细，通过分析分笔数据，可以大致判断资金的进出情况。在使用过程中，对于获取股票某一阶段的历史分笔数据，需要通过参入交易日参数并append到一个DataFrame或者直接append到本地同一个文件里。历史分笔接口只能获取当前交易日之前的数据，当日分笔历史数据请调用get_today_ticks()接口或者在当日18点后通过本接口获取。
参数说明：
code：股票代码，即6位数字代码
date：日期，格式YYYY-MM-DD
retry_count : int, 默认3,如遇网络等问题重复执行的次数
pause : int, 默认 0,重复请求数据过程中暂停的秒数，防止请求间隔时间太短出现的问题
返回值说明：
time：时间
price：成交价格
change：价格变动
volume：成交手
amount：成交金额(元)
type：买卖类型【买盘、卖盘、中性盘】
'''
try:
df = ts.get_tick_data(code,date,retry_count, pause)
if df is None:
return
df = df.replace('--', 0)
df['change'] = df['change'].astype(float)
df['date'] = DateUtil.getTodayStr()
df['code'] = code
df = df.reset_index(level=[0])
del df['index']
self.storeservice.insert_many('ts2_tick_data', df)
except IOError as err:
logging.error("OS|error: {0}".format(err))
except OperationalError as err:
logging.error("OS|error: {0}".format(err))
else:
pass
def job_once_global_1m_analysis(*_args):
'''
线程工作：低频数据接口
:return:
'''
global is_closing
worker = _args[0][0]
arr = _args[0][1]
while not is_closing:
begin = time.time()
ret_arr = arr
total = len(ret_arr)
curr = 0
todayStr = DateUtil.getTodayStr()
for code, listing_date in ret_arr:
curr += 1
logging.info("current fetching progress {}/{} code:{} ".format(
curr, total, code))
if curr < 0:
continue
b1 = time.time()
ret_list = worker.storeservice.find_history_kline(
code, 'hk_1m', listing_date, todayStr)
if len(ret_list) <= 0:
logging.info("fetching {} K_LINE, but no data".format(code))
continue
col_list = [
'code', 'time_key', 'open', 'close', 'high', 'low', 'pe_ratio',
'turnover_rate', 'volume', 'turnover', 'change_rate',
'last_close'
]
data = pd.DataFrame(ret_list, columns=col_list)
data['position'] = np.where(data.change_rate > 0.0, 1, 0)
data.insert(loc=0,
column='period',
value=data.apply(
lambda x: get_minute_index(x['time_key'], 1),
axis='columns'))
xt = pd.crosstab(data.period, data.position)
xt_pct = xt.div(xt.sum(1).astype(float), axis=0)
add_list = []
col_list = [
'code', 'up_count', 'down_count', 'up_probability',
'down_probability', 'period', 'ptype'
]
count = 0
for item in xt:
count += 1
if count == 2:
for week_of_day in range(0, 481, 1):
try:
if xt[0][week_of_day] is None or xt[1] is None or xt[
1][week_of_day] is None:
continue
except KeyError as err:
logging.error("OS|error: {0}".format(err))
continue
up_count = xt[1][week_of_day]
down_count = xt[0][week_of_day]
up_probability = xt_pct[1][week_of_day]
down_probability = xt_pct[0][week_of_day]
add_list.append(
(code, up_count, down_count, up_probability,
down_probability, week_of_day, '1m'))
table = 'ft_stat_probability'
worker.storeservice.insert_many(
table, pd.DataFrame(add_list, columns=col_list))
e1 = time.time()
logging.info(
"fetching {} K_LINE listing_date:{} start: {} end:{} cost time {}"
.format(code, listing_date, listing_date, todayStr, e1 - b1))
if is_closing is True:
break
end = time.time()
logging.info("fetching for one period , cost time: {}".format(
(end - begin)))
break
def get_realtime_quotes(self,codes):
'''
功能：
获取实时分笔数据，可以实时取得股票当前报价和成交信息，其中一种场景是，写一个python定时程序来调用本接口（可两三秒执行一次，性能与行情软件基本一致），然后通过DataFrame的矩阵计算实现交易监控，可实时监测交易量和价格的变化。
参数说明：
symbols：6位数字股票代码，或者指数代码（sh=上证指数 sz=深圳成指 hs300=沪深300指数 sz50=上证50 zxb=中小板 cyb=创业板） 可输入的类型：str、list、set或者pandas的Series对象
返回值说明：
0：name，股票名字
1：open，今日开盘价
2：pre_close，昨日收盘价
3：price，当前价格
4：high，今日最高价
5：low，今日最低价
6：bid，竞买价，即“买一”报价
7：ask，竞卖价，即“卖一”报价
8：volume，成交量 maybe you need do volume/100
9：amount，成交金额（元 CNY）
10：b1_v，委买一（笔数 bid volume）
11：b1_p，委买一（价格 bid price）
12：b2_v，“买二”
13：b2_p，“买二”
14：b3_v，“买三”
15：b3_p，“买三”
16：b4_v，“买四”
17：b4_p，“买四”
18：b5_v，“买五”
19：b5_p，“买五”
20：a1_v，委卖一（笔数 ask volume）
21：a1_p，委卖一（价格 ask price）
...
30：date，日期；
31：time，时间；
'''
try:
df = ts.get_realtime_quotes(codes)
if df is None:
return
df = df.replace('--', 0)
df = df.replace('', 0)
df['b1_v'] = df['b1_v'].astype(int)
df['b2_v'] = df['b2_v'].astype(int)
df['b3_v'] = df['b3_v'].astype(int)
df['b4_v'] = df['b4_v'].astype(int)
df['b5_v'] = df['b5_v'].astype(int)
df['a1_v'] = df['a1_v'].astype(int)
df['a2_v'] = df['a2_v'].astype(int)
df['a3_v'] = df['a3_v'].astype(int)
df['a4_v'] = df['a4_v'].astype(int)
df['a5_v'] = df['a5_v'].astype(int)
df['date'] = DateUtil.getTodayStr()
df = df.reset_index(level=[0])
del df['index']
table = 'ts2_realtime_quotes'
self.storeservice.insert_many(table, df)
except IOError as err:
logging.error("OS|error: {0}".format(err))
except OperationalError as err:
logging.error("OS|error: {0}".format(err))
else:
pass
# content = content.partition('\n\n\n\n\n\n\n\n')
# if content.__len__() == 3:
# content = content[2].partition('\n\n\n\n\n\n\n\n')
# if content.__len__() ==3 and content[0].startswith('.ct_hqimg'):
# content = content[2]
# connection.update({"_id": s['_id']}, {"$set": {"content": content}})
isDateCount = 0
for s in connection.find({}):
url = str(s['href'])
if isinstance(s['date'], str):
date = None
strDate = s['date']
try:
strDate = strDate.replace(' </a>','')
date = DateUtil.string_toDatetime(strDate)
except:
print('2')
if date is not None:
connection.update_one({"_id": s['_id']}, {"$set": {"date": date}})
else:
try:
date = DateUtil.string_toDatetime2(strDate)
if date is not None:
connection.update_one({"_id": s['_id']}, {"$set": {"date": date}})
else:
connection.delete_one({"_id": s['_id']})
except:
def get_live_info(self):
ret_code = -1
ret_data = ''
self.itemArray = []
lasttime = DateUtil.string_toDatetime2('2019-05-01 09:00')
try:
res = requests.get(self.url)
if res.encoding == 'ISO-8859-1':
res.encoding = 'gbk'
html = res.text # .encode(res.encoding)
res.raise_for_status()
if res.status_code == 200 :
contentSoup = bs4.BeautifulSoup(html, 'lxml')
elems = contentSoup.find_all('a', class_='news-link')
for elem in elems:
json = {}
json['code'] = ' '
newstime = elem.select('span')
time = newstime[len(newstime) - 1].getText()
json['date'] = DateUtil.string_toDatetime2(time)
s = json['date']
if s < lasttime :
continue
else:
lasttime = s
h3 = elem.select('h3')
json['title'] = h3[len(h3) - 1].getText()
logger.info("date:{},title:{}".format(s, json['title']))
json['href'] = elem.attrs['href']
json['year'] = json['date'].year
json['sourcefrom'] = 'futunn'
ret,content = self.get_content(json['href'],'utf-8')
# if ret != -1 :
# time.sleep(4 * random.random())
if ret == 0 :
json['content'] = content
self.itemArray.append(json)
ret_code = 0
ret_data = ''
except Exception as err:
#time.sleep(4 * random.random())
logger.warning(err)
ret_code = -1
ret_data = err
except requests.exceptions.ConnectTimeout as err:
logger.warning(err)
ret_code = -1
ret_data = err
except requests.exceptions.ReadTimeout as err:
logger.warning(err)
ret_code = -1
ret_data = err
except requests.exceptions.Timeout as err:
logger.warning(err)
ret_code = -1
ret_data = err
except:
logger.warning('Unfortunitely -- An Unknow Error Happened, Please wait 3 seconds')
time.sleep(random.random())
ret_code = -1
ret_data = ''
finally:
res.close()
return ret_code,ret_data
def job_once_global_m5_append(*_args):
'''
线程工作：低频数据接口
:return:
'''
global is_closing
worker = _args[0][0]
arr = _args[0][1]
while not is_closing:
begin = time.time()
ret_arr = arr
todayStr = DateUtil.getTodayStr()
#last_fetchdate = DateUtil.string_toDate( DateUtil.getDatetimePastStr( DateUtil.string_toDate(todayStr),30) )
last_fetchdate = DateUtil.string_toDate('2018-08-02')
total = len(ret_arr)
curr = 0
for code, listing_date in ret_arr:
curr += 1
logging.info("current fetching progress {}/{} code:{} ".format(curr,total,code))
if curr < 1:
continue
# KLType.K_DAY
start = None
ld = worker.storeservice.find_lastdate2(code,'hk')
lastdate = last_fetchdate if ld == None else ld
if lastdate is not None and lastdate.date() > listing_date:
start = DateUtil.getDatetimeFutureStr( lastdate.date(),1 )
else:
# if listing_date.year == 1970:
# listing_date = listing_date.replace(year=1997)
# start = DateUtil.date_toString(listing_date)
start = DateUtil.date_toString(last_fetchdate)
end = todayStr
gen = DateUtil.getNextHalfYear(DateUtil.string_toDate(start), DateUtil.string_toDate(end))
b = time.time()
while True:
try:
end = next(gen)
if is_closing is True:
break
b2 = time.time()
_, _, lastest_date = worker.get_history_kline(code, start, end, ktype=KLType.K_DAY)
if lastest_date is not None:
worker.storeservice.update_lastdate(code, 'hk', DateUtil.string_toDatetime(lastest_date))
e2 = time.time()
logging.info(
"fetching {} K_DAY listing_date: {} start: {} end:{} cost time {}".format(code, listing_date, start, end, e2-b2))
start = DateUtil.getDatetimeFutureStr(DateUtil.string_toDate(end),1)
except StopIteration as e:
print(e)
break
# KLType.K_5M
start = None
ld = worker.storeservice.find_lastdate2(code,'hk_5m')
lastdate = last_fetchdate if ld == None else ld
if lastdate is not None and lastdate.date() > listing_date:
start = DateUtil.getDatetimeFutureStr(lastdate.date(), 1)
else:
# if listing_date.year == 1970:
# listing_date = listing_date.replace(year=1997)
# start = DateUtil.date_toString(listing_date)
start = DateUtil.date_toString(last_fetchdate)
end = todayStr
gen = DateUtil.getNextHalfYear(DateUtil.string_toDate(start), DateUtil.string_toDate(end))
b = time.time()
while True:
try:
end = next(gen)
if is_closing is True:
break
b1 = time.time()
_,_,lastest_date = worker.get_history_kline(code, start, end, ktype=KLType.K_5M)
if lastest_date is not None:
worker.storeservice.update_lastdate(code, 'hk_5m',lastest_date)
e1 = time.time()
logging.info(
"fetching {} K_5M_LINE listing_date:{} start: {} end:{} cost time {}".format(code,
listing_date,
start, end,
e1 - b1))
start = DateUtil.getDatetimeFutureStr(DateUtil.string_toDate(end), 1)
except StopIteration as e:
print(e)
break
e = time.time()
logging.info("position {} fetching {} const time {}".format(curr, code, e - b))
if is_closing is True:
break
end = time.time()
logging.info("fetching for one period , cost time: {}".format((end - begin)))
break
# -*- coding: UTF-8 -*-
def job_once_global_m5_append_multithread(*_args):
'''
线程工作：低频数据接口
:return:
'''
global is_closing
worker = _args[0][0]
arr = _args[0][1]
while not is_closing:
begin = time.time()
ret_arr = arr
todayStr = DateUtil.getTodayStr()
#last_fetchdate = DateUtil.string_toDate(DateUtil.getDatetimePastStr(DateUtil.string_toDate(todayStr), 30))
last_fetchdate = DateUtil.string_toDate('2018-08-02')
total = len(ret_arr)
curr = 0
for code, listing_date in ret_arr:
curr += 1
#1 - (1~2998包含)
#2 - (2999~15918不含）
#3 - （15918~18986不含）
#4 - （18986~19430不含）default InnoDB,
#5 - (19430~21898不含) MyISAM engine,ft_history_kline_5
#6 - (21898~24768不含) MyISAM engine,ft_history_kline_6
#7 - (24768~26347不含） MyISAM engine, ft_history_kline_7
#8 - (26347~27096不含) MyISAM engine, ft_history_kline_8， trigged by docker upgrade
#9 - (27096~28123不含) MyISAM engine, ft_history_kline_9
#10 - (28123~31918) MyISAM engine, ft_history_kline_10
# ft_history_kline tale as the mrg_myisam
logging.info("current fetching progress {}/{} ".format(curr,total))
if curr < 1:
continue
# KLType.K_DAY
start = None
ld = worker.storeservice.find_lastdate2(code, 'hk')
lastdate = last_fetchdate if ld == None else ld
if lastdate is not None and lastdate.date() > listing_date:
start = DateUtil.getDatetimeFutureStr( lastdate.date(),1 )
else:
# if listing_date.year == 1970:
# listing_date = listing_date.replace(year=1997)
# start = DateUtil.date_toString(listing_date)
start = DateUtil.date_toString(last_fetchdate)
end = todayStr
gen = DateUtil.getNextHalfYear(DateUtil.string_toDate(start), DateUtil.string_toDate(end))
b = time.time()
while True:
try:
end = next(gen)
if is_closing is True:
break
b2 = time.time()
_, _, lastest_date = worker.get_history_kline(code, start, end, ktype=KLType.K_DAY)
if lastest_date is not None:
worker.storeservice.update_lastdate(code, 'hk', DateUtil.string_toDatetime(lastest_date))
e2 = time.time()
logging.info(
"fetching {} K_DAY listing_date: {} start: {} end:{} cost time {}".format(code, listing_date, start, end, e2-b2))
start = DateUtil.getDatetimeFutureStr(DateUtil.string_toDate(end),1)
except StopIteration as e:
print(e)
break
# KLType.K_5M
start = None
ld = worker.storeservice.find_lastdate2(code, 'hk_5m')
lastdate = last_fetchdate if ld == None else ld
if lastdate is not None and lastdate.date() > listing_date:
start = DateUtil.getDatetimeFutureStr(lastdate.date(), 1)
else:
# if listing_date.year == 1970:
# listing_date = listing_date.replace(year=1997)
# start = DateUtil.date_toString(listing_date)
start = DateUtil.date_toString(last_fetchdate)
end = todayStr
gen = DateUtil.getNextHalfYear(DateUtil.string_toDate(start), DateUtil.string_toDate(end))
b = time.time()
while True:
try:
end = next(gen)
if is_closing is True:
break
b1 = time.time()
_, _, lastest_date = worker.get_history_kline(code, start, end, ktype=KLType.K_5M)
if lastest_date is not None:
worker.storeservice.update_lastdate(code, 'hk_5m', lastest_date)
e1 = time.time()
logging.info(
"fetching {} K_5M_LINE listing_date:{} start: {} end:{} cost time {}".format(code,
listing_date,
start, end,
e1 - b1))
start = DateUtil.getDatetimeFutureStr(DateUtil.string_toDate(end), 1)
except StopIteration as e:
print(e)
break
e = time.time()
logging.info("position {} fetching {} const time {}".format(curr, code, e - b))
if is_closing is True:
break
end = time.time()
logging.info("fetching for one period , cost time: {}".format((end - begin)))
break
def job_once_global_day_analysis(*_args):
'''
线程工作：低频数据接口
:return:
'''
global is_closing
worker = _args[0][0]
arr = _args[0][1]
while not is_closing:
begin = time.time()
ret_arr = arr
total = len(ret_arr)
curr = 0
todayStr = DateUtil.getTodayStr()
for code, listing_date in ret_arr:
curr += 1
logging.info("current fetching progress {}/{} code:{} ".format(curr, total, code))
if curr < 1:
continue
b1 = time.time()
ret_list = worker.storeservice.find_history_kline(code, 'hk', listing_date, todayStr)
if len(ret_list) <= 0:
logging.info(
"fetching {} K_LINE, but no data".format(code))
continue
col_list = ['code', 'time_key', 'open', 'close', 'high', 'low', 'pe_ratio', 'turnover_rate', 'volume',
'turnover', 'change_rate', 'last_close']
data = pd.DataFrame(ret_list, columns=col_list)
data['positive']=np.where( data.change_rate > 0.0, 1, 0)
data.insert(loc=0, column='date_week', value=data.apply(lambda x: DateUtil.week_of_date(x['time_key']), axis='columns'))
xt = pd.crosstab(data.date_week,data.positive)
xt_pct = xt.div(xt.sum(1).astype(float), axis=0)
add_list = []
col_list = ['code','up_count','down_count','up_probability','down_probability','week_of_day']
count = 0
for item in xt:
count += 1
if count == 2:
for week_of_day in range(0,5,1):
try:
if xt[0][week_of_day] is None or xt[1] is None or xt[1][week_of_day] is None:
continue
except KeyError as err:
logging.error("OS|error: {0}".format(err))
continue
up_count = xt[1][week_of_day]
down_count = xt[0][week_of_day]
up_probability = xt_pct[1][week_of_day]
down_probability = xt_pct[0][week_of_day]
add_list.append((code,up_count,down_count,up_probability,down_probability,week_of_day))
table = 'ft_stat_week_probability'
worker.storeservice.insert_many(table, pd.DataFrame(add_list, columns=col_list))
# Intel MKL FATAL ERROR: Error on loading function mkl_blas_avx2_get_kernel_api_version
# xt_pct.plot(figsize=(8,5),
# kind='bar',
# stacked=True,
# title='date_week->positive')
# rename can't work, TODO
#xt.rename(columns={'positive':'number'})
#print(xt)
#xt_pct = xt.div(xt.sum(1).astype(float), axis=0)
##xt_pct.rename(index=str,columns={'positive': 'weekprobability'})
#print(xt_pct)
#print(xt_pct+xt)
e1 = time.time()
logging.info(
"fetching {} K_LINE listing_date:{} start: {} end:{} cost time {}".format(code, listing_date, listing_date,
todayStr, e1 - b1))
if is_closing is True:
break
end = time.time()
logging.info("fetching for one period , cost time: {}".format((end - begin)))
break
def get_individual_news(self,market, code):
ret_code = -1
ret_data = ''
self.itemArray = []
url = "https://www.futunn.com/quote/stock-news?m={0}&code={1}".format(market.lower(),code.upper())
try:
header = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36',
'Accept': 'application/json,text/javascript,*.*;q=0.01',
'Origin': 'https://www.futunn.com',
'Referer': 'https://www.futunn.com/quote/stock-info?m={0}&code={1}&type=finance_analyse'.format(market.lower(),code.upper)
}
res = requests.get(url, headers=header)
if res.encoding == 'ISO-8859-1':
res.encoding = 'gbk'
html = res.text # .encode(res.encoding)
res.raise_for_status()
if res.status_code == 200:
contentSoup = bs4.BeautifulSoup(html, 'lxml')
elems = contentSoup.select('.ulList02 > ul > li')
for elem in elems:
json = {}
json['code'] = code
json['market'] = market
json['title'] = elem.select('.txt01')[0].getText()
json['href'] = elem.select('.txt01 > a')[0]['href']
json['date'] = DateUtil.string_toDatetime2(elem.select('.bar01')[0].getText().strip()[3:])
json['year'] = json['date'].year
json['sourcefrom'] = 'futunn'
ret, content = self.get_content(json['href'],'utf-8')
# if ret != -1:
# time.sleep(4 * random.random())
if ret == 0:
json['content'] = content
self.itemArray.append(json)
ret_code = 0
ret_data = ''
except Exception as err:
# time.sleep(4 * random.random())
logger.warning(err)
ret_code = -1
ret_data = err
except requests.exceptions.ConnectTimeout as err:
logger.warning(err)
ret_code = -1
ret_data = err
except requests.exceptions.ReadTimeout as err:
logger.warning(err)
ret_code = -1
ret_data = err
except requests.exceptions.Timeout as err:
logger.warning(err)
ret_code = -1
ret_data = err
except:
logger.warning('Unfortunitely -- An Unknow Error Happened, Please wait 3 seconds')
time.sleep(random.random())
ret_code = -1
ret_data = ''
finally:
res.close()
return ret_code, ret_data
def get_us_page(self, market, code, page, type):
"""
:param market:
:param code:
:param page:
:param type:
:return: (page_number, type), page_number:-1
"""
self.itemArray = []
url = self.generate_page_url(market, code, page)
url = url + type
logger.info('fetch url: {}'.format(url))
try:
res = requests.get(url, timeout=60, headers={'Content-type': 'text/html;charset=gb2312'})
if res.encoding == 'ISO-8859-1':
res.encoding = 'gbk'
html = res.text # .encode(res.encoding)
res.raise_for_status()
if res.status_code == 200:
contentSoup = bs4.BeautifulSoup(html, 'lxml')
elems = contentSoup.select('.xb_news > ul > li')
if page >= 100:
if type.__eq__("1"):
return 1, '2'
else:
return -1, '2'
for elem in elems:
json = {}
json['code'] = code
ele = elem.select('span')
if len(ele) == 0:
continue
json['date'] = DateUtil.format_date_us_history(ele[0].getText())
s = json['date']
ele = elem.select('a')
json['title'] = ele[len(ele) - 1].getText()
logger.info("date:{},title:{}".format(s, json['title']))
json['href'] = ele[len(ele) - 1].attrs['href']
json['year'] = 'real'
ret, content = self.get_content(json['href'], "utf-8")
# if ret != -1:
# time.sleep(4 * random.random())
if ret == 0:
json['content'] = content
self.itemArray.append(json)
except Exception as err:
#time.sleep(4 * random.random())
logger.warning(err)
except requests.exceptions.ConnectTimeout as err:
logger.warning(err)
ret_code = -1
ret_data = err
except requests.exceptions.ReadTimeout as err:
logger.warning(err)
ret_code = -1
ret_data = err
except requests.exceptions.Timeout as err:
logger.warning(err)
ret_code = -1
ret_data = err
except:
logger.warning('Unfortunitely -- An Unknow Error Happened, Please wait 3 seconds')
time.sleep(random.random())
ret_code = -1
ret_data = ''
finally:
res.close()
return page + 1, type
def get_futunn_news(self):
for i in range(94471,94480,1):
url = 'https://news.futunn.com/market/{0}?src=3'.format(i)
urlExist = self.mongodbutil.urlIsExist(url)
if urlExist:
logger.info('This url:{} has existed'.format(url))
continue
json = {}
header = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}
try:
res = requests.get(url, headers=header, timeout=60)
res.raise_for_status()
if res.status_code == 200:
soup = bs4.BeautifulSoup(res.text, 'lxml')
elems = soup.select('.inner')
json['content'] = elems[0].getText()
elems = soup.select('.news-title > h1')
json['title'] = elems[0].getText()
elems = soup.select('.news-title > .timeBar')
pos = elems[0].getText().strip().find('2')
json['date'] = elems[0].getText().strip()[pos:pos+16]
json['href'] = url
json['code'] = ' '
json['year'] = DateUtil.string_toDatetime2(json['date']).year
json['sourcefrom'] = 'futunn'
self.itemArray.append(json)
if len(self.get_item_array()) > 50:
self.mongodbutil.insertItems(self.get_item_array())
logger.info("store items to mongodb ...")
self.clear_item_array()
except Exception as err:
#time.sleep(4 * random.random())
logger.warning(err)
except requests.exceptions.ConnectTimeout as err:
logger.warning(err)
ret_code = -1
ret_data = err
except requests.exceptions.ReadTimeout as err:
logger.warning(err)
ret_code = -1
ret_data = err
except requests.exceptions.Timeout as err:
logger.warning(err)
ret_code = -1
ret_data = err
except:
logger.warning('Unfortunitely -- An Unknow Error Happened, Please wait 3 seconds')
time.sleep(random.random())
ret_code = -1
ret_data = ''
finally:
res.close()
return 1, 'ok'
def get_page(self,market, code,url):
ret_code = -1
ret_data = ''
self.itemArray = []
try:
res = requests.get(url, timeout=60, headers={
'Content-type': 'text/html;charset=gb2312'
})
if res.encoding == 'ISO-8859-1':
res.encoding = 'gbk'
html = res.text # .encode(res.encoding)
res.raise_for_status()
if res.status_code == 200 :
contentSoup = bs4.BeautifulSoup(html,'lxml')
elems = contentSoup.select('#js_ggzx > li,.li_point > ul > li,.col02_22 > ul > li')
for elem in elems:
json = {}
json['code'] = code
temp = elem.__str__()[4:5]
if (temp == '\n') and market == 'US':
continue
ele = elem.select('span')
json['date'] = DateUtil.format_date(ele[0].getText()[1:-1])
s = json['date']
ele = elem.select('a')
json['title'] = ele[len(ele)-1].getText()
logger.info("date:{},title:{}".format(s, json['title']))
json['href'] = ele[len(ele)-1].attrs['href']
json['year'] = 'guess'
ret,content = self.get_content(json['href'],'utf-8')
# if ret != -1 :
# time.sleep(4 * random.random())
if ret == 0 :
json['content'] = content
self.itemArray.append(json)
ret_code = 0
ret_data = ''
except Exception as err:
#time.sleep(4 * random.random())
logger.warning(err)
ret_code = -1
ret_data = err
except requests.exceptions.ConnectTimeout as err:
logger.warning(err)
ret_code = -1
ret_data = err
except requests.exceptions.ReadTimeout as err:
logger.warning(err)
ret_code = -1
ret_data = err
except requests.exceptions.Timeout as err:
logger.warning(err)
ret_code = -1
ret_data = err
except:
logger.warning('Unfortunitely -- An Unknow Error Happened, Please wait 3 seconds')
time.sleep(random.random())
ret_code = -1
ret_data = ''
finally:
res.close()
return ret_code,ret_data
def get_calendars(self):
urls = [
'https://news.futunn.com/new-calendar/events-list?begin_time={0}&end_time=2037-12-31&event_type=%5B%22%E6%B8%AF%E8%82%A1%E6%96%B0%E8%82%A1%22%2C%22%E7%BE%8E%E8%82%A1%E6%96%B0%E8%82%A1%22%2C%22A%E8%82%A1%E6%96%B0%E8%82%A1%22%5D&stock_type=&_={1}',
'https://news.futunn.com/new-calendar/events-list?begin_time={0}&end_time=2037-12-31&event_type=["港股财报"%2C"美股财报"%2C"A股财报"]&stock_type=&_={1}',
'https://news.futunn.com/new-calendar/events-list?begin_time={0}&end_time=2037-12-31&event_type=["港股除权除息"%2C"美股除权除息"%2C"A股除权除息"]&stock_type=&_={1}',
'https://news.futunn.com/new-calendar/events-list?begin_time={0}&end_time=2037-12-31&event_type=["财经事件"]&stock_type=&_={1}',
'https://news.futunn.com/new-calendar/events-list?begin_time={0}&end_time=2037-12-31&event_type=["经济数据"]&stock_type=&_={1}',
'https://news.futunn.com/new-calendar/events-list?begin_time={0}&end_time=2037-12-31&event_type=["休市提醒"]&stock_type=&_={1}'
]
for idx in range(0,len(urls),1):
print(idx)
url = urls[idx].format(DateUtil.getTodayStr(),int(1000*time.mktime(time.localtime())) + idx)
print(url)
logger.info("address current url {0}...".format(url))
arr = []
header = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}
try:
res = requests.get(url, headers=header, timeout=60)
res.raise_for_status()
if res.status_code == 200:
data = res.text
js = json.loads(data)
list = js['data']['list']
for elem in list:
# { unique key, drop duplicate
# "market_type": 1,
# "event_type": 1,
# "event_time": 1
# }
# elem['event_type']
# elem['market_type']
# elem['event_time']
#itemTime = DateUtil.string_toDatetime(elem['time'])
# 'event_type': '港股新股',
# 'market_type': 'HK',
# 'event_text': '认购中<br/><a href="http://www.futunn.com/quote/stock?m=hk&code=01832" target="_blank" data-market="hk" data-code="01832" class="js-nn-stock">海天地悦旅(01832)</a><br/><a href="http://www.futunn.com/quote/stock?m=hk&code=02230" target="_blank" data-market="hk" data-code="02230" class="js-nn-stock">羚邦集团(02230)</a><br/>',
# 'event_time': '2019-05-05 00:00:00',
# 'total': 2}
# if itemTime > lasttime:
# arr.append( elem )
# logger.info(elem)
# else:
# continue
arr.append(elem)
if len(arr) > 0 :
self.mongodbutil_calendar.insertItems(arr)
logger.info("store items to mongodb ...")
else:
logger.info("still have no calendar live message")
except Exception as err:
#time.sleep(4 * random.random())
logger.warning(err)
except requests.exceptions.ConnectTimeout as err:
logger.warning(err)
ret_code = -1
ret_data = err
except requests.exceptions.ReadTimeout as err:
logger.warning(err)
ret_code = -1
ret_data = err
except requests.exceptions.Timeout as err:
logger.warning(err)
ret_code = -1
ret_data = err
except:
logger.warning('Unfortunitely -- An Unknow Error Happened, Please wait 3 seconds')
time.sleep(random.random())
ret_code = -1
ret_data = ''
finally:
res.close()
return 1, 'ok'
def append_one_stock(service, code, dtype, listing_date):
'''
:param worker:
:param code:
:param dtype:
:param last_fetchdate:
:param listing_date:
:return:
'''
global is_closing
todayStr = DateUtil.getTodayStr()
last_fetchdate = DateUtil.string_toDate('2018-08-02')
start = None
ld, tindex = service.storeservice.find_lastdate_and_tindex(code, dtype)
lastdate = last_fetchdate if ld == None else ld
if lastdate is not None and lastdate.date() > listing_date:
start = DateUtil.getDatetimeFutureStr(lastdate.date(), 1)
else:
start = DateUtil.date_toString(last_fetchdate)
end = todayStr
gen = DateUtil.getNextHalfYear(DateUtil.string_toDate(start),
DateUtil.string_toDate(end),
ndays=360)
while True:
try:
end = next(gen)
if is_closing is True:
break
b2 = time.time()
days = DateUtil.diff(start, end)
lastest_date = service.get_hisft_klinetory_data(
code[3:], start, days)
if lastest_date is not None:
service.storeservice.update_lastdate(
code, dtype,
DateUtil.string_toDatetime(
DateUtil.datetime_toString(lastest_date)))
logging.info('********I')
e2 = time.time()
logging.info(
"fetching {} dtype {} listing_date: {} start: {} end:{} cost time {}"
.format(code, dtype, listing_date, start, end, e2 - b2))
start = DateUtil.getDatetimeFutureStr(DateUtil.string_toDate(end),
1)
except StopIteration as e:
print(e)
break