def job_sunday(ts):
"""
线程工作：可周末定时间执行
:return:
"""
# replace for exists table
ts.get_ppi()
ts.get_cpi()
ts.get_gdp_contrib()
ts.get_gdp_pull()
ts.get_gdp_for()
ts.get_gdp_quarter()
ts.get_gdp_year()
ts.get_money_supply_bal()
ts.get_money_supply()
ts.get_rrr()
ts.get_loan_rate()
ts.get_deposit_rate()
ts.get_zz500s()
ts.get_sz50s()
ts.get_hs300s()
ts.get_st_classified()
ts.get_gem_classified()
ts.get_sme_classified()
ts.get_area_classified()
ts.get_concept_classified()
ts.get_industry_classified()
ts.new_stocks()
ts.xsg_data()
# append for exists table ,and memorize the doneYear and doneQuarter
doneYear = AppConfig.pull_year
doneQuarter = AppConfig.pull_quarter
for year in range(doneYear, 2019):
for quarter in range(doneQuarter, 5):
ts.get_cashflow_data(year, quarter)
time.sleep(1)
ts.get_debtpaying_data(year, quarter)
time.sleep(1)
ts.get_growth_data(year, quarter)
time.sleep(1)
ts.get_operation_data(year, quarter)
time.sleep(1)
ts.get_profit_data(year, quarter)
time.sleep(1)
ts.get_report_data(year, quarter)
time.sleep(1)
ts.fund_holdings(year, quarter)
time.sleep(1)
ts.forecast_data(year, quarter)
time.sleep(1)
ts.profit_data(year, 100)
time.sleep(1)
AppConfig.write_pulltime(year, quarter)
def get_latest_news(self,top=5, show_content=True):
'''
功能：
即时新闻
参数说明：
top:int，显示最新消息的条数，默认为80条
show_content:boolean,是否显示新闻内容，默认False
结果返回的数据属性说明如下：
classify :新闻类别
title :新闻标题
time :发布时间
url :新闻链接
content:新闻内容（在show_content为True的情况下出现）
'''
df = None
try:
if time.time() - AppConfig.latest_news_pulltime > 1000:
top = 80
df = ts.get_latest_news(top, show_content)
if df is None:
logging.info('df is None')
return
table = 'ts2_latest_news'
latest_pulltime = None
pulltime = None
dropindex = -1
df.sort_values(by="time", ascending=False)
for i in range(0,len(df)):
pulltime = df.iloc[i]['time']
pulltime = DateUtil.string_toTimestamp(DateUtil.format_date(pulltime))
if i == 0:
latest_pulltime = pulltime
if pulltime <= AppConfig.latest_news_pulltime:
#remove
dropindex = i
print('dropindex:',dropindex)
break
if dropindex != -1:
df = df.drop(range(dropindex,len(df),1))
if len(df) > 0 :
self.storeservice.insert_many(table, df)
AppConfig.write_news_pulltime(latest_pulltime,True)
except IOError as err:
logging.error("OS|error: {0}".format(err))
except OperationalError as err:
logging.error("OS|error: {0}".format(err))
else:
pass
def setUpClass(cls):
config = AppConfig.get_config()
cls.hk_ctx = ft.OpenHKTradeContext(config.get('ftserver', 'host'),
int(config.get('ftserver', 'port')))
cls.us_ctx = ft.OpenUSTradeContext(config.get('ftserver', 'host'),
int(config.get('ftserver', 'port')))
cls.hk_trade = Trade(cls.hk_ctx)
cls.us_trade = Trade(cls.us_ctx)
cls.hk_trade.unlock_trade(None, config.get('ftserver', 'decipher'))
cls.us_trade.unlock_trade(None, config.get('ftserver', 'decipher'))
def __init__(self):
'''
Store Engine Init
'''
self.config = AppConfig.get_config()
self.connect_url = self.config.get('mysql', 'connecturl')
self.mysql_engine = create_engine(self.connect_url,
poolclass=QueuePool,
pool_pre_ping=True,
pool_recycle=3600,
pool_size=20)
self.mysqlStore = MysqlStore(self.mysql_engine)
def setUpClass(cls):
config = AppConfig.get_config()
total = config.get('quota', 'total')
kline = config.get('quota', 'kline')
tiker = config.get('quota', 'ticker')
quote = config.get('quota', 'quote')
order_book = config.get('quota', 'order_book')
rt_data = config.get('quota', 'rt_data')
broker = config.get('quota', 'broker')
cls.ctx = ft.OpenQuoteContext(config.get('ftserver', 'host'),
int(config.get('ftserver', 'port')))
cls.ctx.start()
cls.qh = QueryHistory(cls.ctx)
def gen_one_worker():
global ctx
config = AppConfig.get_config()
total = config.get('quota', 'total')
kline = config.get('quota', 'kline')
tiker = config.get('quota', 'ticker')
quote = config.get('quota', 'quote')
order_book = config.get('quota', 'order_book')
rt_data = config.get('quota', 'rt_data')
broker = config.get('quota', 'broker')
ctx = ft.OpenQuoteContext(config.get('ftserver', 'host'), int(config.get('ftserver', 'port')))
ctx.start()
lf = LF(ctx)
return lf
def main():
global ctx
config = AppConfig.get_config()
total = config.get('quota', 'total')
kline = config.get('quota', 'kline')
tiker = config.get('quota', 'ticker')
quote = config.get('quota', 'quote')
order_book = config.get('quota', 'order_book')
rt_data = config.get('quota', 'rt_data')
broker = config.get('quota', 'broker')
ctx = ft.OpenQuoteContext(config.get('ftserver', 'host'), int(config.get('ftserver', 'port')))
ctx.start()
lf = LF(ctx)
# ret_arr = lf.storeservice.find_all_stocks()
# total_threads = 10
# step_length = int(len(ret_arr)/total_threads)
# mode = len(ret_arr)%total_threads
# for i in range(0,total_threads,1):
# lf = LF(ctx)
# thread_name = 'job_lf_{}'.format(i)
#
# start = i*step_length
# if i == (total_threads-1):
# end = start+step_length+mode
# else:
# end = start + step_length
# arr = ret_arr[start:end]
#
# once_global_m5_task(thread_name,arr,lf)
kline_total_tables = 11
kline_K_5M_total_tables = 17
for tindex in range(1,kline_total_tables+1,1):
ret_arr = lf.storeservice.find_stocks('hk',tindex)
lf = LF(ctx)
thread_name = 'job_lf_{}'.format(tindex)
once_global_m5_task(thread_name, ret_arr, lf)
for tindex in range(1,kline_K_5M_total_tables+1,1):
ret_arr = lf.storeservice.find_stocks('hk_5m',tindex)
print(len(ret_arr))
lf = LF(ctx)
thread_name = 'job_lf_5m_{}'.format(tindex)
time.sleep(5)
once_global_m5_task(thread_name, ret_arr, lf)
def main():
global ctx
config = AppConfig.get_config()
total = config.get('quota', 'total')
kline = config.get('quota', 'kline')
tiker = config.get('quota', 'ticker')
quote = config.get('quota', 'quote')
order_book = config.get('quota', 'order_book')
rt_data = config.get('quota', 'rt_data')
broker = config.get('quota', 'broker')
ctx = ft.OpenQuoteContext(config.get('ftserver', 'host'),
int(config.get('ftserver', 'port')))
ctx.start()
lf = LF(ctx)
once_custom_task(lf)
def main():
global engine
engine = QEEngine()
app = make_app()
setup_logging()
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_term_handler)
server = tornado.httpserver.HTTPServer(app)
server.bind(AppConfig.get_config().get('web', 'port'))
server.start(0) # forks one process per cpu
tornado.ioloop.PeriodicCallback(try_exit, 100).start()
tornado.ioloop.IOLoop.current().start()
engine.stop()
def main():
global ctx
config = AppConfig.get_config()
total = config.get('quota', 'total')
kline = config.get('quota', 'kline')
tiker = config.get('quota', 'ticker')
quote = config.get('quota', 'quote')
order_book = config.get('quota', 'order_book')
rt_data = config.get('quota', 'rt_data')
broker = config.get('quota', 'broker')
ctx = ft.OpenQuoteContext(config.get('ftserver', 'host'), int(config.get('ftserver', 'port')))
ctx.start()
sub = Subscribe(ctx, total, kline, tiker, quote, order_book, rt_data, broker)
hf = HF(ctx, sub)
realtime_quote_task(hf)
def setUpClass(cls):
config = AppConfig.get_config()
total = config.get('quota', 'total')
kline = config.get('quota', 'kline')
tiker = config.get('quota', 'ticker')
quote = config.get('quota', 'quote')
order_book = config.get('quota', 'order_book')
rt_data = config.get('quota', 'rt_data')
broker = config.get('quota', 'broker')
cls.ctx = ft.OpenQuoteContext(config.get('ftserver', 'host'),
int(config.get('ftserver', 'port')))
cls.sub = Subscribe(cls.ctx, total, kline, tiker, quote, order_book,
rt_data, broker)
cls.ctx.start()
cls.lf = LF(cls.ctx)
cls.hf = HF(cls.ctx, cls.sub)
def main():
global hk_ctx
global us_ctx
config = AppConfig.get_config()
total = config.get('quota', 'total')
kline = config.get('quota', 'kline')
tiker = config.get('quota', 'ticker')
quote = config.get('quota', 'quote')
order_book = config.get('quota', 'order_book')
rt_data = config.get('quota', 'rt_data')
broker = config.get('quota', 'broker')
hk_ctx = ft.OpenHKTradeContext(config.get('ftserver', 'host'), int(config.get('ftserver', 'port')))
hk_trade = Trade(hk_ctx)
hk_trade.unlock_trade(None,config.get('ftserver', 'decipher'))
hk_trade_task(hk_trade)
us_ctx = ft.OpenUSTradeContext(config.get('ftserver', 'host'), int(config.get('ftserver', 'port')))
us_trade = Trade(us_ctx)
us_trade.unlock_trade(None, config.get('ftserver', 'decipher'))
us_trade_task(us_trade)
def main():
global lf_ctx
global hf_ctx
global hk_ctx
global us_ctx
config = AppConfig.get_config()
total = config.get('quota', 'total')
kline = config.get('quota', 'kline')
tiker = config.get('quota', 'ticker')
quote = config.get('quota', 'quote')
order_book = config.get('quota', 'order_book')
rt_data = config.get('quota', 'rt_data')
broker = config.get('quota', 'broker')
lf_ctx = ft.OpenQuoteContext(config.get('ftserver', 'host'),
int(config.get('ftserver', 'port')))
lf_ctx.start()
lf = LF(lf_ctx)
#lf_task(lf)
hf_ctx = ft.OpenQuoteContext(config.get('ftserver', 'host'),
int(config.get('ftserver', 'port')))
hf_ctx.start()
sub = Subscribe(hf_ctx, total, kline, tiker, quote, order_book, rt_data,
broker)
hf = HF(hf_ctx, sub)
hf_task(hf)
hk_ctx = ft.OpenHKTradeContext(config.get('ftserver', 'host'),
int(config.get('ftserver', 'port')))
hk_trade = Trade(hk_ctx)
hk_trade.unlock_trade(None, config.get('ftserver', 'decipher'))
#hk_trade_task(hk_trade)
us_ctx = ft.OpenUSTradeContext(config.get('ftserver', 'host'),
int(config.get('ftserver', 'port')))
us_trade = Trade(us_ctx)
us_trade.unlock_trade(None, config.get('ftserver', 'decipher'))
def __init__(self):
'''
QE Engine Init
'''
config = AppConfig.get_config()
self.config = config
quote_ctx = ft.OpenQuoteContext(config.get('ftserver', 'host'),
int(config.get('ftserver', 'port')))
self.quote_ctx = quote_ctx
self.tradehk_ctx = ft.OpenHKTradeContext(
config.get('ftserver', 'host'), int(config.get('ftserver',
'port')))
self.tradeus_ctx = ft.OpenUSTradeContext(
config.get('ftserver', 'host'), int(config.get('ftserver',
'port')))
self.decipher = config.get('ftserver', 'decipher')
total = config.get('quota', 'total')
kline = config.get('quota', 'kline')
tiker = config.get('quota', 'ticker')
quote = config.get('quota', 'quote')
order_book = config.get('quota', 'order_book')
rt_data = config.get('quota', 'rt_data')
broker = config.get('quota', 'broker')
quote_ctx.start()
self.lf = LF(quote_ctx)
self.sub = Subscribe(quote_ctx, total, kline, tiker, quote, order_book,
rt_data, broker)
self.hf = HF(quote_ctx, self.sub)
self.hktrade = Trade(self.tradehk_ctx)
self.hktrade.unlock_trade(self.decipher)
self.ustrade = Trade(self.tradeus_ctx)
self.ustrade.unlock_trade(self.decipher)
self.queryhistory = QueryHistory(quote_ctx)
self.mysqlService = MysqlService()
import pymongo
import re
import datetime
from hsstock.utils.app_config import AppConfig
from hsstock.utils.mongodbutil import MongodbUtil
import hsstock.utils.logger as logger
if __name__ == '__main__':
AppConfig.get_config()
mongodbutil = MongodbUtil(AppConfig.mongodb_ip, AppConfig.mongodb_port,
AppConfig.mongodb_collection)
# PreResUrl = ''.join(
# (os.path.abspath(''.join((__file__, '../../../'))), AppConfig.get_config().get('tf_idf', 'pre.res.filename')))
# config = AppConfig.get_config()
# interval = config.getfloat('portal', 'timer.interval')
# timer = threading.Timer(interval, make_keyword, args=[interval, PreResUrl])
# timer.start()
# make_tag()
connection = mongodbutil.collection
logger.info('Starting time: {}'.format(datetime.datetime.now()))
for s in connection.find({}):
content = str(s['content'])
if content.startswith('\n\n\n\n\n\n\n\n'):
content = content.partition('\n\n\n\n\n\n\n\n')
if content.__len__() == 3:
content = content[2].partition('\n\n\n\n\n\n\n\n')
if content.__len__() == 3 and content[0].startswith(
def signal_int_handler(signum, frame):
global is_closing
logging.info('exiting...')
AppConfig.write_news_pulltime(AppConfig.latest_news_pulltime, True)
is_closing = True
sched.shutdown(True)
# -*- coding: UTF-8 -*-
def setUpClass(cls):
config = AppConfig.get_config()
cls.sender = config.get('sender', 'sender')
cls.password = config.get('sender', 'password')
cls.smtpserver = config.get('sender', 'smtpserver')