def __init__(self):
self.es = ES(host=ES_HOST, port=ES_PORT)
self.mongo = MongoDB(mongo_host=MONGO_HOST,
mongo_port=MONGO_PORT,
db_name=MONGO_DB,
collection=MONGO_COL)
self.mysql = Mysql(db=MYSQL_DB_NAME)
self.simhash = SimHash(stopwords_path=STOPWORDS_PATH)
self.nlp = Predict(sentiment_url=SENTIMENT_URL, type_url=TYPE_URL)
class MysqlThread(WorkerThread):
def __init__(self, *args, **kwargs):
self._mysql = Mysql(kwargs['db_config'])
super(MysqlThread, self).__init__(*args, **kwargs)
self.daemon = True
def get_handler(self):
return self._mysql
def close(self):
self._mysql.close()
class MainClass(object):
def __init__(self):
self.es = ES(host=ES_HOST, port=ES_PORT)
self.mongo = MongoDB(mongo_host=MONGO_HOST,
mongo_port=MONGO_PORT,
db_name=MONGO_DB,
collection=MONGO_COL)
self.mysql = Mysql(db=MYSQL_DB_NAME)
self.simhash = SimHash(stopwords_path=STOPWORDS_PATH)
self.nlp = Predict(sentiment_url=SENTIMENT_URL, type_url=TYPE_URL)
@staticmethod
def get_news(filename):
with open(filename, "r", encoding="utf-8") as f:
news = json.loads(f.read())
return news
def check_repeat(self, news):
news_save = list()
for idx, value in enumerate(news):
try:
simhash_code = self.simhash.build_simhash(value["title"] +
value["content"])
repeat_code = self.mysql.hanming(simhash_code)
except Exception as e:
print(e.args)
print(simhash_code, value)
repeat_code = 1
if repeat_code == 0:
try:
self.mysql.insert_himhash(simhash_code, value)
news_save.append(value)
except Exception as e:
print(e.args)
else:
continue
self.mysql.insert2mysql(news_save)
return news_save
def predict(self, news):
return self.nlp.infer(news)
def save2db(self, news, index, doc_type):
final_news = self.predict(news)
#TODO 验证是否存入成功
self.es.insert2es(data=final_news, index=index, doc_type=doc_type)
self.mongo.insert2mongo(final_news.copy())
print("finish")
def get_mysql_client(self):
ct = threading.current_thread().ident
local = self._thread_local_map.get(ct, None)
if not local:
local = threading.local()
self._thread_local_map[ct] = local
local.mysql = Mysql(self._database_config)
self._mysql_client_count = self._mysql_client_count + 1
return local.mysql
async def main() -> None:
options.log_file_prefix = log_path
options.logging = log_level
tornado.options.parse_command_line()
db_pool = {
'mysql': Mysql(await aiomysql.create_pool(**mysql_options)),
'redis': await aioredis.create_redis_pool(**redis_options)
}
app = RestfulApp(db_pool=db_pool, **settings)
app.listen(options.port)
def before_request():
g.headers = {}
g.pagination = Pager(request.args)
g.request = Request(request)
g.auth = None
g.perms = {}
g.im_rds = rds
cnf = MYSQL
g._db = Mysql(*cnf)
g._imdb = g._db
def __init__(self, *args, **kwargs):
self._mysql = Mysql(kwargs['db_config'])
super(MysqlThread, self).__init__(*args, **kwargs)
self.daemon = True
def __init__(self, *args, **kwargs):
print('**MysqlThread:{}**'.format(kwargs))
self._mysql = Mysql(**kwargs['db_config']) # 初始化数据库
super(MsqlThread, self).__init__(*args, **kwargs) # 也初始化父类