def __init__(self): self.es = ES(host=ES_HOST, port=ES_PORT) self.mongo = MongoDB(mongo_host=MONGO_HOST, mongo_port=MONGO_PORT, db_name=MONGO_DB, collection=MONGO_COL) self.mysql = Mysql(db=MYSQL_DB_NAME) self.simhash = SimHash(stopwords_path=STOPWORDS_PATH) self.nlp = Predict(sentiment_url=SENTIMENT_URL, type_url=TYPE_URL)
class MysqlThread(WorkerThread): def __init__(self, *args, **kwargs): self._mysql = Mysql(kwargs['db_config']) super(MysqlThread, self).__init__(*args, **kwargs) self.daemon = True def get_handler(self): return self._mysql def close(self): self._mysql.close()
class MainClass(object): def __init__(self): self.es = ES(host=ES_HOST, port=ES_PORT) self.mongo = MongoDB(mongo_host=MONGO_HOST, mongo_port=MONGO_PORT, db_name=MONGO_DB, collection=MONGO_COL) self.mysql = Mysql(db=MYSQL_DB_NAME) self.simhash = SimHash(stopwords_path=STOPWORDS_PATH) self.nlp = Predict(sentiment_url=SENTIMENT_URL, type_url=TYPE_URL) @staticmethod def get_news(filename): with open(filename, "r", encoding="utf-8") as f: news = json.loads(f.read()) return news def check_repeat(self, news): news_save = list() for idx, value in enumerate(news): try: simhash_code = self.simhash.build_simhash(value["title"] + value["content"]) repeat_code = self.mysql.hanming(simhash_code) except Exception as e: print(e.args) print(simhash_code, value) repeat_code = 1 if repeat_code == 0: try: self.mysql.insert_himhash(simhash_code, value) news_save.append(value) except Exception as e: print(e.args) else: continue self.mysql.insert2mysql(news_save) return news_save def predict(self, news): return self.nlp.infer(news) def save2db(self, news, index, doc_type): final_news = self.predict(news) #TODO 验证是否存入成功 self.es.insert2es(data=final_news, index=index, doc_type=doc_type) self.mongo.insert2mongo(final_news.copy()) print("finish")
def get_mysql_client(self): ct = threading.current_thread().ident local = self._thread_local_map.get(ct, None) if not local: local = threading.local() self._thread_local_map[ct] = local local.mysql = Mysql(self._database_config) self._mysql_client_count = self._mysql_client_count + 1 return local.mysql
async def main() -> None: options.log_file_prefix = log_path options.logging = log_level tornado.options.parse_command_line() db_pool = { 'mysql': Mysql(await aiomysql.create_pool(**mysql_options)), 'redis': await aioredis.create_redis_pool(**redis_options) } app = RestfulApp(db_pool=db_pool, **settings) app.listen(options.port)
def before_request(): g.headers = {} g.pagination = Pager(request.args) g.request = Request(request) g.auth = None g.perms = {} g.im_rds = rds cnf = MYSQL g._db = Mysql(*cnf) g._imdb = g._db
def __init__(self, *args, **kwargs): self._mysql = Mysql(kwargs['db_config']) super(MysqlThread, self).__init__(*args, **kwargs) self.daemon = True
def __init__(self, *args, **kwargs): print('**MysqlThread:{}**'.format(kwargs)) self._mysql = Mysql(**kwargs['db_config']) # 初始化数据库 super(MsqlThread, self).__init__(*args, **kwargs) # 也初始化父类