class MysqlHandler(logging.Handler): def __init__(self, level=logging.NOTSET): """Setting up msyql handler, initializing mysql database connection via .""" base_dispatcher_config_path = os.path.split( os.path.realpath(__file__))[0] + "/config.conf" self.__config = Conf([base_dispatcher_config_path]) logging.Handler.__init__(self, level) pass def emit(self, record): """Inserting new logging record to msyql database.""" try: msg = re.sub('\'', '\"', record.getMessage()) msg = json.loads(msg) print(msg) except Exception as e: msg = None if isinstance(msg, dict) and msg.get("machine_name") is not None: try: self._mysqlCn = MysqlCn(self.__config) sql = "INSERT INTO bulletin_log(machine,`key`,state) " \ "VALUES(%s,%s,%s)" machine = msg.get("machine_name") key = msg.get("key") state = msg.get("state") self._mysqlCn.insertOne(sql, (machine, key, state)) self._mysqlCn.dispose() except Exception as e: print(e)
def get_not_crawler_from_keyword(self): mysqlCn = MysqlCn(self.config) sql = "SELECT count(id) as `count` FROM keyword_info WHERE `status`>1 AND priority = 1 AND province_id IS NOT NULL;" result = mysqlCn.getAll(sql) mysqlCn.dispose() if result: return result[0].get('count') else: return False
def get_sectino_num(self): mysqlCn = MysqlCn(self.config) sql = "SELECT section_num,province_id,max_num,current_num FROM regno_statistic WHERE status = 0 AND section_num IS NOT NULL LIMIT 1;" result = mysqlCn.getAll(sql) mysqlCn.dispose() if len(result) > 0: return result[0] else: return False
def is_rengo_exit(self, regno): mysqlCn = MysqlCn(self.config) tmd5 = self.md5(regno) sql = "SELECT COUNT(`id`) AS counts FROM business_info WHERE md5 = %s;" result = mysqlCn.getAll(sql, (tmd5, )) mysqlCn.dispose() if result[0].get('counts') == 0: return False else: return True
def inset_keyword(self, keywods): mysqlCn = MysqlCn(self.config) sql = "INSERT INTO keyword_info(keyword,province_id,priority) VALUES(%s,%s,%s);" for i in range(len(keywods)): keyword = keywods[i].get('keyword') province_id = keywods[i].get('province_id') priority = 0 result = mysqlCn.insertOne(sql, (keyword, province_id, priority)) mysqlCn.dispose() return None
def emit(self, record): """Inserting new logging record to msyql database.""" try: msg = re.sub('\'', '\"', record.getMessage()) msg = json.loads(msg) print(msg) except Exception as e: msg = None if isinstance(msg, dict) and msg.get("machine_name") is not None: try: self._mysqlCn = MysqlCn(self.__config) sql = "INSERT INTO bulletin_log(machine,`key`,state) " \ "VALUES(%s,%s,%s)" machine = msg.get("machine_name") key = msg.get("key") state = msg.get("state") self._mysqlCn.insertOne(sql, (machine, key, state)) self._mysqlCn.dispose() except Exception as e: print(e)
def insert(self,msgs): mysqlCn = MysqlCn(self.__config) sql = 'INSERT ignore INTO ws_content_primary(id,date,caseName,caseNo,dataSources,trialProcedure,reason,docID,releaseDate,time,appellor,caseReason,keyWord,caseType,legalBase) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);' sql1 = 'INSERT ignore INTO ws_content_attachment(id,content) VALUES(%s,%s);' for ttemp in msgs: row_id = mysqlCn.insertOne(sql, (ttemp.get('id'),ttemp.get('date'),ttemp.get('caseName'),ttemp.get('caseNo'),ttemp.get('dataSources'),ttemp.get('trialProcedure'),ttemp.get('reason'),ttemp.get('doclD'),ttemp.get('releaseDate'),ttemp.get('time'),ttemp.get('appellor'),ttemp.get('caseReason'),ttemp.get('keyWord'),ttemp.get('caseType'),ttemp.get('legalBase'))) content_row_id = mysqlCn.insertOne(sql, (ttemp['id'], ttemp['content'])) mysqlCn.dispose()
def insert(self,msgs): try: mysqlCn = MysqlCn(self.__config) sql = 'INSERT IGNORE INTO bul_content_primary(id,type,person,party,time,pdf,url,create_time) VALUES(%s,%s,%s,%s,%s,%s,%s,%s)' sql1 = 'INSERT IGNORE INTO bul_content_attachment(id,content) VALUES(%s,%s)' for ttemp in msgs: tid = int(ttemp.get('id')) ttype = str(ttemp.get('type')) person = str(ttemp.get('person')) party = str(ttemp.get('party')) ttime = str(ttemp.get('time')) pdf = str(ttemp.get('pdf')) turl = str(ttemp.get('url')) content = str(ttemp.get('content')) create_time = time.strftime("%Y-%m-%d %H:%M:%S") row_id = mysqlCn.insertOne(sql, (tid,ttype,person,party,ttime,pdf,turl,create_time)) content_row_id = mysqlCn.insertOne(sql1, (tid,content)) mysqlCn.dispose() except Exception as e: print(e,"入库失败")
def update_sectino_num(self, section_num, current_num, status=0): mysqlCn = MysqlCn(self.config) sql = "UPDATE regno_statistic SET `status` = %s,`current_num`= %s WHERE section_num = %s;" result = mysqlCn.update(sql, (status, current_num, section_num)) mysqlCn.dispose() return result