class MysqlHandler(logging.Handler):
def __init__(self, level=logging.NOTSET):
"""Setting up msyql handler, initializing mysql database connection via ."""
base_dispatcher_config_path = os.path.split(
os.path.realpath(__file__))[0] + "/config.conf"
self.__config = Conf([base_dispatcher_config_path])
logging.Handler.__init__(self, level)
pass
def emit(self, record):
"""Inserting new logging record to msyql database."""
try:
msg = re.sub('\'', '\"', record.getMessage())
msg = json.loads(msg)
print(msg)
except Exception as e:
msg = None
if isinstance(msg, dict) and msg.get("machine_name") is not None:
try:
self._mysqlCn = MysqlCn(self.__config)
sql = "INSERT INTO bulletin_log(machine,`key`,state) " \
"VALUES(%s,%s,%s)"
machine = msg.get("machine_name")
key = msg.get("key")
state = msg.get("state")
self._mysqlCn.insertOne(sql, (machine, key, state))
self._mysqlCn.dispose()
except Exception as e:
print(e)
def insert(self,msgs):
mysqlCn = MysqlCn(self.__config)
sql = 'INSERT ignore INTO ws_content_primary(id,date,caseName,caseNo,dataSources,trialProcedure,reason,docID,releaseDate,time,appellor,caseReason,keyWord,caseType,legalBase) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);'
sql1 = 'INSERT ignore INTO ws_content_attachment(id,content) VALUES(%s,%s);'
for ttemp in msgs:
row_id = mysqlCn.insertOne(sql, (ttemp.get('id'),ttemp.get('date'),ttemp.get('caseName'),ttemp.get('caseNo'),ttemp.get('dataSources'),ttemp.get('trialProcedure'),ttemp.get('reason'),ttemp.get('doclD'),ttemp.get('releaseDate'),ttemp.get('time'),ttemp.get('appellor'),ttemp.get('caseReason'),ttemp.get('keyWord'),ttemp.get('caseType'),ttemp.get('legalBase')))
content_row_id = mysqlCn.insertOne(sql, (ttemp['id'], ttemp['content']))
mysqlCn.dispose()
def get_not_crawler_from_keyword(self):
mysqlCn = MysqlCn(self.config)
sql = "SELECT count(id) as `count` FROM keyword_info WHERE `status`>1 AND priority = 1 AND province_id IS NOT NULL;"
result = mysqlCn.getAll(sql)
mysqlCn.dispose()
if result:
return result[0].get('count')
else:
return False
def get_sectino_num(self):
mysqlCn = MysqlCn(self.config)
sql = "SELECT section_num,province_id,max_num,current_num FROM regno_statistic WHERE status = 0 AND section_num IS NOT NULL LIMIT 1;"
result = mysqlCn.getAll(sql)
mysqlCn.dispose()
if len(result) > 0:
return result[0]
else:
return False
def is_rengo_exit(self, regno):
mysqlCn = MysqlCn(self.config)
tmd5 = self.md5(regno)
sql = "SELECT COUNT(`id`) AS counts FROM business_info WHERE md5 = %s;"
result = mysqlCn.getAll(sql, (tmd5, ))
mysqlCn.dispose()
if result[0].get('counts') == 0:
return False
else:
return True
def inset_keyword(self, keywods):
mysqlCn = MysqlCn(self.config)
sql = "INSERT INTO keyword_info(keyword,province_id,priority) VALUES(%s,%s,%s);"
for i in range(len(keywods)):
keyword = keywods[i].get('keyword')
province_id = keywods[i].get('province_id')
priority = 0
result = mysqlCn.insertOne(sql, (keyword, province_id, priority))
mysqlCn.dispose()
return None
def insert(self,msgs):
try:
mysqlCn = MysqlCn(self.__config)
sql = 'INSERT IGNORE INTO bul_content_primary(id,type,person,party,time,pdf,url,create_time) VALUES(%s,%s,%s,%s,%s,%s,%s,%s)'
sql1 = 'INSERT IGNORE INTO bul_content_attachment(id,content) VALUES(%s,%s)'
for ttemp in msgs:
tid = int(ttemp.get('id'))
ttype = str(ttemp.get('type'))
person = str(ttemp.get('person'))
party = str(ttemp.get('party'))
ttime = str(ttemp.get('time'))
pdf = str(ttemp.get('pdf'))
turl = str(ttemp.get('url'))
content = str(ttemp.get('content'))
create_time = time.strftime("%Y-%m-%d %H:%M:%S")
row_id = mysqlCn.insertOne(sql, (tid,ttype,person,party,ttime,pdf,turl,create_time))
content_row_id = mysqlCn.insertOne(sql1, (tid,content))
mysqlCn.dispose()
except Exception as e:
print(e,"入库失败")
def update_sectino_num(self, section_num, current_num, status=0):
mysqlCn = MysqlCn(self.config)
sql = "UPDATE regno_statistic SET `status` = %s,`current_num`= %s WHERE section_num = %s;"
result = mysqlCn.update(sql, (status, current_num, section_num))
mysqlCn.dispose()
return result