def match(self):
self.set_sql_query_rule();
self.set_sql_query_content();
if not self.sql_select_rule:
logging.error("set rule query error");
return -1;
if not self.sql_select_content:
logging.error("set content query error");
return -1;
dbRule = MySQLClient();
#print (multiprocessing.current_process().name);
try:
#print self.sql_select_rule;
dbRule.cursor.execute(self.sql_select_rule);
logging.error("select rule done!");
ruleRow = dbRule.cursor.fetchall();
except Exception as e:
logging.error("sql query error %s", e);
return -1;
dbContent = MySQLClient();
#print (multiprocessing.current_process().name);
try:
dbContent.cursor.execute('update ' + self.typeContent + ' set data_rule_id = null and key_word = null where data_rule_id is not null');
logging.error("update all done!");
dbContent.cursor.execute(self.sql_select_content);
logging.error("select content done!");
contentRow = dbContent.cursor.fetchall();
except Exception as e:
logging.error("sql query error %s", e);
return -1;
logging.error("sql connection done!");
count = 0;
values = [];
for i in contentRow:
for j in ruleRow:
#print i[1];
#print j[1];
flag, result = matchRule(i[1], j[1]);
if(flag == True):
try:
strResult = ",".join(result);
logging.error("sql update strResult %s", i[0]);
values.append((j[0], strResult, i[0]));
#logging.error("sql update strResult %s", values);
count = count + 1;
#sql_update = 'update %s set data_rule_id = %s, key_word = \'%s\' where id = %s' %(self.typeContent, j[0], strResult, i[0]);
#print sql_update;
#dbContent.cursor.execute(sql_update);
break;
except Exception as e:
logging.error("sql update error %s", e);
return -1;
if(count == 100):
print count;
print values;
logging.error("sql update strResult %s", values);
try:
update_sql = 'update ' + self.typeContent + ' set data_rule_id = %s, key_word = %s where id = %s'
#dbContent.cursor.executemany('update content_isproduct set data_rule_id=%s, key_word=%s where id=%s', values);
dbContent.cursor.executemany(update_sql, values);
count = 0;
values = [];
except Exception as e:
logging.error("sql executemany error %s", e);
return -1;
if(count != 0):
print count;
print values;
logging.error("sql update strResult %s", values);
try:
update_sql = 'update ' + self.typeContent + ' set data_rule_id = %s, key_word = %s where id = %s';
logging.error(update_sql);
dbContent.cursor.executemany(update_sql, values);
#dbContent.cursor.executemany('update content_isproduct_copy set data_rule_id=%s, key_word=%s where id=%s', values);
count = 0;
values = [];
except Exception as e:
logging.error("sql executemany error %s", e);
return -1;
dbRule.close();
dbContent.close();
def mysql_save(create_time,content): mysql_client = MySQLClient() with mysql_client: mysql_client.insert(create_time,content)
def match(self):
self.set_sql_query_rule()
self.set_sql_query_content()
if not self.sql_select_rule:
logging.error("set rule query error")
return -1
if not self.sql_select_content:
logging.error("set content query error")
return -1
dbRule = MySQLClient()
#print (multiprocessing.current_process().name);
try:
#print self.sql_select_rule;
dbRule.cursor.execute(self.sql_select_rule)
logging.error("select rule done!")
ruleRow = dbRule.cursor.fetchall()
except Exception as e:
logging.error("sql query error %s", e)
return -1
dbContent = MySQLClient()
#print (multiprocessing.current_process().name);
try:
dbContent.cursor.execute(
'update ' + self.typeContent +
' set data_rule_id = null and key_word = null where data_rule_id is not null'
)
logging.error("update all done!")
dbContent.cursor.execute(self.sql_select_content)
logging.error("select content done!")
contentRow = dbContent.cursor.fetchall()
except Exception as e:
logging.error("sql query error %s", e)
return -1
logging.error("sql connection done!")
count = 0
values = []
for i in contentRow:
for j in ruleRow:
#print i[1];
#print j[1];
flag, result = matchRule(i[1], j[1])
if (flag == True):
try:
strResult = ",".join(result)
logging.error("sql update strResult %s", i[0])
values.append((j[0], strResult, i[0]))
#logging.error("sql update strResult %s", values);
count = count + 1
#sql_update = 'update %s set data_rule_id = %s, key_word = \'%s\' where id = %s' %(self.typeContent, j[0], strResult, i[0]);
#print sql_update;
#dbContent.cursor.execute(sql_update);
break
except Exception as e:
logging.error("sql update error %s", e)
return -1
if (count == 100):
print count
print values
logging.error("sql update strResult %s", values)
try:
update_sql = 'update ' + self.typeContent + ' set data_rule_id = %s, key_word = %s where id = %s'
#dbContent.cursor.executemany('update content_isproduct set data_rule_id=%s, key_word=%s where id=%s', values);
dbContent.cursor.executemany(update_sql, values)
count = 0
values = []
except Exception as e:
logging.error("sql executemany error %s", e)
return -1
if (count != 0):
print count
print values
logging.error("sql update strResult %s", values)
try:
update_sql = 'update ' + self.typeContent + ' set data_rule_id = %s, key_word = %s where id = %s'
logging.error(update_sql)
dbContent.cursor.executemany(update_sql, values)
#dbContent.cursor.executemany('update content_isproduct_copy set data_rule_id=%s, key_word=%s where id=%s', values);
count = 0
values = []
except Exception as e:
logging.error("sql executemany error %s", e)
return -1
dbRule.close()
dbContent.close()
def worker(self,worker_id,):
self.set_sql_querry(worker_id)
if not self.sql_select_str:
logging.error("set query error")
return -1
time.sleep(worker_id)
db = MySQLClient(host=self.host,port=self.port,user=self.user,passwd=self.passwd,db=self.DB_name)
algo = algoClient(self.serIp,self.serport)
print multiprocessing.current_process().name
#执行select语句
try:
logging.info("Start select")
db.cursor.execute(self.sql_select_str)
res = db.cursor.fetchall()
logging.info("End select")
except Exception as e:
traceback.print_exc()
logging.error("sql query error %s", e)
return -1
#处理读出的每一条记录
count=0
values=[]
logging.info("Start fetch result")
#loop all the rows fetched
for i in res:
#query api to get values
try:
resultlist=[]
for j in self.servicelist:
resultlist.append(getattr(algo.client,j)(i[1]))
logging.info("id: %s \t result: %s", i[0],getattr(algo.client,j)(i[1]))
resultlist.append(i[0])
values.append(tuple(resultlist))
count=count+1
except Exception as e:
algo.close()
algo = algoClient(self.serIp,self.serport)
traceback.print_exc()
logging.error("algo error: %s: %s",i[1],e)
continue
#generate update sql and batch update db
try:
tmpstr=''
for d in range(0,len(self.field_name)-1):
tmpstr=tmpstr+self.field_name[d]+' =%s,'
tmpstr=tmpstr+self.field_name[len(self.field_name)-1]+' =%s'
self.sql_update_str='update '+self.table_name+' set '+tmpstr+' where id =%s'
if(count==self.sqlBatchUpdate_num):
logging.info("End fetch result")
db.cursor.executemany(self.sql_update_str,values)
logging.info("Update complete")
count=0
values=[]
except Exception as e:
print "cfs"
traceback.print_exc()
logging.error("sql update error: %s",e)
continue
if count!=0:
db.cursor.executemany(self.sql_update_str,values)
logging.info("Update All comllete")
db.close()
algo.close()
def worker(
self,
worker_id,
):
self.set_sql_querry(worker_id)
if not self.sql_select_str:
logging.error("set query error")
return -1
time.sleep(worker_id)
db = MySQLClient(host=self.host,
port=self.port,
user=self.user,
passwd=self.passwd,
db=self.DB_name)
algo = algoClient(self.serIp, self.serport)
print multiprocessing.current_process().name
#执行select语句
try:
logging.info("Start select")
db.cursor.execute(self.sql_select_str)
res = db.cursor.fetchall()
logging.info("End select")
except Exception as e:
traceback.print_exc()
logging.error("sql query error %s", e)
return -1
#处理读出的每一条记录
count = 0
values = []
logging.info("Start fetch result")
#loop all the rows fetched
for i in res:
#query api to get values
try:
resultlist = []
for j in self.servicelist:
resultlist.append(getattr(algo.client, j)(i[1]))
logging.info("id: %s \t result: %s", i[0],
getattr(algo.client, j)(i[1]))
resultlist.append(i[0])
values.append(tuple(resultlist))
count = count + 1
except Exception as e:
algo.close()
algo = algoClient(self.serIp, self.serport)
traceback.print_exc()
logging.error("algo error: %s: %s", i[1], e)
continue
#generate update sql and batch update db
try:
tmpstr = ''
for d in range(0, len(self.field_name) - 1):
tmpstr = tmpstr + self.field_name[d] + ' =%s,'
tmpstr = tmpstr + self.field_name[len(self.field_name) -
1] + ' =%s'
self.sql_update_str = 'update ' + self.table_name + ' set ' + tmpstr + ' where id =%s'
if (count == self.sqlBatchUpdate_num):
logging.info("End fetch result")
db.cursor.executemany(self.sql_update_str, values)
logging.info("Update complete")
count = 0
values = []
except Exception as e:
print "cfs"
traceback.print_exc()
logging.error("sql update error: %s", e)
continue
if count != 0:
db.cursor.executemany(self.sql_update_str, values)
logging.info("Update All comllete")
db.close()
algo.close()
