def batch_insert(self,entrys,table,db,info_fields=INFO_FIELDS):
LOG.info("Begin To Insert Into [%s.%s]..."%(db,table))
con = None
large_values = []
for entry in entrys:
value = []
for field in info_fields:
value.append(entry.get(field))
large_values.append(value)
values_part = ','.join(['%s'] * len(info_fields))
fields_part = ','.join(info_fields)
sql = "insert into %s.%s (%s) values(%s)"%(db,table,fields_part,values_part)
#设定一批插入的条数
count = 0
batch_size = BATCH_SIZE
values = []
flag = False
for i in xrange(0, len(large_values), batch_size):
values = large_values[i:i+batch_size]
try:
con = self.get_connection()
cur = con.cursor()
cur.executemany(sql,values)
con.commit()
count += 1
except mdb.Error, e:
LOG.error(e)
flag = True
finally:
def recr_table(self, table, database, t_table, t_database):
LOG.info("Re-Create Table [%s.%s]"%(database,table))
sql = "DROP TABLE IF EXISTS %s.%s;" % (database, table)
self.exe_sql(sql)
sql = "CREATE TABLE %s.%s LIKE %s.%s;" % (database, table,
t_database, t_table)
self.exe_sql(sql)
def sendmail(mailfrom,mailto, title, text):
msg = MIMEMultipart()
#text_msg = MIMEText(text, 'plain', 'utf-8')
text_html = MIMEText(text, 'html')
#msg.attach(text_msg)
msg.attach(text_html)
toad = mailto.split(";")
msg['to'] = mailto
msg['from'] = mailfrom
msg['subject'] = title
try:
s = smtplib.SMTP()
s.connect(HOST,PORT)
s.login(USER, PASSWD)
try:
s.sendmail(msg['from'],toad,msg.as_string())
LOG.info("Send Email Successfully, To:%s"%msg['to'])
except Exception,e:
LOG.error(e)
s.quit()
count += 1
except mdb.Error, e:
LOG.error(e)
flag = True
finally:
if con:
con.close()
if count == 0:
if flag: total_num = 0
else: total_num = len(values)
else:
if flag: total_num = (count - 1) * batch_size
else:total_num = (count - 1) * batch_size + len(values)
LOG.info("Finish Inserting [%s] Entrys Has Been Done."%(total_num))
#@param columns 为"*",或者为list
def get_results_by_clms(self,columns,table,database=DB,isdict=False):
res = []
cursor_type = MySQLdb.cursors.Cursor
if columns is not None:
if columns == '*':
columns_str = '*'
elif isinstance(columns,list):
columns_str = ",".join(columns)
else:
LOG.error("Parameter colums type is not validated!")
sys.exit(-1)
limit = ""
sql = "select " + columns_str + " from " + table + limit + ";"