def uploadDir(self, local_dir='./', remote_dir='./'):
if not os.path.isdir(local_dir):
logMsg('error', '%s not exists' % self.local_dir, 2)
raise ValueError('114,%s not exists' % self.local_dir)
###创建目录
all_dirs = remote_dir.split(os.sep)
for all_dir in all_dirs:
try:
self.ftp.mkd(all_dir)
except:
pass
self._checkDir(all_dir)
###进入目录
self._checkDir(remote_dir)
###遍历本地目录
for file in os.listdir(local_dir):
src = os.path.join(local_dir, file)
if os.path.isfile(src):
self.uploadFile(src, file)
elif os.path.isdir(src):
try:
self.ftp.mkd(file)
except:
pass
self.uploadDir(src, file)
self.ftp.cwd('..')
def __init__(self, **args):
###获取参数
self.host = args.get('mail_host')
self.user = args.get('mail_user')
self.pswd = args.get('mail_pswd')
self.send = args.get('mail_send', '*****@*****.**')
self.rece = args.get('mail_rece', '*****@*****.**')
###收件人转成列表
self._rece = self.rece.split(',')
###连接
try:
server = smtplib.SMTP()
server.connect(self.host)
###选择是否基于ssl
try:
server.login(self.user, self.pswd)
except:
server.starttls()
server.login(self.user, self.pswd)
###内部变量
self._server = server
except smtplib.SMTPException:
logMsg('error', '%s connect error' % self.host, 2)
raise ValueError('115,%s connect error %s' % self.host)
def make_tar_file(path_list):
for path in path_list:
suffix_file = path.replace('/', '_') + ".tar.gz"
cmd = "cd %s && tar zcvfP %s %s" % (bash_path, suffix_file, path)
msg = "Start tar %s" % path
logMsg("make_tar", msg, 1)
run_cmd(cmd)
def check_workflow(self):
is_alter = False
# 检查redis中miss_files字段是否有值，写入操作时间及结果，1=True
check_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
self.r.set('monitor_logs_time', check_time)
is_true, miss_file = self.get_local_miss_files()
common.write_file_josn('/tmp/miss_file.log', miss_file)
if is_true:
self.r.set('miss_local_file', 1)
is_alter = True
else:
self.r.set('miss_local_file', 0)
local_file_info = self.get_local_info()
hdfs_file_info = self.get_hdfs_info()
miss_hdfs_file_key = "miss_hdfs_file_%s" % check_time
diff_size_key = "diff_size_%s" % check_time
diff_files = dict()
miss_hdfs_file = list()
for key in local_file_info.keys():
if key not in hdfs_file_info.keys():
self.r.rpush(miss_hdfs_file_key, key)
miss_hdfs_file.append(key)
is_alter = True
if local_file_info.get(key, None) != hdfs_file_info.get(key, None):
diff_files[key] = [
local_file_info.get(key, None),
hdfs_file_info.get(key, None)
]
if diff_files.keys():
self.r.hmset(diff_size_key, diff_files)
is_alter = True
common.write_file_josn('/tmp/miss_hdfs_file.log', miss_hdfs_file)
common.write_file_josn('/tmp/diff_size.log', diff_files)
if is_alter:
msg = "find error alter mail will send"
common.logMsg("sendmail", msg, 1)
mail_dict = dict()
mail_dict['mail_host'] = "smtp.exmail.qq.com" # 设置服务器
mail_dict['mail_user'] = "******" # 用户名
mail_dict['mail_pswd'] = "tel7206324" # 口令
mail_dict['mail_rece'] = list()
mail_dict['mail_rece'].append("*****@*****.**")
mail_dict['mail_rece'].append("*****@*****.**")
mail_dict['mail_rece'].append('*****@*****.**')
m = mailBase(**mail_dict)
sub = "日志检查报警邮件"
content = "本地与HDFS日志文件存在异常，详见附件！"
attachs = ("/tmp/miss_file.log", "/tmp/miss_hdfs_file.log",
"/tmp/diff_size.log")
m.sendMail(sub, content, 'plain', 'utf-8', *attachs)
else:
msg = "miss not found"
common.logMsg("check", msg, 1)
def get_hdfs_path(path):
current_path = "%s/%s" % (base_dir, path)
cmd_mkdir = "mkdir -p %s" % current_path
run_cmd(cmd_mkdir)
get_cmd = "hadoop fs -get %s/* %s " % (path, current_path)
msg = "get path %s" % path
logMsg("get_path", msg, 1)
run_cmd(get_cmd)
def query(self, sql):
res = ''
try:
self.curs.execute(sql)
res = self.curs.fetchall()
logMsg('select', '%s query ' % sql, 1)
except:
logMsg('error', '%s query error' % sql, 2)
# raise ValueError('query error %s'% sql)
return res
def ftp_up(files):
pdb.set_trace()
if not ftp_check(files):
ftp_cmd = "put %s bigdata_hdfs" % files
cmd = "lftp -u %s,%s -e'%s;bye' %s" % (ftp_user, ftp_passwd, ftp_cmd,
ftp_ip)
msg = "put file %s" % files
logMsg("up", msg, 1)
run_cmd(cmd)
else:
msg_err = "files %s was in ftp " % files
logMsg("check", msg_err, 2)
def ftp_check(files):
pdb.set_trace()
ftp = FTP(ftp_ip)
ftp.login(ftp_user, ftp_passwd)
check_file = "bigdata_hdfs/" + files.split(r'/')[-1]
try:
size = ftp.size(check_file)
return True
except:
msg_err = "files %s was in ftp " % files
logMsg("check", msg_err, 2)
return False
def write_kafka_redis(key, values, method='set'):
r = redis.Redis(host=redis_host, port=port, db=db)
if method == 'set':
r.set(key, values)
elif method == "hmset":
try:
r.hmset(key, values)
r.expire(key, redis_expired)
except:
msg = "Write key %s error values %s" % (key, json.dumps(values))
logMsg("write_redis", msg, 2)
raise IOError
else:
raise KeyError
def change(self, sql, many=False):
###过滤unknow table的warning
warnings.filterwarnings('ignore')
resNum = 0
if many:
try:
###多条同时插入
resNum = self.curs.executemany(sql, many)
self.conn.commit()
logMsg('success', '%s exec ' % sql, 1)
except:
self.conn.rollback()
logMsg('error', '%s exec error' % sql, 2)
raise ValueError('exec error %s' % sql)
else:
try:
resNum = self.curs.execute(sql)
###提交
self.conn.commit()
logMsg('change', '%s exec ' % sql, 1)
except:
self.conn.rollback()
logMsg('error', '%s exec error' % sql, 2)
raise ValueError('112,exec error %s' % sql)
return resNum
def delete(self, table, condition):
_sql = 'DELETE FROM `%s` WHERE %s' % (table, condition)
###执行
resNum = 0
try:
resNum = self.curs.execute(_sql)
###提交
self.conn.commit()
logMsg('delete', '%s delete ' % _sql, 1)
except:
self.conn.rollback()
logMsg('error', '%s delete error' % _sql, 2)
raise ValueError('112,delete error %s' % _sql)
return resNum
def __init__(self, **args):
###获取参数
self.host = args.get('ftp_host')
self.user = args.get('ftp_user')
self.pswd = args.get('ftp_pswd')
self.port = args.get('ftp_port', 21)
self.timeout = args.get('time_out', 60)
###连接
try:
self.ftp = FTP()
self.ftp.connect(self.host, self.port, self.timeout)
self.ftp.login(self.user, self.pswd)
except:
logMsg('error', '%s connect error' % self.host, 2)
raise ValueError('%s connect error %s' % self.host)
def downloadFile(self, local_path, remote_path):
###备份旧文件
if os.path.isfile(local_path):
shutil.move(local_path, "%s.%s" % (local_path, "old"))
###下载文件
try:
fp_ftp = open(local_path, 'wb').write
self.ftp.retrbinary('RETR %s' % remote_path, fp_ftp)
except:
down_cmd = "wget ftp://%s:%s@%s/%s -P %s" % (
self.user, self.pswd, self.host, remote_path,
os.path.dirname(local_path))
(get_status_down, get_output_down) = run_cmd(down_cmd)
###确认是否执行成功
if (get_status_down != 0):
logMsg('error', '%s download error' % remote_path, 2)
raise ValueError('114,%s download error' % remote_path)
def update(self, table, data, condition):
_field = ','.join(["`%s`='%s'" % (k_update, str(data[k_update]).replace("'", "\'")) for k_update in data])
_sql = 'UPDATE `%s` SET %s WHERE %s' % (table, _field, condition)
###执行
resNum = 0
try:
resNum = self.curs.execute(_sql)
###提交
self.conn.commit()
logMsg('update', '%s update ' % _sql, 1)
except:
self.conn.rollback()
logMsg('error', '%s update error' % _sql, 2)
raise ValueError('update error %s' % _sql)
return resNum
def sendMail(self, subject, content, subtype='plain', charset='utf-8', *attachs):
if attachs:
###创建一个带附件的实例
msg = MIMEMultipart()
###邮件正文内容
msg.attach(MIMEText(content, subtype, charset))
else:
###txt:plain
msg = MIMEText(content, subtype, charset)
###邮件标题
msg['Subject'] = Header(subject, charset)
###附件
for attach in attachs:
###判读文件是否存在
if not os.path.isfile(attach):
logMsg('error', '%s not exists' % attach, 2)
raise ValueError('115,%s not exists' % attach)
###构造附件
att = MIMEText(open(attach, 'rb').read(), 'base64', 'utf-8')
att["Content-Type"] = 'application/octet-stream'
###分解附件路径中的文件名
file_name = os.path.basename(attach)
###邮件中显示什么名字
att["Content-Disposition"] = 'attachment; filename="%s"' % file_name
msg.attach(att)
###添加发件人和收件人的显示
msg['From'] = self.send
msg['To'] = ";".join(self._rece)
###发送邮件
try:
self._server.sendmail(self.send, self._rece, msg.as_string())
return True
except Exception, e:
logMsg('error', str(e), 2)
return False
def insert(self, table, data):
_field = ','.join(['`%s`' % (k_insert) for k_insert in data.keys()])
_value = ','.join(["'%s'" % (str(v_insert).replace("'", "\'")) for v_insert in data.values()])
###拼接成sql语句
_sql = 'INSERT INTO `%s`(%s) VALUES(%s)' % (table, _field, _value)
###执行
self.curs.lastrowid = 0
try:
self.curs.execute(_sql)
###提交
self.conn.commit()
logMsg('insert', '%s insert ' % _sql, 1)
except:
self.conn.rollback()
logMsg('error', '%s insert error' % _sql, 2)
raise ValueError('112,insert error %s' % _sql)
return self.curs.lastrowid
def __init__(self, **args):
###获取参数
self.host = args.get('host', 'localhost')
self.user = args.get('user')
self.pswd = args.get('pswd')
self.db = args.get('db', 'mysql')
self.port = args.get('port', '3306')
self.charset = args.get('charset', 'utf8')
try:
self.conn = MySQLdb.connect(host=self.host, user=self.user, passwd=self.pswd, db=self.db,
port=int(self.port), charset=self.charset)
self.curs = self.conn.cursor()
self.curs.execute('SET NAMES utf8')
except:
logMsg('error', '%s mysql connect error' % self.host, 2)
raise ValueError('mysql connect error %s' % self.host)
def uploadFile(self, local_path, remote_path='./'):
###判断本地文件
if not os.path.isfile(local_path):
logMsg('error', '%s not exists' % self.local_path, 2)
raise ValueError('%s not exists' % self.local_path)
###上传的目录
remote_dir = self.ftp.pwd()
try:
fp_ftp = open(local_path, 'rb')
self.ftp.storbinary('STOR %s' % remote_path, fp_ftp)
except Exception, e:
local_file = os.path.basename(local_path)
up_cmd = "ncftpput -z -u%s -p%s %s %s %s" % (
self.user, self.pswd, self.host, remote_dir, local_path)
(get_status_up, get_output_up) = run_cmd(up_cmd)
###确认是否执行成功
if (get_status_up != 0):
logMsg('error', '%s upload error' % local_path, 2)
raise ValueError(',%s upload error' % local_path)
role: mysql的增删改查类
usage: m = mysqlBase(host='xxx',db='xxx',user='******',pwd='xxx') 实例化
m.insert('core',{'host_name':'ccc','process_name':'ddd','ip_addr':'192.168.136.41','status':'4'})
m.update('table',{'field1':'value1','field2':'value2'},'id=1') 更新表名 字段名：值 条件
m.delete('core','status=5 and id=12')
m.change("update core set a='aaa' where id=1") 可以多条插入
m.query("select * from core")
"""
from common import logMsg
import warnings
try:
import MySQLdb
except ImportError, e:
logMsg('error', 'MySQLdb import error', 2)
raise ImportError('No module named MySQLdb')
###mysql操作类
class mysqlBase:
###连接数据库
def __init__(self, **args):
###获取参数
self.host = args.get('host', 'localhost')
self.user = args.get('user')
self.pswd = args.get('pswd')
self.db = args.get('db', 'mysql')
self.port = args.get('port', '3306')
self.charset = args.get('charset', 'utf8')
timeStamp = int(time.mktime(timeArray))
timeOld = timeStamp - (3600 * interval)
timeOldArray = time.localtime(timeOld)
timeOldValues = time.strftime("%Y%m%d%H", timeOldArray)
return timeOldValues
if __name__ == '__main__':
lasttime = get_values_redis('lasttime', method='get')
topicName = sys.argv[1].split(",")
for one_topic in topicName:
key_now = "%s_%s" % (one_topic, lasttime)
data_now = get_values_redis(key_now, method='hgetall')
if not monitor_time_interval.get(one_topic, None):
msg = "topic could not find in config files: %s " % one_topic
logMsg("get config", msg, 2)
raise KeyError
time_old = return_time_interval(lasttime,
int(monitor_time_interval[one_topic]))
key_old = "%s_%s" % (one_topic, time_old)
data_old = get_values_redis(key_old, method='hgetall')
if (int(data_now['lag']) - int(data_old['lag'])) < int(
monitor_values_interval[one_topic]):
subjec = "%s 处理异常" % one_topic
sendto = monitor_sendto[one_topic]
body = "Kafka 数值存在异常"
bigdata_send_main(sendto, subjec, body)
