def uploadDir(self, local_dir='./', remote_dir='./'): if not os.path.isdir(local_dir): logMsg('error', '%s not exists' % self.local_dir, 2) raise ValueError('114,%s not exists' % self.local_dir) ###创建目录 all_dirs = remote_dir.split(os.sep) for all_dir in all_dirs: try: self.ftp.mkd(all_dir) except: pass self._checkDir(all_dir) ###进入目录 self._checkDir(remote_dir) ###遍历本地目录 for file in os.listdir(local_dir): src = os.path.join(local_dir, file) if os.path.isfile(src): self.uploadFile(src, file) elif os.path.isdir(src): try: self.ftp.mkd(file) except: pass self.uploadDir(src, file) self.ftp.cwd('..')
def __init__(self, **args): ###获取参数 self.host = args.get('mail_host') self.user = args.get('mail_user') self.pswd = args.get('mail_pswd') self.send = args.get('mail_send', '*****@*****.**') self.rece = args.get('mail_rece', '*****@*****.**') ###收件人转成列表 self._rece = self.rece.split(',') ###连接 try: server = smtplib.SMTP() server.connect(self.host) ###选择是否基于ssl try: server.login(self.user, self.pswd) except: server.starttls() server.login(self.user, self.pswd) ###内部变量 self._server = server except smtplib.SMTPException: logMsg('error', '%s connect error' % self.host, 2) raise ValueError('115,%s connect error %s' % self.host)
def make_tar_file(path_list): for path in path_list: suffix_file = path.replace('/', '_') + ".tar.gz" cmd = "cd %s && tar zcvfP %s %s" % (bash_path, suffix_file, path) msg = "Start tar %s" % path logMsg("make_tar", msg, 1) run_cmd(cmd)
def check_workflow(self): is_alter = False # 检查redis中miss_files字段是否有值,写入操作时间及结果,1=True check_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) self.r.set('monitor_logs_time', check_time) is_true, miss_file = self.get_local_miss_files() common.write_file_josn('/tmp/miss_file.log', miss_file) if is_true: self.r.set('miss_local_file', 1) is_alter = True else: self.r.set('miss_local_file', 0) local_file_info = self.get_local_info() hdfs_file_info = self.get_hdfs_info() miss_hdfs_file_key = "miss_hdfs_file_%s" % check_time diff_size_key = "diff_size_%s" % check_time diff_files = dict() miss_hdfs_file = list() for key in local_file_info.keys(): if key not in hdfs_file_info.keys(): self.r.rpush(miss_hdfs_file_key, key) miss_hdfs_file.append(key) is_alter = True if local_file_info.get(key, None) != hdfs_file_info.get(key, None): diff_files[key] = [ local_file_info.get(key, None), hdfs_file_info.get(key, None) ] if diff_files.keys(): self.r.hmset(diff_size_key, diff_files) is_alter = True common.write_file_josn('/tmp/miss_hdfs_file.log', miss_hdfs_file) common.write_file_josn('/tmp/diff_size.log', diff_files) if is_alter: msg = "find error alter mail will send" common.logMsg("sendmail", msg, 1) mail_dict = dict() mail_dict['mail_host'] = "smtp.exmail.qq.com" # 设置服务器 mail_dict['mail_user'] = "******" # 用户名 mail_dict['mail_pswd'] = "tel7206324" # 口令 mail_dict['mail_rece'] = list() mail_dict['mail_rece'].append("*****@*****.**") mail_dict['mail_rece'].append("*****@*****.**") mail_dict['mail_rece'].append('*****@*****.**') m = mailBase(**mail_dict) sub = "日志检查报警邮件" content = "本地与HDFS日志文件存在异常,详见附件!" attachs = ("/tmp/miss_file.log", "/tmp/miss_hdfs_file.log", "/tmp/diff_size.log") m.sendMail(sub, content, 'plain', 'utf-8', *attachs) else: msg = "miss not found" common.logMsg("check", msg, 1)
def get_hdfs_path(path): current_path = "%s/%s" % (base_dir, path) cmd_mkdir = "mkdir -p %s" % current_path run_cmd(cmd_mkdir) get_cmd = "hadoop fs -get %s/* %s " % (path, current_path) msg = "get path %s" % path logMsg("get_path", msg, 1) run_cmd(get_cmd)
def query(self, sql): res = '' try: self.curs.execute(sql) res = self.curs.fetchall() logMsg('select', '%s query ' % sql, 1) except: logMsg('error', '%s query error' % sql, 2) # raise ValueError('query error %s'% sql) return res
def ftp_up(files): pdb.set_trace() if not ftp_check(files): ftp_cmd = "put %s bigdata_hdfs" % files cmd = "lftp -u %s,%s -e'%s;bye' %s" % (ftp_user, ftp_passwd, ftp_cmd, ftp_ip) msg = "put file %s" % files logMsg("up", msg, 1) run_cmd(cmd) else: msg_err = "files %s was in ftp " % files logMsg("check", msg_err, 2)
def ftp_check(files): pdb.set_trace() ftp = FTP(ftp_ip) ftp.login(ftp_user, ftp_passwd) check_file = "bigdata_hdfs/" + files.split(r'/')[-1] try: size = ftp.size(check_file) return True except: msg_err = "files %s was in ftp " % files logMsg("check", msg_err, 2) return False
def write_kafka_redis(key, values, method='set'): r = redis.Redis(host=redis_host, port=port, db=db) if method == 'set': r.set(key, values) elif method == "hmset": try: r.hmset(key, values) r.expire(key, redis_expired) except: msg = "Write key %s error values %s" % (key, json.dumps(values)) logMsg("write_redis", msg, 2) raise IOError else: raise KeyError
def change(self, sql, many=False): ###过滤unknow table的warning warnings.filterwarnings('ignore') resNum = 0 if many: try: ###多条同时插入 resNum = self.curs.executemany(sql, many) self.conn.commit() logMsg('success', '%s exec ' % sql, 1) except: self.conn.rollback() logMsg('error', '%s exec error' % sql, 2) raise ValueError('exec error %s' % sql) else: try: resNum = self.curs.execute(sql) ###提交 self.conn.commit() logMsg('change', '%s exec ' % sql, 1) except: self.conn.rollback() logMsg('error', '%s exec error' % sql, 2) raise ValueError('112,exec error %s' % sql) return resNum
def delete(self, table, condition): _sql = 'DELETE FROM `%s` WHERE %s' % (table, condition) ###执行 resNum = 0 try: resNum = self.curs.execute(_sql) ###提交 self.conn.commit() logMsg('delete', '%s delete ' % _sql, 1) except: self.conn.rollback() logMsg('error', '%s delete error' % _sql, 2) raise ValueError('112,delete error %s' % _sql) return resNum
def __init__(self, **args): ###获取参数 self.host = args.get('ftp_host') self.user = args.get('ftp_user') self.pswd = args.get('ftp_pswd') self.port = args.get('ftp_port', 21) self.timeout = args.get('time_out', 60) ###连接 try: self.ftp = FTP() self.ftp.connect(self.host, self.port, self.timeout) self.ftp.login(self.user, self.pswd) except: logMsg('error', '%s connect error' % self.host, 2) raise ValueError('%s connect error %s' % self.host)
def downloadFile(self, local_path, remote_path): ###备份旧文件 if os.path.isfile(local_path): shutil.move(local_path, "%s.%s" % (local_path, "old")) ###下载文件 try: fp_ftp = open(local_path, 'wb').write self.ftp.retrbinary('RETR %s' % remote_path, fp_ftp) except: down_cmd = "wget ftp://%s:%s@%s/%s -P %s" % ( self.user, self.pswd, self.host, remote_path, os.path.dirname(local_path)) (get_status_down, get_output_down) = run_cmd(down_cmd) ###确认是否执行成功 if (get_status_down != 0): logMsg('error', '%s download error' % remote_path, 2) raise ValueError('114,%s download error' % remote_path)
def update(self, table, data, condition): _field = ','.join(["`%s`='%s'" % (k_update, str(data[k_update]).replace("'", "\'")) for k_update in data]) _sql = 'UPDATE `%s` SET %s WHERE %s' % (table, _field, condition) ###执行 resNum = 0 try: resNum = self.curs.execute(_sql) ###提交 self.conn.commit() logMsg('update', '%s update ' % _sql, 1) except: self.conn.rollback() logMsg('error', '%s update error' % _sql, 2) raise ValueError('update error %s' % _sql) return resNum
def sendMail(self, subject, content, subtype='plain', charset='utf-8', *attachs): if attachs: ###创建一个带附件的实例 msg = MIMEMultipart() ###邮件正文内容 msg.attach(MIMEText(content, subtype, charset)) else: ###txt:plain msg = MIMEText(content, subtype, charset) ###邮件标题 msg['Subject'] = Header(subject, charset) ###附件 for attach in attachs: ###判读文件是否存在 if not os.path.isfile(attach): logMsg('error', '%s not exists' % attach, 2) raise ValueError('115,%s not exists' % attach) ###构造附件 att = MIMEText(open(attach, 'rb').read(), 'base64', 'utf-8') att["Content-Type"] = 'application/octet-stream' ###分解附件路径中的文件名 file_name = os.path.basename(attach) ###邮件中显示什么名字 att["Content-Disposition"] = 'attachment; filename="%s"' % file_name msg.attach(att) ###添加发件人和收件人的显示 msg['From'] = self.send msg['To'] = ";".join(self._rece) ###发送邮件 try: self._server.sendmail(self.send, self._rece, msg.as_string()) return True except Exception, e: logMsg('error', str(e), 2) return False
def insert(self, table, data): _field = ','.join(['`%s`' % (k_insert) for k_insert in data.keys()]) _value = ','.join(["'%s'" % (str(v_insert).replace("'", "\'")) for v_insert in data.values()]) ###拼接成sql语句 _sql = 'INSERT INTO `%s`(%s) VALUES(%s)' % (table, _field, _value) ###执行 self.curs.lastrowid = 0 try: self.curs.execute(_sql) ###提交 self.conn.commit() logMsg('insert', '%s insert ' % _sql, 1) except: self.conn.rollback() logMsg('error', '%s insert error' % _sql, 2) raise ValueError('112,insert error %s' % _sql) return self.curs.lastrowid
def __init__(self, **args): ###获取参数 self.host = args.get('host', 'localhost') self.user = args.get('user') self.pswd = args.get('pswd') self.db = args.get('db', 'mysql') self.port = args.get('port', '3306') self.charset = args.get('charset', 'utf8') try: self.conn = MySQLdb.connect(host=self.host, user=self.user, passwd=self.pswd, db=self.db, port=int(self.port), charset=self.charset) self.curs = self.conn.cursor() self.curs.execute('SET NAMES utf8') except: logMsg('error', '%s mysql connect error' % self.host, 2) raise ValueError('mysql connect error %s' % self.host)
def uploadFile(self, local_path, remote_path='./'): ###判断本地文件 if not os.path.isfile(local_path): logMsg('error', '%s not exists' % self.local_path, 2) raise ValueError('%s not exists' % self.local_path) ###上传的目录 remote_dir = self.ftp.pwd() try: fp_ftp = open(local_path, 'rb') self.ftp.storbinary('STOR %s' % remote_path, fp_ftp) except Exception, e: local_file = os.path.basename(local_path) up_cmd = "ncftpput -z -u%s -p%s %s %s %s" % ( self.user, self.pswd, self.host, remote_dir, local_path) (get_status_up, get_output_up) = run_cmd(up_cmd) ###确认是否执行成功 if (get_status_up != 0): logMsg('error', '%s upload error' % local_path, 2) raise ValueError(',%s upload error' % local_path)
role: mysql的增删改查类 usage: m = mysqlBase(host='xxx',db='xxx',user='******',pwd='xxx') 实例化 m.insert('core',{'host_name':'ccc','process_name':'ddd','ip_addr':'192.168.136.41','status':'4'}) m.update('table',{'field1':'value1','field2':'value2'},'id=1') 更新表名 字段名:值 条件 m.delete('core','status=5 and id=12') m.change("update core set a='aaa' where id=1") 可以多条插入 m.query("select * from core") """ from common import logMsg import warnings try: import MySQLdb except ImportError, e: logMsg('error', 'MySQLdb import error', 2) raise ImportError('No module named MySQLdb') ###mysql操作类 class mysqlBase: ###连接数据库 def __init__(self, **args): ###获取参数 self.host = args.get('host', 'localhost') self.user = args.get('user') self.pswd = args.get('pswd') self.db = args.get('db', 'mysql') self.port = args.get('port', '3306') self.charset = args.get('charset', 'utf8')
timeStamp = int(time.mktime(timeArray)) timeOld = timeStamp - (3600 * interval) timeOldArray = time.localtime(timeOld) timeOldValues = time.strftime("%Y%m%d%H", timeOldArray) return timeOldValues if __name__ == '__main__': lasttime = get_values_redis('lasttime', method='get') topicName = sys.argv[1].split(",") for one_topic in topicName: key_now = "%s_%s" % (one_topic, lasttime) data_now = get_values_redis(key_now, method='hgetall') if not monitor_time_interval.get(one_topic, None): msg = "topic could not find in config files: %s " % one_topic logMsg("get config", msg, 2) raise KeyError time_old = return_time_interval(lasttime, int(monitor_time_interval[one_topic])) key_old = "%s_%s" % (one_topic, time_old) data_old = get_values_redis(key_old, method='hgetall') if (int(data_now['lag']) - int(data_old['lag'])) < int( monitor_values_interval[one_topic]): subjec = "%s 处理异常" % one_topic sendto = monitor_sendto[one_topic] body = "Kafka 数值存在异常" bigdata_send_main(sendto, subjec, body)