def init(module="hat",
load_host=False,
jid=None,
jname=None,
show_console=True,
tm_tag=True):
from hqpy import hqenv
from hqpy import logger
from hqpy import hqhosts
from hqpy import hqio
if os.environ.has_key('HATROOT'):
hat_work_dir = os.environ['HATROOT']
else:
hat_work_dir = "/var/tmp/hat"
hqio.mkdirs(hat_work_dir)
hqenv.set_var('HQVAR_WORK_DIR', hat_work_dir)
hqenv.set_var('HQVAR_LOG_DIR', os.path.join(hat_work_dir, 'log'))
hqenv.set_var('HQVAR_SRC_DIR', os.path.join(hat_work_dir, 'src'))
hqenv.set_var('HQVAR_BIN_DIR', os.path.join(hat_work_dir, 'bin'))
hqenv.set_var('HQVAR_CONF_DIR', os.path.join(hat_work_dir, 'conf'))
hqenv.set_var('HQVAR_VAR_DIR', os.path.join(hat_work_dir, 'var'))
hqenv.set_var('HQVAR_RLS_DIR', os.path.join(hat_work_dir, 'files/release'))
hqenv.set_var('HQVAR_FILES_DIR', os.path.join(hat_work_dir, 'files'))
hqenv.set_var('HQVAR_SCRIPT_DIR', os.path.join(hat_work_dir, 'script'))
hqenv.set_var('HQVAR_HGAME_DIR',
os.path.abspath(os.path.join(hat_work_dir, "..")))
hqio.mkdirs(hqenv.get_var('HQVAR_LOG_DIR'))
if jid is not None:
#job mode
hqenv.set_var('HQVAR_JOBID', jid)
logger.init_job_logger(hqenv.get_var('HQVAR_LOG_DIR'), jid)
else:
#normal mode
hqenv.set_var('HQVAR_JOBID', 0)
logger.init_normal_logger(hqenv.get_var('HQVAR_LOG_DIR'), module,
show_console, tm_tag)
# load env.json
fini = os.path.join(hqenv.get_var('HQVAR_CONF_DIR'), 'env.ini')
hret, envcfg = hqpy.loadini(fini)
if hret.iserr():
logger.ERR("load env.json failed:%s", hret.string())
return False
else:
hqenv.set_var('HQVAR_ENV_CFG', envcfg)
# load host cfg
if load_host == True:
hret = hqhosts.init_hosts()
if hret.iserr():
logger.ERR('hqhosts init failed:%s', hret.string())
return False
#logger.DEBUG('ALL HQPY VAR:%s', hqenv.get_all())
return True
def do(self):
packs = self.conf['packages']
logger.DEBUG('%s run yum to install packages:%s', self.info(), packs)
cmdstr = 'yum install -y {0}'.format(packs)
ret, ostd, oerr = hqpy.run_shell(cmdstr)
if ret.iserr():
logger.ERR("%s run yum cmd error:%s", self.info(), ret.string())
return ret
def load_job(job_file, param):
fd = None
hret = hqpy.HqError()
je = None
try:
fd = open(job_file)
jscfg = json.load(fd)
je = JobEntry(param)
hret = je.load(jscfg)
if hret.iserr():
je = None
logger.ERR('load job failed:%s', hret.string())
except Exception, ex:
template = "An exception of type {0} occured. Arguments:{1!r}"
message = template.format(type(ex).__name__, ex.args)
hret.errno = hqpy.PYRET_TASK_LOAD_ERR
hret.msg = 'TASK Error:{0}'.format(message)
logger.ERR(traceback.format_exc())
def loadini(inifile):
hret = hqpy.HqError()
iniobj = ConfigParser.ConfigParser()
try:
iniobj.read(inifile)
except Exception, ex:
template = "An exception of type {0} occured. Arguments:{1!r}"
message = template.format(type(ex).__name__, ex.args)
hret.errno = hqpy.PYRET_TASK_LOAD_ERR
hret.msg = 'FUNCS Error loadini:{0}'.format(message)
logger.ERR(traceback.format_exc())
def loadjs(jsfile):
hret = hqpy.HqError()
jsobj = None
try:
fd = open(jsfile)
jsobj = json.load(fd)
fd.close()
except Exception, ex:
template = "An exception of type {0} occured. Arguments:{1!r}"
message = template.format(type(ex).__name__, ex.args)
hret.errno = hqpy.PYRET_TASK_LOAD_ERR
hret.msg = 'FUNCS Error loadjs:{0}'.format(message)
logger.ERR(traceback.format_exc())
def writejs(jsfile, jsdata):
hret = hqpy.HqError()
try:
fd = open(jsfile, 'w')
#jsstr = json.dumps(jsdata, sort_keys=True,indent=4)
json.dump(jsdata, fd, sort_keys=True, indent=4)
fd.close()
except Exception, ex:
template = "An exception of type {0} occured. Arguments:{1!r}"
message = template.format(type(ex).__name__, ex.args)
hret.errno = hqpy.PYRET_TASK_LOAD_ERR
hret.msg = 'FUNCS Error writejs:{0}'.format(message)
logger.ERR(traceback.format_exc())
def run_shell(cmd, timeout=None, show_console=False):
hret = hqpy.HqError()
pipe = None
ostd = None
oerr = None
if timeout == None:
timeout = 180
## paramiko 内timeout不生效
if platform.system() != "Windows":
signal.signal(signal.SIGALRM, __signal_run_timeout)
signal.alarm(timeout)
try:
pipe = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True,
universal_newlines=True)
# st = pipe.wait()
# oerr = pipe.stderr.read()
# ostd = pipe.stdout.read()
stdlines = []
while True:
line = pipe.stdout.readline()
if not line:
break
stdlines.append(line)
if show_console == True:
logger.LOG("%s", line.rstrip())
ostd = ''.join(stdlines)
st = pipe.wait()
oerr = pipe.stderr.read()
strip_err = oerr.strip()
if show_console == True and len(strip_err) > 0:
logger.LOG("%s", oerr)
hret.errno = st
if platform.system() != "Windows":
signal.alarm(0)
except Exception, ex:
template = "An exception of type {0} occured. Arguments:{1!r}"
message = template.format(type(ex).__name__, ex.args)
hret.errno = hqpy.PYRET_CMD_RUN_ERR
hret.msg = 'FUNCS Error run_shell:{0}'.format(message)
logger.ERR(traceback.format_exc())
def get_arg(self, name):
if self.conf.has_key(name) == False:
return False, None
val = self.conf[name]
# if type(val) == unicode:
# val = val.encode("utf-8")
btype = type(val) == str or type(val) == unicode
if btype and val.startswith('JVAR_') == True:
m = re.match('JVAR_(\d+)', val)
if m is None:
logger.ERR('%s invalid job var:%s', self.info(), val)
return False, None
aidx = int(m.group(1))
args = self.get_param().get_args()
if aidx >= len(args):
logger.ERR('%s invalid job var idx:%s', self.info(), val)
return False, None
logger.DEBUG('replace arg:%s,%s', aidx, args[aidx])
return True, args[aidx]
else:
return True, val
def remove(fname):
hret = hqpy.HqError()
if os.path.exists(fname) == False:
return hret
try:
if os.path.isfile(fname):
os.remove(fname)
else:
shutil.rmtree(fname)
except Exception, ex:
template = "An exception of type {0} occured. Arguments:{1!r}"
message = template.format(type(ex).__name__, ex.args)
hret.errno = hqpy.PYRET_IO_LIB_ERR
hret.msg = 'IO remove Error:{0}'.format(message)
logger.ERR(traceback.format_exc())
def tar_zip(src, file_name, flist=None):
cmdstr = ""
if os.path.isfile(src):
cmdstr = "tar cfz %s %s" % (file_name, src)
else:
cmdstr = "cd %s && tar cfz %s * " % (src, file_name)
if flist is not None:
strlist = ' '.join(flist)
cmdstr = "cd %s && tar cfz %s %s " % (src, file_name, strlist)
logger.DEBUG("tar_zip cmd:%s", cmdstr)
hret, ostd, oerr = hqpy.run_shell(cmdstr, show_console=True, timeout=180)
if hret.iserr():
logger.ERR("tar_zip failed:%s,%s,%s \n%s", src, file_name, flist, oerr)
return hret
def do(self):
hqret = hqpy.HqError()
sau = self.jobp.get_auth()
host_info = '[%s@%s:%s]' % (sau.user, sau.host, self.jobp.get_jobid())
logger.INFO('>>>>>>>>>>>>>>>>>> %s do all tasks in [%s] ', host_info,
self.jname)
for t in self.tlist:
logger.INFO('%s start to do', t.info())
hqret = t.do()
if hqret.iserr():
logger.ERR('%s do failed, err:%s', t.info(), hqret.string())
return hqret
else:
logger.INFO('%s do sucesss', t.info())
logger.INFO('<<<<<<<<<<<<<<<<< %s do all tasks in [%s] ok', host_info,
self.jname)
return hqret
def run_shell_std(cmd, timeout=None):
if timeout == None:
timeout = 180
hret = hqpy.HqError()
## paramiko 内timeout不生效
signal.signal(signal.SIGALRM, __signal_run_timeout)
signal.alarm(timeout)
try:
hret.errno = subprocess.call(cmd, shell=True)
signal.alarm(0)
except Exception, ex:
template = "An exception of type {0} occured. Arguments:{1!r}"
message = template.format(type(ex).__name__, ex.args)
hret.errno = hqpy.PYRET_CMD_RUN_ERR
hret.msg = 'FUNCS Error run_shell:{0}'.format(message)
logger.ERR(traceback.format_exc())
def do(self):
ret = hqpy.HqError()
l_files = self.conf['files']
sau = self.get_param().get_auth()
jobid = self.get_param().get_jobid()
tmo = None
if self.conf.has_key('timeout'):
tmo = self.conf['timeout']
for x in self.conf['files']:
local_file = os.path.join(hqenv.get_var('HQVAR_WORK_DIR'), x)
bname = os.path.basename(x)
remote_file = os.path.join(self.conf['path'], bname)
ret = hqpy.sftpPut(sau, local_file, remote_file, tmo)
if ret.iserr():
logger.ERR('%s put file failed:%s', self.info(), x)
return ret
return ret
def do(self):
ret = hqpy.HqError()
r_files = self.conf['files']
sau = self.get_param().get_auth()
jobid = self.get_param().get_jobid()
tmo = None
if self.conf.has_key('timeout'):
tmo = self.conf['timeout']
local_tmp = hqenv.get_var('HQVAR_VAR_DIR')
for r in r_files:
l_file = os.path.join(local_tmp, '%s_%s_%s' % (jobid, sau.host, r))
ret = hqpy.sftpGet(sau, r, l_file, tmo)
if ret.iserr():
logger.ERR('TaskGet get file %s failed:%s', r, ret.string())
return ret
else:
logger.INFO('TaskGet ok:%s', l_file)
return ret
def sftpGet(sau, fremote, flocal, timeout = None):
hret = hqpy.HqError()
#ts = None
sftp = None
if timeout is None:
timeout = 120
## paramiko 内timeout不生效
signal.signal(signal.SIGALRM, signal_get_timeout)
signal.alarm(timeout)
try:
# ts=paramiko.Transport((sau.host, sau.port))
# ts.connect(username=sau.user, password=sau.pwd, timeout=timeout)
# sftp=paramiko.SFTPClient.from_transport(ts)
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname = sau.host,
port = sau.port,
username=sau.user,
password=sau.pwd,
timeout=timeout)
sftp = ssh.open_sftp()
sftp.get(fremote, flocal)
signal.alarm(0)
except socket.timeout:
hret.errno = hqpy.PYRET_TIMEOUT
hret.msg = 'SFTP Get Error: timeout'
except Exception,ex:
template = "An exception of type {0} occured. Arguments:{1!r}"
message = template.format(type(ex).__name__, ex.args)
hret.errno = hqpy.PYRET_SSH_GETERR
hret.msg = 'SFTP Get Error:{0}'.format(ex)
logger.ERR(traceback.format_exc())
def TestLogger():
logger.EXIT("xxxxxxxxxxxxxxxxxx")
logger.WARN('test %s %s', 1, 222.2222)
logger.WARN('xxxxxxxxxxxxxxxxxxxxxxxxx')
time.sleep(1)
logger.DEBUG('xxxxxxxxxxxxxxxxxxxxxxxxx222:%s,%s,%s', 11, 22, '33')
logger.INFO('xxxxxxxxxxxxxxxxxxxxxxxxx222:%s,%s,%s', 11, 22, '33')
logger.WARN('xxxxxxxxxxxxxxxxxxxxxxxxx222:%s,%s,%s', 11, 22, '33')
logger.ERR('xxxxxxxxxxxxxxxxxxxxxxxxx222:%s,%s,%s', 11, 22, '33')
logger.WARN('english 中文 XX YY 混杂')
# import hqpy.console
from hqpy import console
console.red("sssssssssssssss")
console.magenta("sssssssssssssss")
console.cyan("sssssssssssssss")
console.red("sssssssssssssss")
import logging
progress = console.ColoramaConsoleHandler()
co = logging.getLogger('test')
co.setLevel(logging.DEBUG)
ch_format = logging.Formatter(
'%(asctime)s %(filename)s:%(lineno)d [%(levelname)-7s] %(message)s')
progress.setFormatter(ch_format)
progress.setTaskMode()
co.addHandler(progress)
co.info('test1')
co.debug('test1ddd')
co.info('test2')
co.warn("wwwwwwwwwwwwwwwww")
co.error('test1ddd')
def do(self):
ret = hqpy.HqError()
tmo = None
if self.conf.has_key('timeout'):
tmo = self.conf['timeout']
sau = self.get_param().get_auth()
jobid = self.get_param().get_jobid()
if self.conf.has_key('cmd') == True:
shcmd = self.conf['cmd']
logger.DEBUG('%s run ssh cmd: %s', self.info(), shcmd)
ret, ostd, oerr = hqpy.sshRun(sau, shcmd, tmo)
logger.INFO('%s ssh cmd result: %s', self.info(), ret.string())
if ret.iserr():
logger.INFO(
'========================================================std:\n%s',
ostd)
logger.ERR(
'========================================================err:\n%s',
oerr)
return ret
else:
logger.INFO(
'========================================================std:\n%s',
ostd)
logger.INFO(
'========================================================err:\n%s',
oerr)
if self.conf.has_key('script') == True:
script_file = self.conf['script']
if os.path.isabs(script_file) == False:
bhas, val_path = self.get_arg('path')
if bhas:
script_file = os.path.join(val_path, script_file)
else:
script_file = os.path.join(
hqenv.get_var('HQVAR_SCRIPT_DIR'), script_file)
fname = string.split(script_file)[0]
bname = os.path.basename(fname)
shbin = "sh"
if bname.endswith('.py'):
shbin = 'python'
elif bname.endswith('.lua'):
shbin = 'lua'
shcmd = '%s %s' % (shbin, script_file)
ret, ostd, oerr = hqpy.sshRun(sau, shcmd, tmo)
if ret.iserr():
logger.ERR('%s run ssh script error: %s', self.info(),
ret.code())
logger.INFO(
'========================================================std:\n%s',
ostd)
logger.ERR(
'========================================================err:\n%s',
oerr)
return ret
else:
logger.INFO('%s run ssh script ok: %s', self.info(),
ret.code())
logger.INFO(
'========================================================std:\n%s',
ostd)
logger.INFO(
'========================================================err:\n%s',
oerr)
return ret
def signal_get_timeout(signum, frame):
logger.ERR('Sftp Get Timeout:%s', frame)
raise Exception("Sftp Get Timed out")
def do(self):
ret = hqpy.HqError()
script_name = self.conf['script']
bname = os.path.basename(script_name)
# script_name 相对于hat/script 路径名
local_file = os.path.join(hqenv.get_var('HQVAR_SCRIPT_DIR'),
script_name)
# 远端置于 /tmp/hatjobs/
remote_path = '/tmp/hat'
sau = self.get_param().get_auth()
jobid = self.get_param().get_jobid()
remote_file = os.path.join(remote_path, '%s_%s' % (jobid, bname))
shbin = "sh"
if bname.endswith('.py'):
shbin = 'python'
elif bname.endswith('.lua'):
shbin = 'lua'
tmo = None
if self.conf.has_key('timeout'):
tmo = self.conf['timeout']
#先确保远端目录建立
ret, ostd, oerr = hqpy.sshRun(sau, 'mkdir -p %s' % (remote_path), 15)
if ret.iserr():
logger.ERR('TaskPRG dir prepare failed:%s, std:%s, err:%s',
ret.string(), ostd, oerr)
return ret
# 推脚本
ret = hqpy.sftpPut(sau, local_file, remote_file, 30)
if ret.iserr():
logger.ERR('TaskPRG put script failed:%s, std:%s, err:%s',
ret.string(), ostd, oerr)
return ret
# 运行脚本
ret, ostd, oerr = hqpy.sshRun(sau, '%s %s' % (shbin, remote_file), tmo)
if ret.iserr():
logger.ERR('TaskPRG dir prepare failed:%s, std:%s, err:%s',
ret.string(), ostd, oerr)
return ret
# 获取结果
if self.conf.has_key('results'):
ress = self.conf['results']
local_tmp = hqenv.get_var('HQVAR_VAR_DIR')
for r in ress:
r_file = os.path.join(remote_path, r)
l_file = os.path.join(local_tmp,
'%s_%s_%s' % (jobid, sau.host, r))
ret = hqpy.sftpGet(sau, r_file, l_file)
if ret.iserr():
logger.ERR('TaskPRG get result %s failed:%s', r,
ret.string())
return ret
else:
logger.INFO('TaskPRG result get ok:%s', l_file)
## 任务结束
return ret
def init_hosts():
#logger.DEBUG('host cfg load start.')
fd = None
hret = hqpy.HqError()
try:
cfg_file = os.path.join(hqenv.get_var('HQVAR_CONF_DIR'), 'hosts.json')
fd = open(cfg_file)
jscfg = json.load(fd)
def_auth = jscfg['default']
default_user = def_auth['user']
default_passwd = def_auth['pwd']
default_root_user = def_auth['root_user']
default_root_passwd = def_auth['root_pwd']
hosts = jscfg['hosts']
for h in hosts:
hid = h['id']
ip = h['ip']
kind = h['kind']
port = 22
if h.has_key('port'):
port = h['port']
user = default_user
pwd = default_passwd
if h.has_key('user'):
user = h['user']
if h.has_key('pwd'):
pwd = h['pwd']
ruser = default_root_user
rpwd = default_root_passwd
if h.has_key('ruser'):
ruser = h['ruser']
if h.has_key('rpwd'):
ruser = h['rpwd']
hauth = HostAuthObj()
user_auth = hqpy.SshAuth(ip, user, pwd, port)
root_auth = hqpy.SshAuth(ip, ruser, rpwd, port)
hauth.user = user_auth
hauth.root = root_auth
hauth.id = hid
hauth.ip = ip
hauth.port = port
hauth.kind = kind
if __hosts_by_id.has_key(hid):
return hqpy.Err("load host, exist hid:%s" % (hid))
__hosts_by_id[hid] = hauth
for g in kind:
if __hosts_by_group.has_key(g) == False:
__hosts_by_group[g] = []
__deploy_by_kind[g] = []
__hosts_by_group[g].append(hauth)
__deploy_by_kind[g].append(hid)
__deploy_by_id[hid] = {}
if h.has_key('zone'):
v = h['zone']
for x in v:
rid = x[0]
rseq = x[1]
rname = 'zone' + str(rid)
if __deploy_by_role.has_key(rname):
hret = hqpy.Err("load host, exist zone name:%s" % (rname) )
return hret
__deploy_by_role[rname] = {'sau':hauth, 'info':h, 'role':'zone', 'roles':[[rid, rseq]] }
__deploy_by_id[hid]['zone'] = {'sau':hauth, 'info':h, 'role':'zone', 'roles':h['zone'] }
if h.has_key('group'):
v = h['group']
for x in v:
rid = x[0]
rseq = x[1]
rname = 'group' + str(rid)
if __deploy_by_role.has_key(rname):
hret = hqpy.Err("load host, exist group name:%s" % (rname))
return hret
__deploy_by_role[rname] = {'sau':hauth, 'info':h, 'role':'group', 'roles':[[rid, rseq]] }
__deploy_by_id[hid]['group'] = {'sau':hauth, 'info':h, 'role':'group', 'roles':h['group'] }
if h.has_key('global'):
v = h['global']
for x in v:
rid = x[0]
rseq = x[1]
rname = 'global' + str(rid)
if __deploy_by_role.has_key(rname):
hret = hqpy.Err("load host, exist global name:%s" % (rname) )
return hret
__deploy_by_role[rname] = {'sau':hauth, 'info':h, 'role':'global', 'roles':[[rid, rseq]] }
__deploy_by_id[hid]['global'] = {'sau':hauth, 'info':h, 'role':'global', 'roles':h['global'] }
if __deploy_by_id.has_key('hid'):
hret = hqpy.Err("load host, exist hostid :%s" % (hid))
return hret
#print __hosts_by_id
#print __hosts_by_group
#print __deploy_by_id
#print __deploy_by_role
#logger.DEBUG('host cfg load done.')
except Exception,ex:
template = "An exception of type {0} occured. Arguments:{1!r}"
message = template.format(type(ex).__name__, ex.args)
hret.errno = hqpy.PYRET_TASK_LOAD_ERR
hret.msg = 'TASK Error:{0}'.format(message)
logger.ERR(traceback.format_exc())
def do(self):
tmo = None
if self.conf.has_key('timeout'):
tmo = self.conf['timeout']
ret = hqpy.HqError()
if self.conf.has_key('cmd') == True:
shcmd = self.conf['cmd']
logger.DEBUG('%s run cmd: %s', self.info(), shcmd)
ret, ostd, oerr = hqpy.run_shell(shcmd, tmo)
if ret.iserr():
logger.ERR('%s run shell error: %s', self.info(), ret.code())
logger.INFO(
'========================================================std:\n%s',
ostd)
logger.ERR(
'========================================================err:\n%s',
oerr)
return ret
else:
logger.INFO('%s run shell ok: %s', self.info(), ret.code())
logger.INFO(
'========================================================std:\n%s',
ostd)
logger.ERR(
'========================================================err:\n%s',
oerr)
if self.conf.has_key('script') == True:
script_file = self.conf['script']
if os.path.isabs(script_file) == False:
bhas, val_path = self.get_arg('path')
if bhas:
script_file = os.path.join(val_path, script_file)
else:
script_file = os.path.join(
hqenv.get_var('HQVAR_SCRIPT_DIR'), script_file)
bname = os.path.basename(script_file)
shbin = "sh"
if bname.endswith('.py'):
shbin = 'python'
elif bname.endswith('.lua'):
shbin = 'lua'
shcmd = '%s %s' % (shbin, script_file)
ret, ostd, oerr = hqpy.run_shell(shcmd, tmo)
if ret.iserr():
logger.ERR('%s run script error: %s', self.info(), ret.code())
logger.INFO(
'========================================================std:\n%s',
ostd)
logger.ERR(
'========================================================err:\n%s',
oerr)
return ret
else:
logger.INFO('%s run script ok: %s', self.info(), ret.code())
logger.INFO(
'========================================================std:\n%s',
ostd)
logger.ERR(
'========================================================err:\n%s',
oerr)
return ret
def run_shell(cmd, tmo=60):
hret, ostd, oerr = hqpy.run_shell(cmd, tmo)
if hret.iserr():
logger.ERR('bash:%s\n%s\n%s', cmd, ostd, oerr)
return hret
def __signal_run_timeout(signum, frame):
logger.ERR('Run Shell Timeout:%s', frame)
raise Exception("Run Shell Timeout")
