def init(module="hat",
load_host=False,
jid=None,
jname=None,
show_console=True,
tm_tag=True):
from hqpy import hqenv
from hqpy import logger
from hqpy import hqhosts
from hqpy import hqio
if os.environ.has_key('HATROOT'):
hat_work_dir = os.environ['HATROOT']
else:
hat_work_dir = "/var/tmp/hat"
hqio.mkdirs(hat_work_dir)
hqenv.set_var('HQVAR_WORK_DIR', hat_work_dir)
hqenv.set_var('HQVAR_LOG_DIR', os.path.join(hat_work_dir, 'log'))
hqenv.set_var('HQVAR_SRC_DIR', os.path.join(hat_work_dir, 'src'))
hqenv.set_var('HQVAR_BIN_DIR', os.path.join(hat_work_dir, 'bin'))
hqenv.set_var('HQVAR_CONF_DIR', os.path.join(hat_work_dir, 'conf'))
hqenv.set_var('HQVAR_VAR_DIR', os.path.join(hat_work_dir, 'var'))
hqenv.set_var('HQVAR_RLS_DIR', os.path.join(hat_work_dir, 'files/release'))
hqenv.set_var('HQVAR_FILES_DIR', os.path.join(hat_work_dir, 'files'))
hqenv.set_var('HQVAR_SCRIPT_DIR', os.path.join(hat_work_dir, 'script'))
hqenv.set_var('HQVAR_HGAME_DIR',
os.path.abspath(os.path.join(hat_work_dir, "..")))
hqio.mkdirs(hqenv.get_var('HQVAR_LOG_DIR'))
if jid is not None:
#job mode
hqenv.set_var('HQVAR_JOBID', jid)
logger.init_job_logger(hqenv.get_var('HQVAR_LOG_DIR'), jid)
else:
#normal mode
hqenv.set_var('HQVAR_JOBID', 0)
logger.init_normal_logger(hqenv.get_var('HQVAR_LOG_DIR'), module,
show_console, tm_tag)
# load env.json
fini = os.path.join(hqenv.get_var('HQVAR_CONF_DIR'), 'env.ini')
hret, envcfg = hqpy.loadini(fini)
if hret.iserr():
logger.ERR("load env.json failed:%s", hret.string())
return False
else:
hqenv.set_var('HQVAR_ENV_CFG', envcfg)
# load host cfg
if load_host == True:
hret = hqhosts.init_hosts()
if hret.iserr():
logger.ERR('hqhosts init failed:%s', hret.string())
return False
#logger.DEBUG('ALL HQPY VAR:%s', hqenv.get_all())
return True
def TestJob():
from hqpy import hqjob
job_file = os.path.join(hqenv.get_var('HQVAR_FILES_DIR'),
'jobs/test_job.json')
ret, job = hqjob.load_job(job_file, None)
ret = job.do()
print ret.string()
def do(self):
ret = hqpy.HqError()
l_files = self.conf['files']
sau = self.get_param().get_auth()
jobid = self.get_param().get_jobid()
tmo = None
if self.conf.has_key('timeout'):
tmo = self.conf['timeout']
for x in self.conf['files']:
local_file = os.path.join(hqenv.get_var('HQVAR_WORK_DIR'), x)
bname = os.path.basename(x)
remote_file = os.path.join(self.conf['path'], bname)
ret = hqpy.sftpPut(sau, local_file, remote_file, tmo)
if ret.iserr():
logger.ERR('%s put file failed:%s', self.info(), x)
return ret
return ret
def do(self):
ret = hqpy.HqError()
r_files = self.conf['files']
sau = self.get_param().get_auth()
jobid = self.get_param().get_jobid()
tmo = None
if self.conf.has_key('timeout'):
tmo = self.conf['timeout']
local_tmp = hqenv.get_var('HQVAR_VAR_DIR')
for r in r_files:
l_file = os.path.join(local_tmp, '%s_%s_%s' % (jobid, sau.host, r))
ret = hqpy.sftpGet(sau, r, l_file, tmo)
if ret.iserr():
logger.ERR('TaskGet get file %s failed:%s', r, ret.string())
return ret
else:
logger.INFO('TaskGet ok:%s', l_file)
return ret
def init_hosts():
#logger.DEBUG('host cfg load start.')
fd = None
hret = hqpy.HqError()
try:
cfg_file = os.path.join(hqenv.get_var('HQVAR_CONF_DIR'), 'hosts.json')
fd = open(cfg_file)
jscfg = json.load(fd)
def_auth = jscfg['default']
default_user = def_auth['user']
default_passwd = def_auth['pwd']
default_root_user = def_auth['root_user']
default_root_passwd = def_auth['root_pwd']
hosts = jscfg['hosts']
for h in hosts:
hid = h['id']
ip = h['ip']
kind = h['kind']
port = 22
if h.has_key('port'):
port = h['port']
user = default_user
pwd = default_passwd
if h.has_key('user'):
user = h['user']
if h.has_key('pwd'):
pwd = h['pwd']
ruser = default_root_user
rpwd = default_root_passwd
if h.has_key('ruser'):
ruser = h['ruser']
if h.has_key('rpwd'):
ruser = h['rpwd']
hauth = HostAuthObj()
user_auth = hqpy.SshAuth(ip, user, pwd, port)
root_auth = hqpy.SshAuth(ip, ruser, rpwd, port)
hauth.user = user_auth
hauth.root = root_auth
hauth.id = hid
hauth.ip = ip
hauth.port = port
hauth.kind = kind
if __hosts_by_id.has_key(hid):
return hqpy.Err("load host, exist hid:%s" % (hid))
__hosts_by_id[hid] = hauth
for g in kind:
if __hosts_by_group.has_key(g) == False:
__hosts_by_group[g] = []
__deploy_by_kind[g] = []
__hosts_by_group[g].append(hauth)
__deploy_by_kind[g].append(hid)
__deploy_by_id[hid] = {}
if h.has_key('zone'):
v = h['zone']
for x in v:
rid = x[0]
rseq = x[1]
rname = 'zone' + str(rid)
if __deploy_by_role.has_key(rname):
hret = hqpy.Err("load host, exist zone name:%s" % (rname) )
return hret
__deploy_by_role[rname] = {'sau':hauth, 'info':h, 'role':'zone', 'roles':[[rid, rseq]] }
__deploy_by_id[hid]['zone'] = {'sau':hauth, 'info':h, 'role':'zone', 'roles':h['zone'] }
if h.has_key('group'):
v = h['group']
for x in v:
rid = x[0]
rseq = x[1]
rname = 'group' + str(rid)
if __deploy_by_role.has_key(rname):
hret = hqpy.Err("load host, exist group name:%s" % (rname))
return hret
__deploy_by_role[rname] = {'sau':hauth, 'info':h, 'role':'group', 'roles':[[rid, rseq]] }
__deploy_by_id[hid]['group'] = {'sau':hauth, 'info':h, 'role':'group', 'roles':h['group'] }
if h.has_key('global'):
v = h['global']
for x in v:
rid = x[0]
rseq = x[1]
rname = 'global' + str(rid)
if __deploy_by_role.has_key(rname):
hret = hqpy.Err("load host, exist global name:%s" % (rname) )
return hret
__deploy_by_role[rname] = {'sau':hauth, 'info':h, 'role':'global', 'roles':[[rid, rseq]] }
__deploy_by_id[hid]['global'] = {'sau':hauth, 'info':h, 'role':'global', 'roles':h['global'] }
if __deploy_by_id.has_key('hid'):
hret = hqpy.Err("load host, exist hostid :%s" % (hid))
return hret
#print __hosts_by_id
#print __hosts_by_group
#print __deploy_by_id
#print __deploy_by_role
#logger.DEBUG('host cfg load done.')
except Exception,ex:
template = "An exception of type {0} occured. Arguments:{1!r}"
message = template.format(type(ex).__name__, ex.args)
hret.errno = hqpy.PYRET_TASK_LOAD_ERR
hret.msg = 'TASK Error:{0}'.format(message)
logger.ERR(traceback.format_exc())
def do(self):
ret = hqpy.HqError()
script_name = self.conf['script']
bname = os.path.basename(script_name)
# script_name 相对于hat/script 路径名
local_file = os.path.join(hqenv.get_var('HQVAR_SCRIPT_DIR'),
script_name)
# 远端置于 /tmp/hatjobs/
remote_path = '/tmp/hat'
sau = self.get_param().get_auth()
jobid = self.get_param().get_jobid()
remote_file = os.path.join(remote_path, '%s_%s' % (jobid, bname))
shbin = "sh"
if bname.endswith('.py'):
shbin = 'python'
elif bname.endswith('.lua'):
shbin = 'lua'
tmo = None
if self.conf.has_key('timeout'):
tmo = self.conf['timeout']
#先确保远端目录建立
ret, ostd, oerr = hqpy.sshRun(sau, 'mkdir -p %s' % (remote_path), 15)
if ret.iserr():
logger.ERR('TaskPRG dir prepare failed:%s, std:%s, err:%s',
ret.string(), ostd, oerr)
return ret
# 推脚本
ret = hqpy.sftpPut(sau, local_file, remote_file, 30)
if ret.iserr():
logger.ERR('TaskPRG put script failed:%s, std:%s, err:%s',
ret.string(), ostd, oerr)
return ret
# 运行脚本
ret, ostd, oerr = hqpy.sshRun(sau, '%s %s' % (shbin, remote_file), tmo)
if ret.iserr():
logger.ERR('TaskPRG dir prepare failed:%s, std:%s, err:%s',
ret.string(), ostd, oerr)
return ret
# 获取结果
if self.conf.has_key('results'):
ress = self.conf['results']
local_tmp = hqenv.get_var('HQVAR_VAR_DIR')
for r in ress:
r_file = os.path.join(remote_path, r)
l_file = os.path.join(local_tmp,
'%s_%s_%s' % (jobid, sau.host, r))
ret = hqpy.sftpGet(sau, r_file, l_file)
if ret.iserr():
logger.ERR('TaskPRG get result %s failed:%s', r,
ret.string())
return ret
else:
logger.INFO('TaskPRG result get ok:%s', l_file)
## 任务结束
return ret
def do(self):
ret = hqpy.HqError()
ms = hqhosts.get_host_byid('master')
if ms is None:
ret.errno = hqpy.PYRET_ERR
ret.ms = 'master svr not found'
return ret
sau = ms.get()
bhas, hat_path = self.get_arg('path')
if bhas == False:
ret.errno = hqpy.PYRET_ERR
ret.ms = 'path arg not found'
return ret
bhas, dst_set = self.get_arg('set')
if bhas == False:
ret.errno = hqpy.PYRET_ERR
ret.ms = 'set arg not found'
return ret
bhas, pkg_version = self.get_arg('version')
if bhas == False:
ret.errno = hqpy.PYRET_ERR
ret.ms = 'version arg not found'
return ret
pkg_name = "hgame.svr.%s.tgz" % (pkg_version)
local_file = os.path.join(hqenv.get_var('HQVAR_VAR_DIR'), pkg_name)
remote_file = os.path.join("/hgame/hat/files/pkg_server", pkg_name)
print remote_file
logger.DEBUG('%s get file:%s from:%s to:%s' %
(self.info(), pkg_name, sau.host, local_file))
ret = hqpy.sftpGet(sau, remote_file, local_file)
if ret.iserr():
#logger.ERR('%s get file failed:%s', self.info(), ret.string())
return ret
## TODO
dst_dir = os.path.join(hat_path, dst_set)
hqio.mkdirs(dst_dir)
tf = tarfile.open(local_file)
logger.DEBUG('%s update all files to %s', self.info(), dst_dir)
tf.extractall(path=dst_dir)
# file list
# else:
# names = tf.getnames()
# paths = []
# fnames = []
# for f in flist:
# if f.endswith("*") == True:
# paths.append(f[:(len(f)-1)])
# for name in names:
# if name in flist:
# fnames.append(name)
# else:
# for p in paths:
# if name.startswith(p) == True:
# fnames.append(name)
# logger.DEBUG('%s update files to %s:\n%s', self.info(), dpath, fnames)
# for f in fnames:
# tf.extract(f, path=dpath)
return hqpy.HqError()
def do(self):
tmo = None
if self.conf.has_key('timeout'):
tmo = self.conf['timeout']
ret = hqpy.HqError()
if self.conf.has_key('cmd') == True:
shcmd = self.conf['cmd']
logger.DEBUG('%s run cmd: %s', self.info(), shcmd)
ret, ostd, oerr = hqpy.run_shell(shcmd, tmo)
if ret.iserr():
logger.ERR('%s run shell error: %s', self.info(), ret.code())
logger.INFO(
'========================================================std:\n%s',
ostd)
logger.ERR(
'========================================================err:\n%s',
oerr)
return ret
else:
logger.INFO('%s run shell ok: %s', self.info(), ret.code())
logger.INFO(
'========================================================std:\n%s',
ostd)
logger.ERR(
'========================================================err:\n%s',
oerr)
if self.conf.has_key('script') == True:
script_file = self.conf['script']
if os.path.isabs(script_file) == False:
bhas, val_path = self.get_arg('path')
if bhas:
script_file = os.path.join(val_path, script_file)
else:
script_file = os.path.join(
hqenv.get_var('HQVAR_SCRIPT_DIR'), script_file)
bname = os.path.basename(script_file)
shbin = "sh"
if bname.endswith('.py'):
shbin = 'python'
elif bname.endswith('.lua'):
shbin = 'lua'
shcmd = '%s %s' % (shbin, script_file)
ret, ostd, oerr = hqpy.run_shell(shcmd, tmo)
if ret.iserr():
logger.ERR('%s run script error: %s', self.info(), ret.code())
logger.INFO(
'========================================================std:\n%s',
ostd)
logger.ERR(
'========================================================err:\n%s',
oerr)
return ret
else:
logger.INFO('%s run script ok: %s', self.info(), ret.code())
logger.INFO(
'========================================================std:\n%s',
ostd)
logger.ERR(
'========================================================err:\n%s',
oerr)
return ret
def do(self):
ret = hqpy.HqError()
tmo = None
if self.conf.has_key('timeout'):
tmo = self.conf['timeout']
sau = self.get_param().get_auth()
jobid = self.get_param().get_jobid()
if self.conf.has_key('cmd') == True:
shcmd = self.conf['cmd']
logger.DEBUG('%s run ssh cmd: %s', self.info(), shcmd)
ret, ostd, oerr = hqpy.sshRun(sau, shcmd, tmo)
logger.INFO('%s ssh cmd result: %s', self.info(), ret.string())
if ret.iserr():
logger.INFO(
'========================================================std:\n%s',
ostd)
logger.ERR(
'========================================================err:\n%s',
oerr)
return ret
else:
logger.INFO(
'========================================================std:\n%s',
ostd)
logger.INFO(
'========================================================err:\n%s',
oerr)
if self.conf.has_key('script') == True:
script_file = self.conf['script']
if os.path.isabs(script_file) == False:
bhas, val_path = self.get_arg('path')
if bhas:
script_file = os.path.join(val_path, script_file)
else:
script_file = os.path.join(
hqenv.get_var('HQVAR_SCRIPT_DIR'), script_file)
fname = string.split(script_file)[0]
bname = os.path.basename(fname)
shbin = "sh"
if bname.endswith('.py'):
shbin = 'python'
elif bname.endswith('.lua'):
shbin = 'lua'
shcmd = '%s %s' % (shbin, script_file)
ret, ostd, oerr = hqpy.sshRun(sau, shcmd, tmo)
if ret.iserr():
logger.ERR('%s run ssh script error: %s', self.info(),
ret.code())
logger.INFO(
'========================================================std:\n%s',
ostd)
logger.ERR(
'========================================================err:\n%s',
oerr)
return ret
else:
logger.INFO('%s run ssh script ok: %s', self.info(),
ret.code())
logger.INFO(
'========================================================std:\n%s',
ostd)
logger.INFO(
'========================================================err:\n%s',
oerr)
return ret