def load(self, arg):
ret = hqpy.HqError()
self.jname = arg['jname']
self.ptasks = arg['tasks']
self.jargn = 0
if arg.has_key('jargn'):
self.jargn = arg['jargn']
if len(self.jobp.get_args()) < self.jargn:
ret.errno = hqpy.PYRET_TASK_CHECK_ERR
ret.msg = '%s need %s args' % (self.jname, self.jargn)
return ret
idxs = 1
for x in self.ptasks:
x['idx'] = idxs
name = x['name']
ts = None
if name == 'sget':
ts = TaskSGet(x, self)
elif name == 'sput':
ts = TaskSPut(x, self)
elif name == 'yum':
ts = TaskYum(x, self)
elif name == 'cmd':
ts = TaskCmd(x, self)
elif name == 'update':
ts = TaskUpdate(x, self)
elif name == 'prg':
ts = TaskPRG(x, self)
elif name == 'put':
ts = TaskPut(x, self)
elif name == 'get':
ts = TaskGet(x, self)
elif name == 'sshcmd':
ts = TaskSshCmd(x, self)
else:
ts = TaskBase(x, self)
bret, sres = ts.check()
if bret == False:
logger.DEBUG('%s check failed:%s', ts.info(), sres)
ret.errno = hqpy.PYRET_TASK_CHECK_ERR
ret.msg = sres
return ret
self.tlist.append(ts)
idxs = idxs + 1
logger.DEBUG('%s load sucesss', ts.info())
return ret
def test_thread_job(p):
logger.INFO('thread job:%s start', p)
import random
ts = random.randint(1, 5)
time.sleep(ts)
logger.DEBUG('-----------thread job:%s,%s end', p, ts)
return p * p
def do(self):
packs = self.conf['packages']
logger.DEBUG('%s run yum to install packages:%s', self.info(), packs)
cmdstr = 'yum install -y {0}'.format(packs)
ret, ostd, oerr = hqpy.run_shell(cmdstr)
if ret.iserr():
logger.ERR("%s run yum cmd error:%s", self.info(), ret.string())
return ret
def run_make_clean():
global var_work_dir
global var_build_targets
os.chdir(var_work_dir)
for k,p in var_build_targets.items():
logger.DEBUG("<<<<<<<< clean %s", k)
pname = p["name"]
bindir = os.path.join(var_bin_dir, pname)
if os.path.exists(bindir) == True:
os.remove(bindir)
return hqpy.HqError()
def make_func(pcfg):
pname = pcfg["name"]
ppath = pcfg["path"]
logger.DEBUG(">>>>>>>> build %s", pname)
srcdir = ppath
binpath = os.path.join(var_bin_dir, pname)
#print "make project: %s in path:%s" % (pname, srcdir)
cmdstr = "go build -o %s %s" % (binpath, srcdir)
hret = run_shell(cmdstr)
if hret.iserr():
return hqpy.HqError(13, "builder: target[%s] gobuild failed:%s" % (pname, hret.string()) )
return hqpy.HqError()
def tar_zip(src, file_name, flist=None):
cmdstr = ""
if os.path.isfile(src):
cmdstr = "tar cfz %s %s" % (file_name, src)
else:
cmdstr = "cd %s && tar cfz %s * " % (src, file_name)
if flist is not None:
strlist = ' '.join(flist)
cmdstr = "cd %s && tar cfz %s %s " % (src, file_name, strlist)
logger.DEBUG("tar_zip cmd:%s", cmdstr)
hret, ostd, oerr = hqpy.run_shell(cmdstr, show_console=True, timeout=180)
if hret.iserr():
logger.ERR("tar_zip failed:%s,%s,%s \n%s", src, file_name, flist, oerr)
return hret
def run_main():
global var_build_projects
global var_work_dir
hret = hqpy.HqError()
hqpy.init(module="smake")
## set GOPATH
os.environ['GOPATH']=var_work_dir
subprocess.call("echo GOPATH:$GOPATH", shell=True)
# options
from optparse import OptionParser
parser = OptionParser(usage=var_usage)
parser.add_option("-t", "--total", dest="total", default=False, help="make clean; make build", action="store_true")
parser.add_option("-c", "--clean", dest="clean", default=False, help="make clean", action="store_true")
parser.add_option("-b", "--build", dest="build", default=False, help="make build", action="store_true")
(options, args) = parser.parse_args()
if options.total:
options.clean = True
options.build = True
hret = run_set_target(options, args)
hqpy.check_exit(hret, "set target")
logger.DEBUG('hgame maker start.')
print "options: ", options
## clean
if options.clean:
logger.DEBUG("make clean start")
hret = run_make_clean()
hqpy.check_exit(hret, "make clean")
logger.DEBUG("make clean done")
## build
if options.build:
logger.DEBUG("make build start")
hret = run_make_build()
hqpy.check_exit(hret, "make build")
logger.DEBUG("make build done")
##
logger.DEBUG('hgame maker done.')
def TestLogger():
logger.EXIT("xxxxxxxxxxxxxxxxxx")
logger.WARN('test %s %s', 1, 222.2222)
logger.WARN('xxxxxxxxxxxxxxxxxxxxxxxxx')
time.sleep(1)
logger.DEBUG('xxxxxxxxxxxxxxxxxxxxxxxxx222:%s,%s,%s', 11, 22, '33')
logger.INFO('xxxxxxxxxxxxxxxxxxxxxxxxx222:%s,%s,%s', 11, 22, '33')
logger.WARN('xxxxxxxxxxxxxxxxxxxxxxxxx222:%s,%s,%s', 11, 22, '33')
logger.ERR('xxxxxxxxxxxxxxxxxxxxxxxxx222:%s,%s,%s', 11, 22, '33')
logger.WARN('english 中文 XX YY 混杂')
# import hqpy.console
from hqpy import console
console.red("sssssssssssssss")
console.magenta("sssssssssssssss")
console.cyan("sssssssssssssss")
console.red("sssssssssssssss")
import logging
progress = console.ColoramaConsoleHandler()
co = logging.getLogger('test')
co.setLevel(logging.DEBUG)
ch_format = logging.Formatter(
'%(asctime)s %(filename)s:%(lineno)d [%(levelname)-7s] %(message)s')
progress.setFormatter(ch_format)
progress.setTaskMode()
co.addHandler(progress)
co.info('test1')
co.debug('test1ddd')
co.info('test2')
co.warn("wwwwwwwwwwwwwwwww")
co.error('test1ddd')
def get_arg(self, name):
if self.conf.has_key(name) == False:
return False, None
val = self.conf[name]
# if type(val) == unicode:
# val = val.encode("utf-8")
btype = type(val) == str or type(val) == unicode
if btype and val.startswith('JVAR_') == True:
m = re.match('JVAR_(\d+)', val)
if m is None:
logger.ERR('%s invalid job var:%s', self.info(), val)
return False, None
aidx = int(m.group(1))
args = self.get_param().get_args()
if aidx >= len(args):
logger.ERR('%s invalid job var idx:%s', self.info(), val)
return False, None
logger.DEBUG('replace arg:%s,%s', aidx, args[aidx])
return True, args[aidx]
else:
return True, val
def do(self):
logger.DEBUG('hq default task do')
return hqpy.HqError()
def do(self):
ret = hqpy.HqError()
ms = hqhosts.get_host_byid('master')
if ms is None:
ret.errno = hqpy.PYRET_ERR
ret.ms = 'master svr not found'
return ret
sau = ms.get()
bhas, hat_path = self.get_arg('path')
if bhas == False:
ret.errno = hqpy.PYRET_ERR
ret.ms = 'path arg not found'
return ret
bhas, dst_set = self.get_arg('set')
if bhas == False:
ret.errno = hqpy.PYRET_ERR
ret.ms = 'set arg not found'
return ret
bhas, pkg_version = self.get_arg('version')
if bhas == False:
ret.errno = hqpy.PYRET_ERR
ret.ms = 'version arg not found'
return ret
pkg_name = "hgame.svr.%s.tgz" % (pkg_version)
local_file = os.path.join(hqenv.get_var('HQVAR_VAR_DIR'), pkg_name)
remote_file = os.path.join("/hgame/hat/files/pkg_server", pkg_name)
print remote_file
logger.DEBUG('%s get file:%s from:%s to:%s' %
(self.info(), pkg_name, sau.host, local_file))
ret = hqpy.sftpGet(sau, remote_file, local_file)
if ret.iserr():
#logger.ERR('%s get file failed:%s', self.info(), ret.string())
return ret
## TODO
dst_dir = os.path.join(hat_path, dst_set)
hqio.mkdirs(dst_dir)
tf = tarfile.open(local_file)
logger.DEBUG('%s update all files to %s', self.info(), dst_dir)
tf.extractall(path=dst_dir)
# file list
# else:
# names = tf.getnames()
# paths = []
# fnames = []
# for f in flist:
# if f.endswith("*") == True:
# paths.append(f[:(len(f)-1)])
# for name in names:
# if name in flist:
# fnames.append(name)
# else:
# for p in paths:
# if name.startswith(p) == True:
# fnames.append(name)
# logger.DEBUG('%s update files to %s:\n%s', self.info(), dpath, fnames)
# for f in fnames:
# tf.extract(f, path=dpath)
return hqpy.HqError()
def do(self):
tmo = None
if self.conf.has_key('timeout'):
tmo = self.conf['timeout']
ret = hqpy.HqError()
if self.conf.has_key('cmd') == True:
shcmd = self.conf['cmd']
logger.DEBUG('%s run cmd: %s', self.info(), shcmd)
ret, ostd, oerr = hqpy.run_shell(shcmd, tmo)
if ret.iserr():
logger.ERR('%s run shell error: %s', self.info(), ret.code())
logger.INFO(
'========================================================std:\n%s',
ostd)
logger.ERR(
'========================================================err:\n%s',
oerr)
return ret
else:
logger.INFO('%s run shell ok: %s', self.info(), ret.code())
logger.INFO(
'========================================================std:\n%s',
ostd)
logger.ERR(
'========================================================err:\n%s',
oerr)
if self.conf.has_key('script') == True:
script_file = self.conf['script']
if os.path.isabs(script_file) == False:
bhas, val_path = self.get_arg('path')
if bhas:
script_file = os.path.join(val_path, script_file)
else:
script_file = os.path.join(
hqenv.get_var('HQVAR_SCRIPT_DIR'), script_file)
bname = os.path.basename(script_file)
shbin = "sh"
if bname.endswith('.py'):
shbin = 'python'
elif bname.endswith('.lua'):
shbin = 'lua'
shcmd = '%s %s' % (shbin, script_file)
ret, ostd, oerr = hqpy.run_shell(shcmd, tmo)
if ret.iserr():
logger.ERR('%s run script error: %s', self.info(), ret.code())
logger.INFO(
'========================================================std:\n%s',
ostd)
logger.ERR(
'========================================================err:\n%s',
oerr)
return ret
else:
logger.INFO('%s run script ok: %s', self.info(), ret.code())
logger.INFO(
'========================================================std:\n%s',
ostd)
logger.ERR(
'========================================================err:\n%s',
oerr)
return ret
def do(self):
ret = hqpy.HqError()
tmo = None
if self.conf.has_key('timeout'):
tmo = self.conf['timeout']
sau = self.get_param().get_auth()
jobid = self.get_param().get_jobid()
if self.conf.has_key('cmd') == True:
shcmd = self.conf['cmd']
logger.DEBUG('%s run ssh cmd: %s', self.info(), shcmd)
ret, ostd, oerr = hqpy.sshRun(sau, shcmd, tmo)
logger.INFO('%s ssh cmd result: %s', self.info(), ret.string())
if ret.iserr():
logger.INFO(
'========================================================std:\n%s',
ostd)
logger.ERR(
'========================================================err:\n%s',
oerr)
return ret
else:
logger.INFO(
'========================================================std:\n%s',
ostd)
logger.INFO(
'========================================================err:\n%s',
oerr)
if self.conf.has_key('script') == True:
script_file = self.conf['script']
if os.path.isabs(script_file) == False:
bhas, val_path = self.get_arg('path')
if bhas:
script_file = os.path.join(val_path, script_file)
else:
script_file = os.path.join(
hqenv.get_var('HQVAR_SCRIPT_DIR'), script_file)
fname = string.split(script_file)[0]
bname = os.path.basename(fname)
shbin = "sh"
if bname.endswith('.py'):
shbin = 'python'
elif bname.endswith('.lua'):
shbin = 'lua'
shcmd = '%s %s' % (shbin, script_file)
ret, ostd, oerr = hqpy.sshRun(sau, shcmd, tmo)
if ret.iserr():
logger.ERR('%s run ssh script error: %s', self.info(),
ret.code())
logger.INFO(
'========================================================std:\n%s',
ostd)
logger.ERR(
'========================================================err:\n%s',
oerr)
return ret
else:
logger.INFO('%s run ssh script ok: %s', self.info(),
ret.code())
logger.INFO(
'========================================================std:\n%s',
ostd)
logger.INFO(
'========================================================err:\n%s',
oerr)
return ret
