def _common_check(self, autoidentity, retStatus, logcheck):
"""
:return:
"""
# common check
# 1, pid exist
cmd = util.get('CMD_TABLE_PS')
proc = Process()
data, error, retCode = proc.run(cmd)
if 0 != retCode:
raise Exception('Tablet server status error.')
# 2, warn/error log check
if retStatus and logcheck:
logTokens = ['E ', 'W ']
logfile = util.get('RTIDB_LOG')
for token in logTokens:
cmd = 'grep -E "%(token)s" %(logfile)s' % locals()
data, error, retCode = proc.run(cmd)
if 0 == retCode:
raise Exception(
'Tablet server logchecker %(token)s: %(data)s' %
locals())
# 3, identify
if autoidentity and not retStatus:
retStatus, retMsg = self.identify(self.input_message(),
self.scanout_message())
if not retStatus:
raise Exception('Identify check error:%(retMsg)s' % locals())
def tearDown(self):
"""
environment clear
:return:
"""
cmd = util.get('CMD_TABLE_STOP')
proc = Process()
proc.run(cmd)
def local_wrapper(contextMap):
"""
:param picoDir:
:return:
"""
cmd = ('%(wrapperBin)s ' '%(appBin)s ' '--config_file=%(yamlFile)s')
cmd = cmd % contextMap
proc = Process()
data, error, retCode = proc.run(cmd, timeout=300)
return cmd, data, error, retCode
def start_tablet_server(workDir):
"""
:param workDir:
:return:
"""
binDir = get('BIN_DIR')
logfile = os.path.join(workDir, 'run.log')
cmd = 'cd %(binDir)s && ./rtidb --role=tablet >%(logfile)s 2>&1 &' % locals()
p = Process()
data, error, retCode = p.run(cmd)
if 0 != retCode:
raise Exception('Start tablet server error: %s' % FileUtils.read(logfile))
def setUp(self):
"""
environment setup
:return:
"""
#build && start tablet
FileUtils.mkdir(util.get('RUNENV_DIR'))
FileUtils.rm(util.get('RTIDB_LOG'))
for cmd in [
'CMD_BUILD', 'CMD_BUILD_JAVA_CLIENT',
'CMD_BUILD_PYTHON_CLIENT', 'CMD_TABLE_STOP', 'CMD_TABLE_START'
]:
log.info(cmd)
cmd = util.get(cmd)
proc = Process()
proc.run(cmd)
def yarn_wrapper(contextMap):
"""
:param picoDir:
:return:
"""
cmd = (
'%(yarnBin)s jar %(hadoopPatchJar)s com.tfp.hadoop.yarn.launcher.Client '
'--appname %(jobName)s '
'--jar %(hadoopPatchJar)s '
'--shell_command "./pico_yarn_runner.sh --enable-hdfs-proxy ./%(appName)s pico.yaml" '
'--container_memory=4000 '
'--num_containers=3 '
'--shell_env HADOOP_USER_NAME=root '
'--shell_env HADOOP_HOME=%(hadoopHome)s '
'--shell_env WEBHDFS_USER=root'
'--shell_env WEBHDFS_HDFS=%(hdfsBin)s '
'--file %(appBin)s '
'--file %(wrapperBin)s '
'--file %(yamlFile)s '
'--file %(hdfsproxyClient)s '
'--file %(picoYarnRunnerSh)s '
'--file %(hdfsProxyJar)s ')
wrapperBin = contextMap['contextMap']
appBin = contextMap['appBin']
yamlFile = contextMap['yamlFile']
appName = os.path.basename(appBin)
jobName = 'PTest_' + appBin + '_'.join(yamlFile.split('/')[-4:-1])
hadoopHome = util.get('HADOOP_HOME')
yarnBin = os.path.join(hadoopHome, 'bin/yarn')
hdfsBin = os.path.join(hadoopHome, 'bin/hdfs')
picoYarnRunnerSh = os.path.join(util.get('TESTLIB_DIR'),
'script/pico_yarn_runner.sh')
hadoopPatchJar = os.path.join(util.get('DEPEND_DIR'),
'hadoop-patch.jar')
hdfsproxyClient = os.path.join(util.get('DEPEND_DIR'),
'hdfsproxy-client')
hdfsProxyJar = os.path.join(util.get('DEPEND_DIR'), 'hdfs-proxy.jar')
cmd = cmd % locals()
proc = Process()
data, error, retCode = proc.run(cmd, timeout=600)
appId = re.findall(r"appId = application_[0-9]+_[0-9]+",
data + error)[0]
appId = appId.split('=')[-1]
cmdLog = 'yarn logs -applicationId %(appId)s' % locals()
data, error, _ = proc.run(cmdLog)
return cmd, data, error, retCode
def _init_prepare(self):
"""
:return:
"""
self.taskList = []
proc = Process()
for cmd in ['CMD_TABLE_STOP', 'CMD_TABLE_START']:
cmd = util.get(cmd)
proc.run(cmd)
self.rtidbClient = RtidbClient(endpoint=util.get('ENDPOINT'))
self.append(self.rtidbClient.drop_table)
self.run(failonerror=False, autoidentity=False)
self.taskList = []
self.pk = ''.join(random.sample(string.lowercase + string.digits, 20))
def mpi_wrapper(contextMap):
"""
:return:
"""
cmd = ('%(mpirunBin)s '
'-np %(np)s '
'--allow-run-as-root '
'%(wrapperBin)s '
'%(appBin)s '
'--config_file=%(yamlFile)s ')
wrapperBin = contextMap['contextMap']
appBin = contextMap['appBin']
yamlFile = contextMap['yamlFile']
np = contextMap['np']
mpirunBin = os.path.join(util.get('TESTLIB_DIR'), 'bin/mpirun')
cmd = cmd % locals()
proc = Process()
return proc.runXP(cmd)