def prepare_data(self, top_suite_name):
for i in range(4):
cases = list()
path = top_suite_name + '/' + self.dir_rule[i]
start_time = time.time()
if not os.path.isfile(path + '/' + self.shell_cmd[i]):
print("case %s skipped." % self.shell_cmd[i])
case = junit.TestCase(self.shell_cmd[i].split('.')[0])
else:
command = "bash %s" % self.shell_cmd[i]
(cmd_result,
timedelta) = command_shell.Command.run(command.split(),
cwd=path)
print("command %s.\ntime %s. result %s" %
(self.shell_cmd[i], timedelta, cmd_result.returncode))
status = "failed"
if cmd_result.returncode == 0:
status = "success"
cmd_result.show(False)
case = junit.TestCase(self.shell_cmd[i].split('.')[0], status,
timedelta, cmd_result.stdout,
cmd_result.stderr)
end_time = time.time()
test_suites = junit.TestSuites(top_suite_name, self.dir_rule[i],
end_time - start_time)
cases.append(case)
self.generate_reports(test_suites, cases)
def _data_init(self, params):
cases = list()
local_dir = None
hdfs_dir = None
if not params:
return
for param in params:
if param.has_key('local_dir'):
local_dir = param['local_dir']
if param.has_key('hdfs_dir'):
hdfs_dir = param['hdfs_dir']
command_workflow = list()
command = dict(name="remove_hdfs", cmd="hdfs dfs -rmr %s" % hdfs_dir)
command_workflow.append(command)
command = dict(name='mkdir_cmd',
cmd="hdfs dfs -mkdir -p %s" % hdfs_dir)
command_workflow.append(command)
command = dict(name='chmod_cmd',
cmd="hdfs dfs -chmod -R 777 %s" % hdfs_dir)
command_workflow.append(command)
command = dict(name='put_cmd',
cmd="hdfs dfs -put %s %s" % (local_dir, hdfs_dir))
command_workflow.append(command)
command = dict(name='chmod_cmd2',
cmd="hdfs dfs -chmod -R 777 %s" % hdfs_dir)
command_workflow.append(command)
path = os.path.join(self.workdir, self.cur_suite, self.phasename[0])
start_time = time.time()
for command in command_workflow:
print("DataInit: " + command['cmd'])
cmdresult = command_shell.Command.run(command['cmd'].split(),
cwd=path)
case = self.run_case(command['name'], cmdresult)
cases.append(case)
end_time = time.time()
test_suites = junit.TestSuites(self.cur_suite, 'data_init',
end_time - start_time)
self.generate_reports(test_suites, cases)
def _do_transwarp_test(self, path, suite_name):
prefix_command = "transwarp -t -h %s -database %s -f" % (self.hostname,
self.database)
cases = list()
start_time = time.time()
for root, dirs, files in os.walk(path):
for file in files:
if not re.match(".*\.sql$", file):
continue
command = "%s %s" % (prefix_command, file)
(cmd_result,
timedelta) = command_shell.Command.run(command.split(),
cwd=root)
casename = file.split('.')[0]
status = "failed"
if cmd_result.returncode == 0:
status = "success"
file_content = None
with open(os.path.join(root, file), 'r') as f:
file_content = f.read()
cmd_result.show(False)
case = junit.TestCase(casename, status, timedelta,
cmd_result.stdout, cmd_result.stderr,
file_content)
cases.append(case)
print("command %s.\ntime %s. result %s" %
(command, timedelta, cmd_result.returncode))
print("Start Compare Results %s" % command)
#ToDo: diff with 05_ans
answer_file = os.path.join(path, "../05_answer/%s.out" % file)
sort_and_diff.SortDiff.sort_differ_buffers(
cmd_result.stdout, answer_file)
end_time = time.time()
test_suites = junit.TestSuites(suite_name, "query_execsql",
end_time - start_time)
self.generate_reports(test_suites, cases)
return cases
class TranswarpTest(GenerateTest):
def __init__(self):
super(TranswarpTest, self).__init__()
self.phasename = [
"00_data_init", "01_hyperbase_ddl", "02_inceptor_ddl",
"03_inceptor_insert", "04_query", "05_answer"
]
self.transwarp_suites = list()
self.jdbc_suites = list()
self.workdir = os.getcwd()
self.cur_suite = ""
def _parse_conf(self):
self.transwarp_suites = list()
self.jdbc_suites = list()
for suites_conf in self.conf:
if not suites_conf.has_key("transwarp_suites"):
continue
if suites_conf.has_key("hostname"):
self.hostname = suites_conf['hostname']
if suites_conf.has_key("database"):
self.database = suites_conf['database']
if suites_conf.has_key("transwarp_suites"):
self.transwarp_suites = suites_conf["transwarp_suites"]
if suites_conf.has_key("rootdir"):
self.workdir = os.path.join(suites_conf["rootdir"])
print("WORKDIR %s" % self.workdir)
if suites_conf.has_key("jdbc_suites"):
self.jdbc_suites = suites_conf["jdbc_suites"]
def _data_init(self, params):
cases = list()
local_dir = None
hdfs_dir = None
if not params:
return
for param in params:
if param.has_key('local_dir'):
local_dir = param['local_dir']
if param.has_key('hdfs_dir'):
hdfs_dir = param['hdfs_dir']
command_workflow = list()
command = dict(name="remove_hdfs", cmd="hdfs dfs -rmr %s" % hdfs_dir)
command_workflow.append(command)
command = dict(name='mkdir_cmd',
cmd="hdfs dfs -mkdir -p %s" % hdfs_dir)
command_workflow.append(command)
command = dict(name='chmod_cmd',
cmd="hdfs dfs -chmod -R 777 %s" % hdfs_dir)
command_workflow.append(command)
command = dict(name='put_cmd',
cmd="hdfs dfs -put %s %s" % (local_dir, hdfs_dir))
command_workflow.append(command)
command = dict(name='chmod_cmd2',
cmd="hdfs dfs -chmod -R 777 %s" % hdfs_dir)
command_workflow.append(command)
path = os.path.join(self.workdir, self.cur_suite, self.phasename[0])
start_time = time.time()
for command in command_workflow:
print("DataInit: " + command['cmd'])
cmdresult = command_shell.Command.run(command['cmd'].split(),
cwd=path)
case = self.run_case(command['name'], cmdresult)
cases.append(case)
end_time = time.time()
test_suites = junit.TestSuites(self.cur_suite, 'data_init',
end_time - start_time)
self.generate_reports(test_suites, cases)
def _hyperbase_ddl(self, params):
cases = list()
prefix_command = "hbase shell"
sql_files = [
"initialize_hbase.sql",
]
if params:
print(params)
start_time = time.time()
for sql_file in sql_files:
path = os.path.join(self.workdir, self.cur_suite,
self.phasename[1])
if not os.path.isfile(os.path.join(path, sql_file)):
case = self.run_case(sql_file.split('.')[0])
cases.append(case)
continue
#ToDo: Run command interative
command = "bash %s %s" % (os.path.join(
os.getcwd(), self.workdir,
"runner/hbase/initialize_hyperbase.sh"), sql_file)
print("HBase DDL: " + command)
cmdresult = command_shell.Command.run(command.split(), cwd=path)
case = self.run_case(sql_file.split('.')[0], cmdresult)
cases.append(case)
end_time = time.time()
test_suites = junit.TestSuites(self.cur_suite, 'hyperbase_ddl',
end_time - start_time)
self.generate_reports(test_suites, cases)
start_time = time.time()
for sql_file in sql_files:
path = os.path.join(self.workdir, self.cur_suite,
self.phasename[2])
if not os.path.isfile(os.path.join(path, sql_file)):
case = self.run_case(sql_file.split('.')[0])
cases.append(case)
continue
command = "%s %s" % (prefix_command, sql_file)
print("InceptorDDL: " + command)
cmdresult = command_shell.Command.run(command.split(), cwd=path)
case = self.run_case(sql_file.split('.')[0], cmdresult)
cases.append(case)
end_time = time.time()
test_suites = junit.TestSuites(self.cur_suite, 'inceptor_ddl',
end_time - start_time)
self.generate_reports(test_suites, cases)
def _inceptor_insert(self, params):
cases = list()
prefix_command = "transwarp -t -h %s -f" % self.hostname
sql_files = [
"insert.sql",
]
if params:
print(params)
start_time = time.time()
for sql_file in sql_files:
path = os.path.join(self.workdir, self.cur_suite,