def generate_suite_file(**kwargs):
"""
task for generateing new suite file in Hasal working dir
@param kwargs:
kwargs['cmd_obj']['configs']['DEFAULT_GENERATED_SUITE_FN'] :: the new suite file name you are going to create
kwargs['cmd_obj']['configs']['OVERWRITE_HASAL_SUITE_CASE_LIST'] :: the case list for generating suite ex: tests.regression.gdoc.test_firefox_gdoc_read_basic_txt_1, tests.regression.gdoc.test_firefox_gdoc_read_basic_txt_2
@return:
"""
DEFAULT_GENERATE_SUITE_FN = "ejenti.suite"
DEFAULT_SUITE_FN = "suite.txt"
# get queue msg, consumer config from kwargs
queue_msg, consumer_config, task_config = init_task(kwargs)
# get Hasal working dir path
hasal_working_dir = get_hasal_repo_path(task_config)
output_suite_fn = task_config.get("DEFAULT_GENERATED_SUITE_FN",
DEFAULT_GENERATE_SUITE_FN)
default_suite_fp = os.path.join(hasal_working_dir, DEFAULT_SUITE_FN)
output_suite_fp = os.path.join(hasal_working_dir, output_suite_fn)
case_list_str = task_config.get('OVERWRITE_HASAL_SUITE_CASE_LIST', None)
if case_list_str:
case_list = case_list_str.split(",")
else:
with open(default_suite_fp) as fh:
case_list = fh.readlines()
with open(output_suite_fp, 'w') as write_fh:
for case_path in case_list:
write_fh.write(case_path.strip() + '\n')
return output_suite_fp
def git_pull(**kwargs):
"""
git pull cmd wrapper
@param kwargs:
kwargs['cmd_obj']['configs']['GIT_PULL_PARAMETER_REMOTE_URL'] :: remote url parameter
kwargs['cmd_obj']['configs']['GIT_PULL_PARAMETER_BRANCH_NAME'] :: branch name
@return:
"""
DEFAULT_GIT_CMD_PULL = ["git", "pull"]
# get queue msg, consumer config from kwargs
queue_msg, consumer_config, task_config = init_task(kwargs)
# get default repo path
repo_path = get_hasal_repo_path(task_config)
# get remote url and branch
cmd_parameter_list = parse_cmd_parameters(queue_msg)
if len(cmd_parameter_list) == 2:
remote_url = cmd_parameter_list[1]
branch_name = ""
elif len(cmd_parameter_list) == 3:
remote_url = cmd_parameter_list[1]
branch_name = cmd_parameter_list[2]
else:
remote_url = task_config.get("GIT_PULL_PARAMETER_REMOTE_URL", "")
branch_name = task_config.get("GIT_PULL_PARAMETER_BRANCH_NAME", "")
if remote_url == "" and branch_name == "":
exec_cmd = DEFAULT_GIT_CMD_PULL
elif remote_url != "":
if remote_url.startswith("https://") or remote_url in get_remote_url_list(repo_path):
if branch_name:
exec_cmd = DEFAULT_GIT_CMD_PULL + [remote_url, branch_name]
else:
exec_cmd = DEFAULT_GIT_CMD_PULL + [remote_url]
else:
logging.error("Remote name cannot find in your repo [%s]" % remote_url)
return False
else:
logging.error("Incorrect usage for git pull remote_url:[%s], branch_name:[%s]" % (remote_url, branch_name))
return False
logging.debug("git pull execute cmd [%s]" % exec_cmd)
return_code, output = CommonUtil.subprocess_checkoutput_wrapper(exec_cmd, cwd=repo_path)
if return_code == 0:
logging.info("git pull command execute success [%s]! output [%s]" % (queue_msg['input_cmd_str'], output))
return True
else:
return False
def exec_hasal_runtest(**kwargs):
"""
a wrapper to wrap the runtest cmd
@param kwargs:
kwargs['cmd_obj']['configs']['DEFAULT_RUNTEST_CMD_FMT'] :: runtest cmd format, default: ["python", "runtest.py"]
kwargs['cmd_obj']['configs']['DEFAULT_RUNTEST_OUTPUT_LOG_FN'] :: runtest will redirect output to a physical log file, deafult will be: hasal_runtest.log
kwargs['cmd_obj']['configs']['RUNTEST_CONFIG_PARAMETERS'] :: runtest parameter config, ex: {'--index-config': "configs/index/inputlatencyxxxx.json", "--exec-config": "configs/exec/default.json"}
@return:
"""
DEFAULT_RUNTEST_CMD_FMT = ["python", "runtest.py"]
DEFAULT_JOB_LOG_FN = "hasal_runtest.log"
# get queue msg, consumer config from kwargs
queue_msg, consumer_config, task_config = init_task(kwargs)
# get input cmd parameters
cmd_parameter_list = parse_cmd_parameters(queue_msg)
# get setting from task config
default_cmd_fmt = task_config.get("DEFAULT_RUNTEST_CMD_FMT",
DEFAULT_RUNTEST_CMD_FMT)
default_log_fn = task_config.get("DEFAULT_RUNTEST_OUTPUT_LOG_FN",
DEFAULT_JOB_LOG_FN)
specify_config_settings = task_config.get("RUNTEST_CONFIG_PARAMETERS", {})
workding_dir = get_hasal_repo_path(task_config)
exec_cmd_list = copy.deepcopy(default_cmd_fmt)
if len(cmd_parameter_list) > 1:
exec_cmd_list.extend(cmd_parameter_list[1:])
else:
for config_name in specify_config_settings:
exec_cmd_list.extend(
[config_name, specify_config_settings[config_name]])
exec_cmd_str = " ".join(exec_cmd_list)
system2(exec_cmd_str,
cwd=workding_dir,
logger=default_log_fn,
stdout=True,
exec_env=os.environ.copy())
def git_fetch(**kwargs):
"""
git fetch command wrapper
@param kwargs:
kwargs['cmd_obj']['configs']['GIT_FETCH_PARAMETER_REMOTE_URL'] :: remote url
@return:
"""
DEFAULT_GIT_CMD_FETCH = ["git", "fetch"]
# get queue msg, consumer config from kwargs
queue_msg, consumer_config, task_config = init_task(kwargs)
# get default repo path
repo_path = get_hasal_repo_path(task_config)
# get remote url and branch
cmd_parameter_list = parse_cmd_parameters(queue_msg)
if len(cmd_parameter_list) == 2:
remote_url = cmd_parameter_list[1]
else:
remote_url = task_config.get("GIT_FETCH_PARAMETER_REMOTE_URL", "")
if remote_url == "":
exec_cmd = DEFAULT_GIT_CMD_FETCH
else:
if remote_url.startswith("https://") or remote_url in get_remote_url_list(repo_path):
exec_cmd = DEFAULT_GIT_CMD_FETCH + [remote_url]
else:
logging.error("Remote name cannot find in your repo [%s]" % remote_url)
return False
logging.debug("git fetch execute cmd [%s]" % exec_cmd)
return_code, output = CommonUtil.subprocess_checkoutput_wrapper(exec_cmd, cwd=repo_path)
if return_code == 0:
logging.info("git fetch command execute success [%s]! output [%s]" % (queue_msg['input_cmd_str'], output))
return True
else:
return False
def git_reset(**kwargs):
"""
git reset command wrapper
@param kwargs:
@return:
"""
DEFAULT_GIT_CMD_RESET = ["git", "reset", "--hard", "HEAD"]
# get queue msg, consumer config from kwargs
queue_msg, consumer_config, task_config = init_task(kwargs)
# get default repo path
repo_path = get_hasal_repo_path(task_config)
logging.debug("git reset execute cmd [%s]" % DEFAULT_GIT_CMD_RESET)
return_code, output = CommonUtil.subprocess_checkoutput_wrapper(DEFAULT_GIT_CMD_RESET, cwd=repo_path)
if return_code == 0:
logging.info("git clean command execute success [%s]! output [%s]" % (queue_msg['input_cmd_str'], output))
return True
else:
return False
def git_checkout(**kwargs):
"""
git checkout command wrapper
@param kwargs:
kwargs['cmd_obj']['configs']['GIT_CHECKOUT_PARAMETER_BRANCH_NAME'] :: branch name
@return:
"""
DEFAULT_GIT_CMD_CHECKOUT = ["git", "checkout"]
# get queue msg, consumer config from kwargs
queue_msg, consumer_config, task_config = init_task(kwargs)
# get default repo path
repo_path = get_hasal_repo_path(task_config)
# get remote url and branch
cmd_parameter_list = parse_cmd_parameters(queue_msg)
if len(cmd_parameter_list) == 2:
branch_name = cmd_parameter_list[1]
else:
branch_name = task_config.get("GIT_CHECKOUT_PARAMETER_BRANCH_NAME", "")
if branch_name:
exec_cmd = DEFAULT_GIT_CMD_CHECKOUT + [branch_name]
else:
logging.error("Please specify the checkout branch after cmd or configs")
return False
logging.debug("git checkout execute cmd [%s]" % exec_cmd)
return_code, output = CommonUtil.subprocess_checkoutput_wrapper(exec_cmd, cwd=repo_path)
if return_code == 0:
logging.info("git checkout command execute success [%s]! output [%s]" % (queue_msg['input_cmd_str'], output))
return True
else:
return False
def query_and_remove_hasal_output_folder(task_config,
remove_date=None,
remove=False):
"""
task for querying and removing Hasal output folder
"""
# verify remove date
if remove_date is None:
remove_date = datetime.now().strftime('%Y-%m-%d')
try:
remove_date_datetime = datetime.strptime(remove_date, '%Y-%m-%d')
except:
return 'Remove Date is not correct: {}'.format(remove_date)
# get Hasal working dir path
hasal_working_dir = get_hasal_repo_path(task_config)
output_folder_name = 'output'
image_folder_name = 'images'
image_output_folder_name = 'output'
image_output_folder_path = os.path.join(hasal_working_dir,
output_folder_name,
image_folder_name,
image_output_folder_name)
abs_image_output_folder_path = os.path.abspath(image_output_folder_path)
folder_name_list = os.listdir(abs_image_output_folder_path)
file_obj_list = []
for folder_name in folder_name_list:
sub_output_image_folder_path = os.path.join(
abs_image_output_folder_path, folder_name)
m_timestamp = os.stat(sub_output_image_folder_path).st_mtime
modified_datetime = datetime.fromtimestamp(m_timestamp)
modified_date_str = modified_datetime.strftime('%Y-%m-%d')
if modified_datetime <= remove_date_datetime:
file_obj = {
'path': sub_output_image_folder_path,
'date': modified_date_str,
'ts': m_timestamp
}
file_obj_list.append(file_obj)
if remove:
logging.warn(
'Remove folder: {}'.format(sub_output_image_folder_path))
shutil.rmtree(sub_output_image_folder_path)
if remove:
ret_message = '*Removed Hasal Output before {}*\n'.format(remove_date)
else:
ret_message = '*Query Hasal Output before {}*\n'.format(remove_date)
for file_obj in file_obj_list:
ret_message += '{}, {}\n'.format(file_obj.get('path'),
file_obj.get('date'))
return ret_message
def generate_hasal_config(**kwargs):
"""
generate hasal config jsons for ejenti, default should generate agent/chrome/exec/firefox/global/index/online jsons
@param kwargs: will have two keys queue_msg, consumer_config
kwargs['cmd_obj']['configs']['DEFAULT_HASAL_CONFIG_CONTENT_TEMPLATE'] :: default tempalate will use for generating config content
kwargs['cmd_obj']['configs']['DEFAULT_HASAL_RUNTEST_CMD_PARAMETERS_TEMPLATE'] :: default runtest exec parameters template
kwargs['cmd_obj']['configs']['OVERWIRTE_HASAL_CONFIG_CTNT'] :: the ctnt use for overwrite the current config example as below:
{
"configs": {
"exec": {
"default.json": {
"key1": "value1"
}
},
"firefox": {
"default.json": {
"key2": "value2",
"key3": "value3"
}
},
"online": {
"abc.json":{
"key3": "value3",
"key4": "value4"
}
}
}
}
@return:
"""
DEFAULT_HASAL_CONFIG = {
"configs": {
"exec": {
"default.json": {}
},
"firefox": {
"default.json": {}
},
"chrome": {
"default.json": {}
},
"index": {
"runtimeDctGenerator.json": {}
},
"upload": {
"default.json": {}
},
"global": {
"default.json": {}
}
}
}
DEFAULT_HASAL_RUNTEST_CONFIGS = {
"--exec-config": "",
"--firefox-config": "",
"--index-config": "",
"--upload-config": "",
"--global-config": "",
"--chrome-config": ""
}
# get queue msg, consumer config from kwargs
queue_msg, consumer_config, task_config = init_task(kwargs)
# get override config
cmd_parameter_list = queue_msg.get('input_cmd_str', "").split(" ", 1)
default_config_settings = task_config.get(
"DEFAULT_HASAL_CONFIG_CONTENT_TEMPLATE", DEFAULT_HASAL_CONFIG)
default_runtest_configs = task_config.get(
"DEFAULT_HASAL_RUNTEST_CMD_PARAMETERS_TEMPLATE",
DEFAULT_HASAL_RUNTEST_CONFIGS)
# get input config from user interactive mode
if len(cmd_parameter_list) == 2:
input_json_str = cmd_parameter_list[1]
logging.debug("input cmd parameter : [%s]" % input_json_str)
input_json_obj = CommonUtil.load_json_string(input_json_str)
logging.debug("load json obj from input cmd: [%s]" % input_json_obj)
else:
input_json_obj = task_config.get("OVERWIRTE_HASAL_CONFIG_CTNT", {})
logging.debug("load json obj from input config: [%s]" % input_json_obj)
if len(input_json_obj.keys()) == 0:
logging.info(
"No input config object [%s] detected, will use the default config setting instead"
% input_json_obj)
else:
json_path = get_hasal_repo_path(task_config)
# merge default and input
full_config_obj = merge_user_input_config_with_default_config(
input_json_obj, default_config_settings)
# generate config path and need to modify key-value pair dict
full_config_path_mapping = generate_config_path_json_mapping(
json_path, full_config_obj, {})
full_exec_runtest_config = copy.deepcopy(default_runtest_configs)
# dump to json file
for config_path in full_config_path_mapping:
tmp_json_obj = CommonUtil.load_json_file(config_path)
tmp_json_obj.update(full_config_path_mapping[config_path])
dir_name = os.path.dirname(config_path)
new_config_path = os.path.join(dir_name, "ejenti.json")
parameter_name = "--{}-config".format(dir_name.split(os.sep)[-1])
full_exec_runtest_config[parameter_name] = new_config_path
with open(new_config_path, 'w') as fh:
json.dump(tmp_json_obj, fh)
logging.debug("exec runtest config [%s]" % full_exec_runtest_config)
return full_exec_runtest_config