def deploy_fx_package(**kwargs):
"""
deploy firefox package to system default path
@param kwargs:
kwargs['cmd_obj']['configs']['INPUT_FX_DL_PKG_PATH'] :: the firefox package path your want to deploy to system default path
@return:
"""
# get queue msg, consumer config from kwargs
queue_msg, consumer_config, task_config = init_task(kwargs)
# get fx pkg path
cmd_parameter_list = parse_cmd_parameters(queue_msg)
if len(cmd_parameter_list) == 2:
fx_dl_pkg_path = cmd_parameter_list[1]
else:
fx_dl_pkg_path = task_config.get("INPUT_FX_DL_PKG_PATH", None)
if fx_dl_pkg_path:
if extract_fx_pkg(fx_dl_pkg_path):
link_fx_pkg()
return True
else:
logging.error("cannot extract firefox package [%s]" % fx_dl_pkg_path)
else:
logging.warn("please specify firefox downloaded package path after cmd")
return False
def generate_suite_file(**kwargs):
"""
task for generateing new suite file in Hasal working dir
@param kwargs:
kwargs['cmd_obj']['configs']['DEFAULT_GENERATED_SUITE_FN'] :: the new suite file name you are going to create
kwargs['cmd_obj']['configs']['OVERWRITE_HASAL_SUITE_CASE_LIST'] :: the case list for generating suite ex: tests.regression.gdoc.test_firefox_gdoc_read_basic_txt_1, tests.regression.gdoc.test_firefox_gdoc_read_basic_txt_2
@return:
"""
DEFAULT_GENERATE_SUITE_FN = "ejenti.suite"
DEFAULT_SUITE_FN = "suite.txt"
# get queue msg, consumer config from kwargs
queue_msg, consumer_config, task_config = init_task(kwargs)
# get Hasal working dir path
hasal_working_dir = get_hasal_repo_path(task_config)
output_suite_fn = task_config.get("DEFAULT_GENERATED_SUITE_FN",
DEFAULT_GENERATE_SUITE_FN)
default_suite_fp = os.path.join(hasal_working_dir, DEFAULT_SUITE_FN)
output_suite_fp = os.path.join(hasal_working_dir, output_suite_fn)
case_list_str = task_config.get('OVERWRITE_HASAL_SUITE_CASE_LIST', None)
if case_list_str:
case_list = case_list_str.split(",")
else:
with open(default_suite_fp) as fh:
case_list = fh.readlines()
with open(output_suite_fp, 'w') as write_fh:
for case_path in case_list:
write_fh.write(case_path.strip() + '\n')
return output_suite_fp
def git_pull(**kwargs):
"""
git pull cmd wrapper
@param kwargs:
kwargs['cmd_obj']['configs']['GIT_PULL_PARAMETER_REMOTE_URL'] :: remote url parameter
kwargs['cmd_obj']['configs']['GIT_PULL_PARAMETER_BRANCH_NAME'] :: branch name
@return:
"""
DEFAULT_GIT_CMD_PULL = ["git", "pull"]
# get queue msg, consumer config from kwargs
queue_msg, consumer_config, task_config = init_task(kwargs)
# get default repo path
repo_path = get_hasal_repo_path(task_config)
# get remote url and branch
cmd_parameter_list = parse_cmd_parameters(queue_msg)
if len(cmd_parameter_list) == 2:
remote_url = cmd_parameter_list[1]
branch_name = ""
elif len(cmd_parameter_list) == 3:
remote_url = cmd_parameter_list[1]
branch_name = cmd_parameter_list[2]
else:
remote_url = task_config.get("GIT_PULL_PARAMETER_REMOTE_URL", "")
branch_name = task_config.get("GIT_PULL_PARAMETER_BRANCH_NAME", "")
if remote_url == "" and branch_name == "":
exec_cmd = DEFAULT_GIT_CMD_PULL
elif remote_url != "":
if remote_url.startswith("https://") or remote_url in get_remote_url_list(repo_path):
if branch_name:
exec_cmd = DEFAULT_GIT_CMD_PULL + [remote_url, branch_name]
else:
exec_cmd = DEFAULT_GIT_CMD_PULL + [remote_url]
else:
logging.error("Remote name cannot find in your repo [%s]" % remote_url)
return False
else:
logging.error("Incorrect usage for git pull remote_url:[%s], branch_name:[%s]" % (remote_url, branch_name))
return False
logging.debug("git pull execute cmd [%s]" % exec_cmd)
return_code, output = CommonUtil.subprocess_checkoutput_wrapper(exec_cmd, cwd=repo_path)
if return_code == 0:
logging.info("git pull command execute success [%s]! output [%s]" % (queue_msg['input_cmd_str'], output))
return True
else:
return False
def exec_hasal_runtest(**kwargs):
"""
a wrapper to wrap the runtest cmd
@param kwargs:
kwargs['cmd_obj']['configs']['DEFAULT_RUNTEST_CMD_FMT'] :: runtest cmd format, default: ["python", "runtest.py"]
kwargs['cmd_obj']['configs']['DEFAULT_RUNTEST_OUTPUT_LOG_FN'] :: runtest will redirect output to a physical log file, deafult will be: hasal_runtest.log
kwargs['cmd_obj']['configs']['RUNTEST_CONFIG_PARAMETERS'] :: runtest parameter config, ex: {'--index-config': "configs/index/inputlatencyxxxx.json", "--exec-config": "configs/exec/default.json"}
@return:
"""
DEFAULT_RUNTEST_CMD_FMT = ["python", "runtest.py"]
DEFAULT_JOB_LOG_FN = "hasal_runtest.log"
# get queue msg, consumer config from kwargs
queue_msg, consumer_config, task_config = init_task(kwargs)
# get input cmd parameters
cmd_parameter_list = parse_cmd_parameters(queue_msg)
# get setting from task config
default_cmd_fmt = task_config.get("DEFAULT_RUNTEST_CMD_FMT",
DEFAULT_RUNTEST_CMD_FMT)
default_log_fn = task_config.get("DEFAULT_RUNTEST_OUTPUT_LOG_FN",
DEFAULT_JOB_LOG_FN)
specify_config_settings = task_config.get("RUNTEST_CONFIG_PARAMETERS", {})
workding_dir = get_hasal_repo_path(task_config)
exec_cmd_list = copy.deepcopy(default_cmd_fmt)
if len(cmd_parameter_list) > 1:
exec_cmd_list.extend(cmd_parameter_list[1:])
else:
for config_name in specify_config_settings:
exec_cmd_list.extend(
[config_name, specify_config_settings[config_name]])
exec_cmd_str = " ".join(exec_cmd_list)
system2(exec_cmd_str,
cwd=workding_dir,
logger=default_log_fn,
stdout=True,
exec_env=os.environ.copy())
def remove_hasal_output_folder(**kwargs):
"""
task for REMOVE Hasal output folder
"""
key_date = 'date'
# get queue msg, consumer config from kwargs
queue_msg, consumer_config, task_config = init_task(kwargs)
slack_sending_queue = kwargs.get('slack_sending_queue')
remove_date = task_config.get(key_date)
logging.info('REMOVE Hasal ouput folder')
ret_msg = query_and_remove_hasal_output_folder(task_config,
remove_date=remove_date,
remove=True)
msg_obj = task_generate_slack_sending_message(ret_msg)
task_checking_sending_queue(sending_queue=slack_sending_queue)
slack_sending_queue.put(msg_obj)
def git_fetch(**kwargs):
"""
git fetch command wrapper
@param kwargs:
kwargs['cmd_obj']['configs']['GIT_FETCH_PARAMETER_REMOTE_URL'] :: remote url
@return:
"""
DEFAULT_GIT_CMD_FETCH = ["git", "fetch"]
# get queue msg, consumer config from kwargs
queue_msg, consumer_config, task_config = init_task(kwargs)
# get default repo path
repo_path = get_hasal_repo_path(task_config)
# get remote url and branch
cmd_parameter_list = parse_cmd_parameters(queue_msg)
if len(cmd_parameter_list) == 2:
remote_url = cmd_parameter_list[1]
else:
remote_url = task_config.get("GIT_FETCH_PARAMETER_REMOTE_URL", "")
if remote_url == "":
exec_cmd = DEFAULT_GIT_CMD_FETCH
else:
if remote_url.startswith("https://") or remote_url in get_remote_url_list(repo_path):
exec_cmd = DEFAULT_GIT_CMD_FETCH + [remote_url]
else:
logging.error("Remote name cannot find in your repo [%s]" % remote_url)
return False
logging.debug("git fetch execute cmd [%s]" % exec_cmd)
return_code, output = CommonUtil.subprocess_checkoutput_wrapper(exec_cmd, cwd=repo_path)
if return_code == 0:
logging.info("git fetch command execute success [%s]! output [%s]" % (queue_msg['input_cmd_str'], output))
return True
else:
return False
def git_reset(**kwargs):
"""
git reset command wrapper
@param kwargs:
@return:
"""
DEFAULT_GIT_CMD_RESET = ["git", "reset", "--hard", "HEAD"]
# get queue msg, consumer config from kwargs
queue_msg, consumer_config, task_config = init_task(kwargs)
# get default repo path
repo_path = get_hasal_repo_path(task_config)
logging.debug("git reset execute cmd [%s]" % DEFAULT_GIT_CMD_RESET)
return_code, output = CommonUtil.subprocess_checkoutput_wrapper(DEFAULT_GIT_CMD_RESET, cwd=repo_path)
if return_code == 0:
logging.info("git clean command execute success [%s]! output [%s]" % (queue_msg['input_cmd_str'], output))
return True
else:
return False
def git_checkout(**kwargs):
"""
git checkout command wrapper
@param kwargs:
kwargs['cmd_obj']['configs']['GIT_CHECKOUT_PARAMETER_BRANCH_NAME'] :: branch name
@return:
"""
DEFAULT_GIT_CMD_CHECKOUT = ["git", "checkout"]
# get queue msg, consumer config from kwargs
queue_msg, consumer_config, task_config = init_task(kwargs)
# get default repo path
repo_path = get_hasal_repo_path(task_config)
# get remote url and branch
cmd_parameter_list = parse_cmd_parameters(queue_msg)
if len(cmd_parameter_list) == 2:
branch_name = cmd_parameter_list[1]
else:
branch_name = task_config.get("GIT_CHECKOUT_PARAMETER_BRANCH_NAME", "")
if branch_name:
exec_cmd = DEFAULT_GIT_CMD_CHECKOUT + [branch_name]
else:
logging.error("Please specify the checkout branch after cmd or configs")
return False
logging.debug("git checkout execute cmd [%s]" % exec_cmd)
return_code, output = CommonUtil.subprocess_checkoutput_wrapper(exec_cmd, cwd=repo_path)
if return_code == 0:
logging.info("git checkout command execute success [%s]! output [%s]" % (queue_msg['input_cmd_str'], output))
return True
else:
return False
def download_latest_nightly_build(**kwargs):
"""
you can specify output_dp in task configs or after the cmd
@param kwargs:
kwargs['cmd_obj']['configs']['DOWNLOAD_PKG_OUTPUT_DP'] :: the path where your downloaded firefox package want to store, default: current dir
@return: json obj
"""
# get queue msg, consumer config from kwargs
queue_msg, consumer_config, task_config = init_task(kwargs)
# init return result
return_result = {}
# init download url
remote_url_str = ARCHIVE_URL + NIGHTLY_LATEST_URL_FOLDER
# init platform value
if sys.platform == "linux2":
dl_pkg_platform = "linux64"
elif sys.platform == "darwin":
dl_pkg_platform = "mac"
else:
dl_pkg_platform = "win64"
# get output dir path from kwargs
cmd_parameter_list = parse_cmd_parameters(queue_msg)
if len(cmd_parameter_list) == 2:
output_dp = cmd_parameter_list[1]
else:
output_dp = task_config.get("DOWNLOAD_PKG_OUTPUT_DP", os.getcwd())
return_result['output_dp'] = output_dp
# get download fx package local path and json path
download_fx_fp, download_json_fp = download_from_remote_url_folder(dl_pkg_platform, remote_url_str, output_dp)
return_result['FX-DL-PACKAGE-PATH'] = download_fx_fp
return_result['FX-DL-JSON-PATH'] = download_json_fp
if download_fx_fp is None or download_json_fp is None:
logging.error("something wrong with your build download process, please check the setting and job status.")
return return_result
else:
current_platform_release = platform.release().strip()
# parse downloaded fx pkg json file
with open(download_json_fp) as dl_json_fh:
dl_json_data = json.load(dl_json_fh)
perfherder_revision = dl_json_data['moz_source_stamp']
build_pkg_platform = dl_json_data['moz_pkg_platform']
# mapping the perfherder pkg platform to nomenclature of builddot
builddot_mapping_platform = {"linux-i686": {"_": "linux32"},
"linux-x86_64": {"_": "linux64"},
"mac": {"_": "osx-10-10"},
"win32": {"_": "windows7-32"},
"win64": {"_": "windows8-64",
"7": "windows8-64",
"10": "windows10-64"}
}
return_result['PERFHERDER-REVISION'] = perfherder_revision
if current_platform_release in builddot_mapping_platform[build_pkg_platform].keys():
return_result['PERFHERDER-PKG-PLATFORM'] = builddot_mapping_platform[build_pkg_platform][
current_platform_release]
else:
return_result['PERFHERDER-PKG-PLATFORM'] = builddot_mapping_platform[build_pkg_platform]["_"]
return return_result
def generate_hasal_config(**kwargs):
"""
generate hasal config jsons for ejenti, default should generate agent/chrome/exec/firefox/global/index/online jsons
@param kwargs: will have two keys queue_msg, consumer_config
kwargs['cmd_obj']['configs']['DEFAULT_HASAL_CONFIG_CONTENT_TEMPLATE'] :: default tempalate will use for generating config content
kwargs['cmd_obj']['configs']['DEFAULT_HASAL_RUNTEST_CMD_PARAMETERS_TEMPLATE'] :: default runtest exec parameters template
kwargs['cmd_obj']['configs']['OVERWIRTE_HASAL_CONFIG_CTNT'] :: the ctnt use for overwrite the current config example as below:
{
"configs": {
"exec": {
"default.json": {
"key1": "value1"
}
},
"firefox": {
"default.json": {
"key2": "value2",
"key3": "value3"
}
},
"online": {
"abc.json":{
"key3": "value3",
"key4": "value4"
}
}
}
}
@return:
"""
DEFAULT_HASAL_CONFIG = {
"configs": {
"exec": {
"default.json": {}
},
"firefox": {
"default.json": {}
},
"chrome": {
"default.json": {}
},
"index": {
"runtimeDctGenerator.json": {}
},
"upload": {
"default.json": {}
},
"global": {
"default.json": {}
}
}
}
DEFAULT_HASAL_RUNTEST_CONFIGS = {
"--exec-config": "",
"--firefox-config": "",
"--index-config": "",
"--upload-config": "",
"--global-config": "",
"--chrome-config": ""
}
# get queue msg, consumer config from kwargs
queue_msg, consumer_config, task_config = init_task(kwargs)
# get override config
cmd_parameter_list = queue_msg.get('input_cmd_str', "").split(" ", 1)
default_config_settings = task_config.get(
"DEFAULT_HASAL_CONFIG_CONTENT_TEMPLATE", DEFAULT_HASAL_CONFIG)
default_runtest_configs = task_config.get(
"DEFAULT_HASAL_RUNTEST_CMD_PARAMETERS_TEMPLATE",
DEFAULT_HASAL_RUNTEST_CONFIGS)
# get input config from user interactive mode
if len(cmd_parameter_list) == 2:
input_json_str = cmd_parameter_list[1]
logging.debug("input cmd parameter : [%s]" % input_json_str)
input_json_obj = CommonUtil.load_json_string(input_json_str)
logging.debug("load json obj from input cmd: [%s]" % input_json_obj)
else:
input_json_obj = task_config.get("OVERWIRTE_HASAL_CONFIG_CTNT", {})
logging.debug("load json obj from input config: [%s]" % input_json_obj)
if len(input_json_obj.keys()) == 0:
logging.info(
"No input config object [%s] detected, will use the default config setting instead"
% input_json_obj)
else:
json_path = get_hasal_repo_path(task_config)
# merge default and input
full_config_obj = merge_user_input_config_with_default_config(
input_json_obj, default_config_settings)
# generate config path and need to modify key-value pair dict
full_config_path_mapping = generate_config_path_json_mapping(
json_path, full_config_obj, {})
full_exec_runtest_config = copy.deepcopy(default_runtest_configs)
# dump to json file
for config_path in full_config_path_mapping:
tmp_json_obj = CommonUtil.load_json_file(config_path)
tmp_json_obj.update(full_config_path_mapping[config_path])
dir_name = os.path.dirname(config_path)
new_config_path = os.path.join(dir_name, "ejenti.json")
parameter_name = "--{}-config".format(dir_name.split(os.sep)[-1])
full_exec_runtest_config[parameter_name] = new_config_path
with open(new_config_path, 'w') as fh:
json.dump(tmp_json_obj, fh)
logging.debug("exec runtest config [%s]" % full_exec_runtest_config)
return full_exec_runtest_config