def action_processactions(gopts,command_data, opts): zkroot = command_data["zkdetails"]["zkroot"] def get_valid_client(): if not is_existing_client(zkroot, client_name): print "Invalid client[{client_name}]".format(**locals()) sys.exit(1) return client_name def get_valid_input_date_string(): input_date_string = opts.input_date_string if input_date_string == None: print "Need input date string!" sys.exit(1) return input_date_string client_name = opts.client_name if client_name != None: client_name = get_valid_client() job_info = command_data["conf_data"]["processactions"]["job_info"] if client_name != None: job_info["cmd_args"].append("--single-client") job_info["cmd_args"].append("%CLIENT_NAME%") conf_data = command_data["conf_data"] spark_executor_memory = conf_data["spark_executor_memory"] spark_driver_memory = conf_data["spark_driver_memory"] input_date_string = get_valid_input_date_string() replacements = [ ("%INPUT_DATE_STRING%", input_date_string), ("%SPARK_EXECUTOR_MEMORY%", spark_executor_memory), ("%SPARK_DRIVER_MEMORY%", spark_driver_memory) ] def appy_replacements(item): for rpair in replacements: if rpair[1] != None: item = item.replace(rpair[0],rpair[1]) return item cmd_args = job_info["cmd_args"] job_info["cmd_args"] = map(appy_replacements, cmd_args) spark_utils.run_spark_job(command_data, job_info, client_name)
def action_processactions(gopts, command_data, opts): zkroot = command_data["zkdetails"]["zkroot"] def get_valid_client(): if not is_existing_client(zkroot, client_name): print "Invalid client[{client_name}]".format(**locals()) sys.exit(1) return client_name def get_valid_input_date_string(): input_date_string = opts.input_date_string if input_date_string == None: print "Need input date string!" sys.exit(1) return input_date_string client_name = opts.client_name if client_name != None: client_name = get_valid_client() job_info = command_data["conf_data"]["processactions"]["job_info"] if client_name != None: job_info["cmd_args"].append("--single-client") job_info["cmd_args"].append("%CLIENT_NAME%") conf_data = command_data["conf_data"] spark_executor_memory = conf_data["spark_executor_memory"] spark_driver_memory = conf_data["spark_driver_memory"] input_date_string = get_valid_input_date_string() replacements = [("%INPUT_DATE_STRING%", input_date_string), ("%SPARK_EXECUTOR_MEMORY%", spark_executor_memory), ("%SPARK_DRIVER_MEMORY%", spark_driver_memory)] def appy_replacements(item): for rpair in replacements: if rpair[1] != None: item = item.replace(rpair[0], rpair[1]) return item cmd_args = job_info["cmd_args"] job_info["cmd_args"] = map(appy_replacements, cmd_args) spark_utils.run_spark_job(command_data, job_info, client_name)