def action_processactions(gopts,command_data, opts):
zkroot = command_data["zkdetails"]["zkroot"]
def get_valid_client():
if not is_existing_client(zkroot, client_name):
print "Invalid client[{client_name}]".format(**locals())
sys.exit(1)
return client_name
def get_valid_input_date_string():
input_date_string = opts.input_date_string
if input_date_string == None:
print "Need input date string!"
sys.exit(1)
return input_date_string
client_name = opts.client_name
if client_name != None:
client_name = get_valid_client()
job_info = command_data["conf_data"]["processactions"]["job_info"]
if client_name != None:
job_info["cmd_args"].append("--single-client")
job_info["cmd_args"].append("%CLIENT_NAME%")
conf_data = command_data["conf_data"]
spark_executor_memory = conf_data["spark_executor_memory"]
spark_driver_memory = conf_data["spark_driver_memory"]
input_date_string = get_valid_input_date_string()
replacements = [
("%INPUT_DATE_STRING%", input_date_string),
("%SPARK_EXECUTOR_MEMORY%", spark_executor_memory),
("%SPARK_DRIVER_MEMORY%", spark_driver_memory)
]
def appy_replacements(item):
for rpair in replacements:
if rpair[1] != None:
item = item.replace(rpair[0],rpair[1])
return item
cmd_args = job_info["cmd_args"]
job_info["cmd_args"] = map(appy_replacements, cmd_args)
spark_utils.run_spark_job(command_data, job_info, client_name)
def action_processactions(gopts, command_data, opts):
zkroot = command_data["zkdetails"]["zkroot"]
def get_valid_client():
if not is_existing_client(zkroot, client_name):
print "Invalid client[{client_name}]".format(**locals())
sys.exit(1)
return client_name
def get_valid_input_date_string():
input_date_string = opts.input_date_string
if input_date_string == None:
print "Need input date string!"
sys.exit(1)
return input_date_string
client_name = opts.client_name
if client_name != None:
client_name = get_valid_client()
job_info = command_data["conf_data"]["processactions"]["job_info"]
if client_name != None:
job_info["cmd_args"].append("--single-client")
job_info["cmd_args"].append("%CLIENT_NAME%")
conf_data = command_data["conf_data"]
spark_executor_memory = conf_data["spark_executor_memory"]
spark_driver_memory = conf_data["spark_driver_memory"]
input_date_string = get_valid_input_date_string()
replacements = [("%INPUT_DATE_STRING%", input_date_string),
("%SPARK_EXECUTOR_MEMORY%", spark_executor_memory),
("%SPARK_DRIVER_MEMORY%", spark_driver_memory)]
def appy_replacements(item):
for rpair in replacements:
if rpair[1] != None:
item = item.replace(rpair[0], rpair[1])
return item
cmd_args = job_info["cmd_args"]
job_info["cmd_args"] = map(appy_replacements, cmd_args)
spark_utils.run_spark_job(command_data, job_info, client_name)