def submit_python_zip(cl_args, unknown_args):
# set up the system properties
java_system_props = setup_java_system_properties(cl_args)
props = read_client_properties(cl_args)
# execute main of the job to create the job definition
job_file = cl_args['job-file-name']
main_class = cl_args['job-class-name']
java_system_props += ["python_file=" + job_file, "main_file=" + main_class]
res = execute.twister2_class(
class_name="edu.iu.dsc.tws.python.PythonWorker",
lib_jars=config.get_twister2_libs(jars.job_jars()),
extra_jars=[],
args=tuple(unknown_args),
java_defines=java_system_props,
client_props=props)
result.render(res)
if not res.is_successful():
err_context = ("Failed to create job definition " \
"Please check logs for more information")
res.add_context(err_context)
return res
return res
def run(command, cl_args, action, extra_args=[], extra_lib_jars=[]):
'''
helper function to take action on topologies
:param command:
:param cl_args:
:param action: description of action taken
:return:
'''
job_id = cl_args['job-id']
new_args = [
"--cluster",
cl_args['cluster'],
"--twister2_home",
config.get_twister2_dir(),
"--config_path",
config.get_twister2_conf_dir(),
"--job_id",
job_id,
"--command",
command,
]
new_args += extra_args
lib_jars = config.get_twister2_libs(jars.resource_scheduler_jars() +
jars.statemgr_jars())
lib_jars += extra_lib_jars
if Log.getEffectiveLevel() == logging.DEBUG:
new_args.append("--verbose")
java_defines = []
conf_dir_common = config.get_twister2_cluster_conf_dir(
"common", config.get_twister2_conf_dir())
java_defines.append("java.util.logging.config.file=" + conf_dir_common +
"/logger.properties")
# invoke the runtime manager to kill the job
result = execute.twister2_class(
'edu.iu.dsc.tws.rsched.core.RuntimeManagerMain',
lib_jars,
extra_jars=[],
args=new_args,
java_defines=java_defines)
err_msg = "Failed to %s %s" % (action, job_id)
succ_msg = "Successfully %s %s" % (action, job_id)
result.add_context(err_msg, succ_msg)
return result
def run(command, cl_args, action, extra_args=[], extra_lib_jars=[]):
'''
helper function to take action on topologies
:param command:
:param cl_args:
:param action: description of action taken
:return:
'''
topology_name = cl_args['job-name']
new_args = [
"--cluster",
cl_args['cluster'],
"--twister2_home",
config.get_twister2_dir(),
"--config_path",
cl_args['config_path'],
"--override_config_file",
cl_args['override_config_file'],
"--job_name",
job_name,
"--command",
command,
]
new_args += extra_args
lib_jars = config.get_twister2_libs(jars.resource_scheduler_jars() +
jars.statemgr_jars())
lib_jars += extra_lib_jars
if Log.getEffectiveLevel() == logging.DEBUG:
new_args.append("--verbose")
# invoke the runtime manager to kill the job
result = execute.twister2_class(
'com.twitter.twister2.scheduler.RuntimeManagerMain',
lib_jars,
extra_jars=[],
args=new_args)
err_msg = "Failed to %s %s" % (action, topology_name)
succ_msg = "Successfully %s %s" % (action, topology_name)
result.add_context(err_msg, succ_msg)
return result
def submit_fatjar(cl_args, unknown_args):
'''
We use the packer to make a package for the jar and dump it
to a well-known location. We then run the main method of class
with the specified arguments. We pass arguments as an environment variable TWISTER2_OPTIONS.
This will run the jar file with the job_class_name. The submitter
inside will write out the job defn file to a location that
we specify. Then we write the job defn file to a well known
location. We then write to appropriate places in zookeeper
and launch the scheduler jobs
:param cl_args:
:param unknown_args:
:param tmp_dir:
:return:
'''
# set up the system properties
java_system_props = setup_java_system_properties(cl_args)
props = read_client_properties(cl_args)
# execute main of the job to create the job definition
job_file = cl_args['job-file-name']
main_class = cl_args['job-class-name']
res = execute.twister2_class(class_name=main_class,
lib_jars=config.get_twister2_libs(
jars.job_jars()),
extra_jars=[job_file],
args=tuple(unknown_args),
java_defines=java_system_props,
client_props=props)
result.render(res)
if not res.is_successful():
err_context = ("Failed to create job definition " \
"Please check logs for more information")
res.add_context(err_context)
return res
return res
def run(command, parser, args, unknown_args):
'''
:param command:
:param parser:
:param args:
:param unknown_args:
:return:
'''
job_id = args['job-id']
conf_path = os.path.join(config.get_twister2_dir(), "conf/common")
new_args = [
"--twister2_home", config.get_twister2_dir(),
"--config_path",conf_path,
"--job_id", job_id,
"--command", command,
]
new_args += unknown_args
lib_jars = config.get_twister2_libs(jars.resource_scheduler_jars())
if Log.getEffectiveLevel() == logging.DEBUG:
new_args.append("--verbose")
java_defines = []
conf_dir_common = config.get_twister2_cluster_conf_dir("common", config.get_twister2_conf_dir())
java_defines.append("java.util.logging.config.file=" + conf_dir_common + "/logger.properties")
# invoke the runtime manager to kill the job
result = execute.twister2_class(
'edu.iu.dsc.tws.rsched.job.ZKJobLister',
lib_jars,
extra_jars=[],
args=new_args,
java_defines=java_defines
)
err_msg = "Failed to %s %s" % (command, job_id)
succ_msg = "Successfully %s %s" % (command, job_id)
result.add_context(err_msg, succ_msg)
return result