def submit_python_zip(cl_args, unknown_args):
# set up the system properties
java_system_props = setup_java_system_properties(cl_args)
props = read_client_properties(cl_args)
# execute main of the job to create the job definition
job_file = cl_args['job-file-name']
main_class = cl_args['job-class-name']
java_system_props += ["python_file=" + job_file, "main_file=" + main_class]
res = execute.twister2_class(
class_name="edu.iu.dsc.tws.python.PythonWorker",
lib_jars=config.get_twister2_libs(jars.job_jars()),
extra_jars=[],
args=tuple(unknown_args),
java_defines=java_system_props,
client_props=props)
result.render(res)
if not res.is_successful():
err_context = ("Failed to create job definition " \
"Please check logs for more information")
res.add_context(err_context)
return res
return res
def submit_java_zip(cl_args, unknown_args):
# set up the system properties
java_system_props = setup_java_system_properties(cl_args)
props = read_client_properties(cl_args)
# execute main of the job to create the job definition
job_file = cl_args['job-file-name']
main_class = cl_args['job-class-name']
res = execute.twister2_tar(class_name=main_class,
topology_tar=job_file,
arguments=tuple(unknown_args),
tmpdir_root="/tmp",
java_defines=java_system_props,
client_props=props)
result.render(res)
if not res.is_successful():
err_context = ("Failed to create job definition " \
"Please check logs for more information")
res.add_context(err_context)
return res
return res
def main():
'''
Run the command
:return:
'''
# verify if the environment variables are correctly set
check_environment()
for_python = len(sys.argv) > 4 and sys.argv[3] == "python"
# create the argument parser
parser = create_parser(for_python)
# if no argument is provided, print help and exit
if len(sys.argv[1:]) == 0:
parser.print_help()
return 0
# insert the boolean values for some of the options
sys.argv = config.insert_bool_values(sys.argv)
try:
# parse the args
args, unknown_args = parser.parse_known_args()
# print "args ", argparse, " unknown_args", unknown_args
except ValueError as ex:
Log.error("Error while parsing arguments: %s", str(ex))
Log.debug(traceback.format_exc())
sys.exit(1)
command_line_args = vars(args)
# command to be execute
command = command_line_args['subcommand']
# print the input parameters, if verbose is enabled
Log.debug(command_line_args)
start = time.time()
results = run(command, parser, command_line_args, unknown_args)
if command not in ('help', 'version'):
result.render(results)
end = time.time()
if command not in ('help', 'version'):
sys.stdout.flush()
Log.info('Elapsed time: %.3fs.', (end - start))
return 0 if result.isAllSuccessful(results) else 1
def submit_fatjar(cl_args, unknown_args):
'''
We use the packer to make a package for the jar and dump it
to a well-known location. We then run the main method of class
with the specified arguments. We pass arguments as an environment variable TWISTER2_OPTIONS.
This will run the jar file with the job_class_name. The submitter
inside will write out the job defn file to a location that
we specify. Then we write the job defn file to a well known
location. We then write to appropriate places in zookeeper
and launch the scheduler jobs
:param cl_args:
:param unknown_args:
:param tmp_dir:
:return:
'''
# set up the system properties
java_system_props = setup_java_system_properties(cl_args)
props = read_client_properties(cl_args)
# execute main of the job to create the job definition
job_file = cl_args['job-file-name']
main_class = cl_args['job-class-name']
res = execute.twister2_class(class_name=main_class,
lib_jars=config.get_twister2_libs(
jars.job_jars()),
extra_jars=[job_file],
args=tuple(unknown_args),
java_defines=java_system_props,
client_props=props)
result.render(res)
if not res.is_successful():
err_context = ("Failed to create job definition " \
"Please check logs for more information")
res.add_context(err_context)
return res
return res