def submit_python_zip(cl_args, unknown_args): # set up the system properties java_system_props = setup_java_system_properties(cl_args) props = read_client_properties(cl_args) # execute main of the job to create the job definition job_file = cl_args['job-file-name'] main_class = cl_args['job-class-name'] java_system_props += ["python_file=" + job_file, "main_file=" + main_class] res = execute.twister2_class( class_name="edu.iu.dsc.tws.python.PythonWorker", lib_jars=config.get_twister2_libs(jars.job_jars()), extra_jars=[], args=tuple(unknown_args), java_defines=java_system_props, client_props=props) result.render(res) if not res.is_successful(): err_context = ("Failed to create job definition " \ "Please check logs for more information") res.add_context(err_context) return res return res
def submit_java_zip(cl_args, unknown_args): # set up the system properties java_system_props = setup_java_system_properties(cl_args) props = read_client_properties(cl_args) # execute main of the job to create the job definition job_file = cl_args['job-file-name'] main_class = cl_args['job-class-name'] res = execute.twister2_tar(class_name=main_class, topology_tar=job_file, arguments=tuple(unknown_args), tmpdir_root="/tmp", java_defines=java_system_props, client_props=props) result.render(res) if not res.is_successful(): err_context = ("Failed to create job definition " \ "Please check logs for more information") res.add_context(err_context) return res return res
def main(): ''' Run the command :return: ''' # verify if the environment variables are correctly set check_environment() for_python = len(sys.argv) > 4 and sys.argv[3] == "python" # create the argument parser parser = create_parser(for_python) # if no argument is provided, print help and exit if len(sys.argv[1:]) == 0: parser.print_help() return 0 # insert the boolean values for some of the options sys.argv = config.insert_bool_values(sys.argv) try: # parse the args args, unknown_args = parser.parse_known_args() # print "args ", argparse, " unknown_args", unknown_args except ValueError as ex: Log.error("Error while parsing arguments: %s", str(ex)) Log.debug(traceback.format_exc()) sys.exit(1) command_line_args = vars(args) # command to be execute command = command_line_args['subcommand'] # print the input parameters, if verbose is enabled Log.debug(command_line_args) start = time.time() results = run(command, parser, command_line_args, unknown_args) if command not in ('help', 'version'): result.render(results) end = time.time() if command not in ('help', 'version'): sys.stdout.flush() Log.info('Elapsed time: %.3fs.', (end - start)) return 0 if result.isAllSuccessful(results) else 1
def submit_fatjar(cl_args, unknown_args): ''' We use the packer to make a package for the jar and dump it to a well-known location. We then run the main method of class with the specified arguments. We pass arguments as an environment variable TWISTER2_OPTIONS. This will run the jar file with the job_class_name. The submitter inside will write out the job defn file to a location that we specify. Then we write the job defn file to a well known location. We then write to appropriate places in zookeeper and launch the scheduler jobs :param cl_args: :param unknown_args: :param tmp_dir: :return: ''' # set up the system properties java_system_props = setup_java_system_properties(cl_args) props = read_client_properties(cl_args) # execute main of the job to create the job definition job_file = cl_args['job-file-name'] main_class = cl_args['job-class-name'] res = execute.twister2_class(class_name=main_class, lib_jars=config.get_twister2_libs( jars.job_jars()), extra_jars=[job_file], args=tuple(unknown_args), java_defines=java_system_props, client_props=props) result.render(res) if not res.is_successful(): err_context = ("Failed to create job definition " \ "Please check logs for more information") res.add_context(err_context) return res return res