def submit_python_zip(cl_args, unknown_args): # set up the system properties java_system_props = setup_java_system_properties(cl_args) props = read_client_properties(cl_args) # execute main of the job to create the job definition job_file = cl_args['job-file-name'] main_class = cl_args['job-class-name'] java_system_props += ["python_file=" + job_file, "main_file=" + main_class] res = execute.twister2_class( class_name="edu.iu.dsc.tws.python.PythonWorker", lib_jars=config.get_twister2_libs(jars.job_jars()), extra_jars=[], args=tuple(unknown_args), java_defines=java_system_props, client_props=props) result.render(res) if not res.is_successful(): err_context = ("Failed to create job definition " \ "Please check logs for more information") res.add_context(err_context) return res return res
def run(command, cl_args, action, extra_args=[], extra_lib_jars=[]): ''' helper function to take action on topologies :param command: :param cl_args: :param action: description of action taken :return: ''' job_id = cl_args['job-id'] new_args = [ "--cluster", cl_args['cluster'], "--twister2_home", config.get_twister2_dir(), "--config_path", config.get_twister2_conf_dir(), "--job_id", job_id, "--command", command, ] new_args += extra_args lib_jars = config.get_twister2_libs(jars.resource_scheduler_jars() + jars.statemgr_jars()) lib_jars += extra_lib_jars if Log.getEffectiveLevel() == logging.DEBUG: new_args.append("--verbose") java_defines = [] conf_dir_common = config.get_twister2_cluster_conf_dir( "common", config.get_twister2_conf_dir()) java_defines.append("java.util.logging.config.file=" + conf_dir_common + "/logger.properties") # invoke the runtime manager to kill the job result = execute.twister2_class( 'edu.iu.dsc.tws.rsched.core.RuntimeManagerMain', lib_jars, extra_jars=[], args=new_args, java_defines=java_defines) err_msg = "Failed to %s %s" % (action, job_id) succ_msg = "Successfully %s %s" % (action, job_id) result.add_context(err_msg, succ_msg) return result
def run(command, cl_args, action, extra_args=[], extra_lib_jars=[]): ''' helper function to take action on topologies :param command: :param cl_args: :param action: description of action taken :return: ''' topology_name = cl_args['job-name'] new_args = [ "--cluster", cl_args['cluster'], "--twister2_home", config.get_twister2_dir(), "--config_path", cl_args['config_path'], "--override_config_file", cl_args['override_config_file'], "--job_name", job_name, "--command", command, ] new_args += extra_args lib_jars = config.get_twister2_libs(jars.resource_scheduler_jars() + jars.statemgr_jars()) lib_jars += extra_lib_jars if Log.getEffectiveLevel() == logging.DEBUG: new_args.append("--verbose") # invoke the runtime manager to kill the job result = execute.twister2_class( 'com.twitter.twister2.scheduler.RuntimeManagerMain', lib_jars, extra_jars=[], args=new_args) err_msg = "Failed to %s %s" % (action, topology_name) succ_msg = "Successfully %s %s" % (action, topology_name) result.add_context(err_msg, succ_msg) return result
def submit_fatjar(cl_args, unknown_args): ''' We use the packer to make a package for the jar and dump it to a well-known location. We then run the main method of class with the specified arguments. We pass arguments as an environment variable TWISTER2_OPTIONS. This will run the jar file with the job_class_name. The submitter inside will write out the job defn file to a location that we specify. Then we write the job defn file to a well known location. We then write to appropriate places in zookeeper and launch the scheduler jobs :param cl_args: :param unknown_args: :param tmp_dir: :return: ''' # set up the system properties java_system_props = setup_java_system_properties(cl_args) props = read_client_properties(cl_args) # execute main of the job to create the job definition job_file = cl_args['job-file-name'] main_class = cl_args['job-class-name'] res = execute.twister2_class(class_name=main_class, lib_jars=config.get_twister2_libs( jars.job_jars()), extra_jars=[job_file], args=tuple(unknown_args), java_defines=java_system_props, client_props=props) result.render(res) if not res.is_successful(): err_context = ("Failed to create job definition " \ "Please check logs for more information") res.add_context(err_context) return res return res
def run(command, parser, args, unknown_args): ''' :param command: :param parser: :param args: :param unknown_args: :return: ''' job_id = args['job-id'] conf_path = os.path.join(config.get_twister2_dir(), "conf/common") new_args = [ "--twister2_home", config.get_twister2_dir(), "--config_path",conf_path, "--job_id", job_id, "--command", command, ] new_args += unknown_args lib_jars = config.get_twister2_libs(jars.resource_scheduler_jars()) if Log.getEffectiveLevel() == logging.DEBUG: new_args.append("--verbose") java_defines = [] conf_dir_common = config.get_twister2_cluster_conf_dir("common", config.get_twister2_conf_dir()) java_defines.append("java.util.logging.config.file=" + conf_dir_common + "/logger.properties") # invoke the runtime manager to kill the job result = execute.twister2_class( 'edu.iu.dsc.tws.rsched.job.ZKJobLister', lib_jars, extra_jars=[], args=new_args, java_defines=java_defines ) err_msg = "Failed to %s %s" % (command, job_id) succ_msg = "Successfully %s %s" % (command, job_id) result.add_context(err_msg, succ_msg) return result