def twister2_class(class_name,
lib_jars,
extra_jars=None,
args=None,
java_defines=None,
client_props=None):
'''
Execute a twister2 class given the args and the jars needed for class path
:param class_name:
:param lib_jars:
:param extra_jars:
:param args:
:param java_defines:
:return:
'''
# default optional params to empty list if not provided
if extra_jars is None:
extra_jars = []
if args is None:
args = []
if java_defines is None:
java_defines = []
# Format all java -D options that need to be passed while running
# the class locally.
java_opts = ['-D' + opt for opt in java_defines]
if client_props:
if 'twister2.client.debug' in client_props:
java_opts.append(client_props['twister2.client.debug'])
# Construct the command line for the sub process to run
# Because of the way Python execute works,
# the java opts must be passed as part of the list
all_args = [config.get_java_path(), "-client", "-Xmx1g"] + \
java_opts + \
["-cp", config.get_classpath(extra_jars + lib_jars)]
all_args += [class_name] + list(args)
# set twister2_config environment variable
twister2_env = os.environ.copy()
twister2_env['TWISTER2_OPTIONS'] = opts.get_twister2_config()
# print the verbose message
Log.debug("Invoking class using command: ``%s''", ' '.join(all_args))
Log.debug("Twister2 options: {%s}", str(twister2_env["TWISTER2_OPTIONS"]))
# invoke the command with subprocess and print error message, if any
proc = subprocess.Popen(
all_args,
env=twister2_env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=1,
universal_newlines=True,
)
# stdout message has the information Java program sends back
# stderr message has extra information, such as debugging message
return ProcessResult(proc)
def check_release_file_exists():
"""Check if the release.yaml file exists"""
release_file = get_twister2_release_file()
# if the file does not exist and is not a file
if not os.path.isfile(release_file):
Log.error("Required file not found: %s" % release_file)
return False
return True
def parse_cluster_role_env(cluster_role_env, config_path):
"""Parse cluster/[role]/[environ], supply default, if not provided, not required"""
parts = cluster_role_env.split('/')[:3]
Log.info("Using config file under %s" % config_path)
if not os.path.isdir(config_path):
Log.error("Config path cluster directory does not exist: %s" % config_path)
raise Exception("Invalid config path")
# if cluster/role/env is not completely provided, check further
if len(parts) < 3:
cli_conf_file = os.path.join(config_path, CLIENT_YAML)
# if client conf doesn't exist, use default value
if not os.path.isfile(cli_conf_file):
if len(parts) == 1:
parts.append(getpass.getuser())
if len(parts) == 2:
parts.append(ENVIRON)
else:
cli_confs = {}
with open(cli_conf_file, 'r') as conf_file:
tmp_confs = yaml.load(conf_file)
# the return value of yaml.load can be None if conf_file is an empty file
if tmp_confs is not None:
cli_confs = tmp_confs
else:
print("Failed to read: %s due to it is empty" % (CLIENT_YAML))
# if role is required but not provided, raise exception
if len(parts) == 1:
if (IS_ROLE_REQUIRED in cli_confs) and (cli_confs[IS_ROLE_REQUIRED] is True):
raise Exception("role required but not provided (cluster/role/env = %s). See %s in %s"
% (cluster_role_env, IS_ROLE_REQUIRED, CLIENT_YAML))
else:
parts.append(getpass.getuser())
# if environ is required but not provided, raise exception
if len(parts) == 2:
if (IS_ENV_REQUIRED in cli_confs) and (cli_confs[IS_ENV_REQUIRED] is True):
raise Exception("environ required but not provided (cluster/role/env = %s). See %s in %s"
% (cluster_role_env, IS_ENV_REQUIRED, CLIENT_YAML))
else:
parts.append(ENVIRON)
# if cluster or role or environ is empty, print
if len(parts[0]) == 0 or len(parts[1]) == 0 or len(parts[2]) == 0:
print("Failed to parse")
sys.exit(1)
return (parts[0], parts[1], parts[2])
def check_java_home_set():
"""Check if the java home set"""
# check if environ variable is set
if "JAVA_HOME" not in os.environ:
Log.error("JAVA_HOME not set")
return False
# check if the value set is correct
java_path = get_java_path()
if os.path.isfile(java_path) and os.access(java_path, os.X_OK):
return True
Log.error("JAVA_HOME/bin/java either does not exist or not an executable")
return False
def run(command, cl_args, action, extra_args=[], extra_lib_jars=[]):
'''
helper function to take action on topologies
:param command:
:param cl_args:
:param action: description of action taken
:return:
'''
job_id = cl_args['job-id']
new_args = [
"--cluster",
cl_args['cluster'],
"--twister2_home",
config.get_twister2_dir(),
"--config_path",
config.get_twister2_conf_dir(),
"--job_id",
job_id,
"--command",
command,
]
new_args += extra_args
lib_jars = config.get_twister2_libs(jars.resource_scheduler_jars() +
jars.statemgr_jars())
lib_jars += extra_lib_jars
if Log.getEffectiveLevel() == logging.DEBUG:
new_args.append("--verbose")
java_defines = []
conf_dir_common = config.get_twister2_cluster_conf_dir(
"common", config.get_twister2_conf_dir())
java_defines.append("java.util.logging.config.file=" + conf_dir_common +
"/logger.properties")
# invoke the runtime manager to kill the job
result = execute.twister2_class(
'edu.iu.dsc.tws.rsched.core.RuntimeManagerMain',
lib_jars,
extra_jars=[],
args=new_args,
java_defines=java_defines)
err_msg = "Failed to %s %s" % (action, job_id)
succ_msg = "Successfully %s %s" % (action, job_id)
result.add_context(err_msg, succ_msg)
return result
def run(command, parser, args, unknown_args):
'''
:param command:
:param parser:
:param args:
:param unknown_args:
:return:
'''
# get the command for detailed help
command_help = args['help-command']
# if no command is provided, just print main help
if command_help == 'help':
parser.print_help()
return SimpleResult(Status.Ok)
# get the subparser for the specific command
subparser = config.get_subparser(parser, command_help)
if subparser:
print subparser.format_help()
return SimpleResult(Status.Ok)
else:
Log.error("Unknown subcommand \'%s\'", command_help)
return SimpleResult(Status.InvocationError)
def run(command, cl_args, action, extra_args=[], extra_lib_jars=[]):
'''
helper function to take action on topologies
:param command:
:param cl_args:
:param action: description of action taken
:return:
'''
topology_name = cl_args['job-name']
new_args = [
"--cluster",
cl_args['cluster'],
"--twister2_home",
config.get_twister2_dir(),
"--config_path",
cl_args['config_path'],
"--override_config_file",
cl_args['override_config_file'],
"--job_name",
job_name,
"--command",
command,
]
new_args += extra_args
lib_jars = config.get_twister2_libs(jars.resource_scheduler_jars() +
jars.statemgr_jars())
lib_jars += extra_lib_jars
if Log.getEffectiveLevel() == logging.DEBUG:
new_args.append("--verbose")
# invoke the runtime manager to kill the job
result = execute.twister2_class(
'com.twitter.twister2.scheduler.RuntimeManagerMain',
lib_jars,
extra_jars=[],
args=new_args)
err_msg = "Failed to %s %s" % (action, topology_name)
succ_msg = "Successfully %s %s" % (action, topology_name)
result.add_context(err_msg, succ_msg)
return result
def run(command, parser, args, unknown_args):
'''
:param command:
:param parser:
:param args:
:param unknown_args:
:return:
'''
job_id = args['job-id']
conf_path = os.path.join(config.get_twister2_dir(), "conf/common")
new_args = [
"--twister2_home", config.get_twister2_dir(),
"--config_path",conf_path,
"--job_id", job_id,
"--command", command,
]
new_args += unknown_args
lib_jars = config.get_twister2_libs(jars.resource_scheduler_jars())
if Log.getEffectiveLevel() == logging.DEBUG:
new_args.append("--verbose")
java_defines = []
conf_dir_common = config.get_twister2_cluster_conf_dir("common", config.get_twister2_conf_dir())
java_defines.append("java.util.logging.config.file=" + conf_dir_common + "/logger.properties")
# invoke the runtime manager to kill the job
result = execute.twister2_class(
'edu.iu.dsc.tws.rsched.job.ZKJobLister',
lib_jars,
extra_jars=[],
args=new_args,
java_defines=java_defines
)
err_msg = "Failed to %s %s" % (command, job_id)
succ_msg = "Successfully %s %s" % (command, job_id)
result.add_context(err_msg, succ_msg)
return result