def twister2_tar(class_name, topology_tar, arguments, tmpdir_root, java_defines):
'''
:param class_name:
:param topology_tar:
:param arguments:
:param tmpdir_root:
:param java_defines:
:return:
'''
# Extract tar to a tmp folder.
tmpdir = tempfile.mkdtemp(dir=tmpdir_root, prefix='tmp')
with contextlib.closing(tarfile.open(topology_tar)) as tar:
tar.extractall(path=tmpdir)
# A tar generated by pants has all dependency jars under libs/
# in addition to the topology jar at top level. Pants keeps
# filename for jar and tar the same except for extension.
topology_jar = os.path.basename(topology_tar).replace(".tar.gz", "").replace(".tar", "") + ".jar"
extra_jars = [
os.path.join(tmpdir, "twister2-instance.jar"),
os.path.join(tmpdir, topology_jar),
os.path.join(tmpdir, "*"),
os.path.join(tmpdir, "libs/*")
]
lib_jars = config.get_twister2_libs(jars.job_jars())
# Now execute the class
return twister2_class(class_name, lib_jars, extra_jars, arguments, java_defines)
def submit_python_zip(cl_args, unknown_args):
# set up the system properties
java_system_props = setup_java_system_properties(cl_args)
props = read_client_properties(cl_args)
# execute main of the job to create the job definition
job_file = cl_args['job-file-name']
main_class = cl_args['job-class-name']
java_system_props += ["python_file=" + job_file, "main_file=" + main_class]
res = execute.twister2_class(
class_name="edu.iu.dsc.tws.python.PythonWorker",
lib_jars=config.get_twister2_libs(jars.job_jars()),
extra_jars=[],
args=tuple(unknown_args),
java_defines=java_system_props,
client_props=props)
result.render(res)
if not res.is_successful():
err_context = ("Failed to create job definition " \
"Please check logs for more information")
res.add_context(err_context)
return res
return res
def twister2_tar(class_name,
topology_tar,
arguments,
tmpdir_root,
java_defines,
client_props=None):
'''
:param class_name:
:param topology_tar:
:param arguments:
:param tmpdir_root:
:param java_defines:
:return:
'''
# Extract tar to a tmp folder.
tmpdir = tempfile.mkdtemp(dir=tmpdir_root, prefix='tmp')
topology_jar = os.path.basename(topology_tar).replace(".zip", "")
with ZipFile(topology_tar, 'r') as zipObj:
print(topology_tar, tmpdir, topology_jar)
# Extract all the contents of zip file in different directory
zipObj.extractall(tmpdir)
if java_defines is None:
java_defines = []
# Format all java -D options that need to be passed while running
# the class locally.
java_opts = ['-D' + opt for opt in java_defines]
extra_jars = [
os.path.join(tmpdir, topology_jar, "*.jar"),
os.path.join(tmpdir, topology_jar, "libs/*.jar")
]
for root, dirs, files in os.walk(os.path.join(tmpdir, topology_jar)):
for file in files:
if file.endswith('.jar'):
extra_jars.append(os.path.abspath(os.path.join(root, file)))
print(extra_jars)
lib_jars = config.get_twister2_libs(jars.job_jars())
print(lib_jars)
if client_props:
if 'twister2.client.debug' in client_props:
java_opts.append(client_props['twister2.client.debug'])
# Now execute the class
return twister2_class(class_name, lib_jars, extra_jars, arguments,
java_defines, client_props)
def run(command, cl_args, action, extra_args=[], extra_lib_jars=[]):
'''
helper function to take action on topologies
:param command:
:param cl_args:
:param action: description of action taken
:return:
'''
job_id = cl_args['job-id']
new_args = [
"--cluster",
cl_args['cluster'],
"--twister2_home",
config.get_twister2_dir(),
"--config_path",
config.get_twister2_conf_dir(),
"--job_id",
job_id,
"--command",
command,
]
new_args += extra_args
lib_jars = config.get_twister2_libs(jars.resource_scheduler_jars() +
jars.statemgr_jars())
lib_jars += extra_lib_jars
if Log.getEffectiveLevel() == logging.DEBUG:
new_args.append("--verbose")
java_defines = []
conf_dir_common = config.get_twister2_cluster_conf_dir(
"common", config.get_twister2_conf_dir())
java_defines.append("java.util.logging.config.file=" + conf_dir_common +
"/logger.properties")
# invoke the runtime manager to kill the job
result = execute.twister2_class(
'edu.iu.dsc.tws.rsched.core.RuntimeManagerMain',
lib_jars,
extra_jars=[],
args=new_args,
java_defines=java_defines)
err_msg = "Failed to %s %s" % (action, job_id)
succ_msg = "Successfully %s %s" % (action, job_id)
result.add_context(err_msg, succ_msg)
return result
def run(command, cl_args, action, extra_args=[], extra_lib_jars=[]):
'''
helper function to take action on topologies
:param command:
:param cl_args:
:param action: description of action taken
:return:
'''
topology_name = cl_args['job-name']
new_args = [
"--cluster",
cl_args['cluster'],
"--twister2_home",
config.get_twister2_dir(),
"--config_path",
cl_args['config_path'],
"--override_config_file",
cl_args['override_config_file'],
"--job_name",
job_name,
"--command",
command,
]
new_args += extra_args
lib_jars = config.get_twister2_libs(jars.resource_scheduler_jars() +
jars.statemgr_jars())
lib_jars += extra_lib_jars
if Log.getEffectiveLevel() == logging.DEBUG:
new_args.append("--verbose")
# invoke the runtime manager to kill the job
result = execute.twister2_class(
'com.twitter.twister2.scheduler.RuntimeManagerMain',
lib_jars,
extra_jars=[],
args=new_args)
err_msg = "Failed to %s %s" % (action, topology_name)
succ_msg = "Successfully %s %s" % (action, topology_name)
result.add_context(err_msg, succ_msg)
return result
def submit_fatjar(cl_args, unknown_args):
'''
We use the packer to make a package for the jar and dump it
to a well-known location. We then run the main method of class
with the specified arguments. We pass arguments as an environment variable TWISTER2_OPTIONS.
This will run the jar file with the job_class_name. The submitter
inside will write out the job defn file to a location that
we specify. Then we write the job defn file to a well known
location. We then write to appropriate places in zookeeper
and launch the scheduler jobs
:param cl_args:
:param unknown_args:
:param tmp_dir:
:return:
'''
# set up the system properties
java_system_props = setup_java_system_properties(cl_args)
props = read_client_properties(cl_args)
# execute main of the job to create the job definition
job_file = cl_args['job-file-name']
main_class = cl_args['job-class-name']
res = execute.twister2_class(class_name=main_class,
lib_jars=config.get_twister2_libs(
jars.job_jars()),
extra_jars=[job_file],
args=tuple(unknown_args),
java_defines=java_system_props,
client_props=props)
result.render(res)
if not res.is_successful():
err_context = ("Failed to create job definition " \
"Please check logs for more information")
res.add_context(err_context)
return res
return res
def run(command, parser, args, unknown_args):
'''
:param command:
:param parser:
:param args:
:param unknown_args:
:return:
'''
job_id = args['job-id']
conf_path = os.path.join(config.get_twister2_dir(), "conf/common")
new_args = [
"--twister2_home", config.get_twister2_dir(),
"--config_path",conf_path,
"--job_id", job_id,
"--command", command,
]
new_args += unknown_args
lib_jars = config.get_twister2_libs(jars.resource_scheduler_jars())
if Log.getEffectiveLevel() == logging.DEBUG:
new_args.append("--verbose")
java_defines = []
conf_dir_common = config.get_twister2_cluster_conf_dir("common", config.get_twister2_conf_dir())
java_defines.append("java.util.logging.config.file=" + conf_dir_common + "/logger.properties")
# invoke the runtime manager to kill the job
result = execute.twister2_class(
'edu.iu.dsc.tws.rsched.job.ZKJobLister',
lib_jars,
extra_jars=[],
args=new_args,
java_defines=java_defines
)
err_msg = "Failed to %s %s" % (command, job_id)
succ_msg = "Successfully %s %s" % (command, job_id)
result.add_context(err_msg, succ_msg)
return result