def twister2_tar(class_name, topology_tar, arguments, tmpdir_root, java_defines):
'''
:param class_name:
:param topology_tar:
:param arguments:
:param tmpdir_root:
:param java_defines:
:return:
'''
# Extract tar to a tmp folder.
tmpdir = tempfile.mkdtemp(dir=tmpdir_root, prefix='tmp')
with contextlib.closing(tarfile.open(topology_tar)) as tar:
tar.extractall(path=tmpdir)
# A tar generated by pants has all dependency jars under libs/
# in addition to the topology jar at top level. Pants keeps
# filename for jar and tar the same except for extension.
topology_jar = os.path.basename(topology_tar).replace(".tar.gz", "").replace(".tar", "") + ".jar"
extra_jars = [
os.path.join(tmpdir, "twister2-instance.jar"),
os.path.join(tmpdir, topology_jar),
os.path.join(tmpdir, "*"),
os.path.join(tmpdir, "libs/*")
]
lib_jars = config.get_twister2_libs(jars.job_jars())
# Now execute the class
return twister2_class(class_name, lib_jars, extra_jars, arguments, java_defines)
def submit_python_zip(cl_args, unknown_args):
# set up the system properties
java_system_props = setup_java_system_properties(cl_args)
props = read_client_properties(cl_args)
# execute main of the job to create the job definition
job_file = cl_args['job-file-name']
main_class = cl_args['job-class-name']
java_system_props += ["python_file=" + job_file, "main_file=" + main_class]
res = execute.twister2_class(
class_name="edu.iu.dsc.tws.python.PythonWorker",
lib_jars=config.get_twister2_libs(jars.job_jars()),
extra_jars=[],
args=tuple(unknown_args),
java_defines=java_system_props,
client_props=props)
result.render(res)
if not res.is_successful():
err_context = ("Failed to create job definition " \
"Please check logs for more information")
res.add_context(err_context)
return res
return res
def twister2_tar(class_name,
topology_tar,
arguments,
tmpdir_root,
java_defines,
client_props=None):
'''
:param class_name:
:param topology_tar:
:param arguments:
:param tmpdir_root:
:param java_defines:
:return:
'''
# Extract tar to a tmp folder.
tmpdir = tempfile.mkdtemp(dir=tmpdir_root, prefix='tmp')
topology_jar = os.path.basename(topology_tar).replace(".zip", "")
with ZipFile(topology_tar, 'r') as zipObj:
print(topology_tar, tmpdir, topology_jar)
# Extract all the contents of zip file in different directory
zipObj.extractall(tmpdir)
if java_defines is None:
java_defines = []
# Format all java -D options that need to be passed while running
# the class locally.
java_opts = ['-D' + opt for opt in java_defines]
extra_jars = [
os.path.join(tmpdir, topology_jar, "*.jar"),
os.path.join(tmpdir, topology_jar, "libs/*.jar")
]
for root, dirs, files in os.walk(os.path.join(tmpdir, topology_jar)):
for file in files:
if file.endswith('.jar'):
extra_jars.append(os.path.abspath(os.path.join(root, file)))
print(extra_jars)
lib_jars = config.get_twister2_libs(jars.job_jars())
print(lib_jars)
if client_props:
if 'twister2.client.debug' in client_props:
java_opts.append(client_props['twister2.client.debug'])
# Now execute the class
return twister2_class(class_name, lib_jars, extra_jars, arguments,
java_defines, client_props)
def submit_fatjar(cl_args, unknown_args):
'''
We use the packer to make a package for the jar and dump it
to a well-known location. We then run the main method of class
with the specified arguments. We pass arguments as an environment variable TWISTER2_OPTIONS.
This will run the jar file with the job_class_name. The submitter
inside will write out the job defn file to a location that
we specify. Then we write the job defn file to a well known
location. We then write to appropriate places in zookeeper
and launch the scheduler jobs
:param cl_args:
:param unknown_args:
:param tmp_dir:
:return:
'''
# set up the system properties
java_system_props = setup_java_system_properties(cl_args)
props = read_client_properties(cl_args)
# execute main of the job to create the job definition
job_file = cl_args['job-file-name']
main_class = cl_args['job-class-name']
res = execute.twister2_class(class_name=main_class,
lib_jars=config.get_twister2_libs(
jars.job_jars()),
extra_jars=[job_file],
args=tuple(unknown_args),
java_defines=java_system_props,
client_props=props)
result.render(res)
if not res.is_successful():
err_context = ("Failed to create job definition " \
"Please check logs for more information")
res.add_context(err_context)
return res
return res