def heron_tar(class_name, topology_tar, arguments, tmpdir_root, java_defines):
'''
:param class_name:
:param topology_tar:
:param arguments:
:param tmpdir_root:
:param java_defines:
:return:
'''
# Extract tar to a tmp folder.
tmpdir = tempfile.mkdtemp(dir=tmpdir_root, prefix='tmp')
with contextlib.closing(tarfile.open(topology_tar)) as tar:
tar.extractall(path=tmpdir)
# A tar generated by pants has all dependency jars under libs/
# in addition to the topology jar at top level. Pants keeps
# filename for jar and tar the same except for extension.
topology_jar = os.path.basename(topology_tar).replace(".tar.gz", "").replace(".tar", "") + ".jar"
extra_jars = [
os.path.join(tmpdir, "heron-instance.jar"),
os.path.join(tmpdir, topology_jar),
os.path.join(tmpdir, "*"),
os.path.join(tmpdir, "libs/*")
]
lib_jars = config.get_heron_libs(jars.topology_jars())
# Now execute the class
return heron_class(class_name, lib_jars, extra_jars, arguments, java_defines)
def heron_tar(class_name, topology_tar, arguments, tmpdir_root, java_defines):
'''
:param class_name:
:param topology_tar:
:param arguments:
:param tmpdir_root:
:param java_defines:
:return:
'''
# Extract tar to a tmp folder.
tmpdir = tempfile.mkdtemp(dir=tmpdir_root, prefix='tmp')
with contextlib.closing(tarfile.open(topology_tar)) as tar:
tar.extractall(path=tmpdir)
# A tar generated by pants has all dependency jars under libs/
# in addition to the topology jar at top level. Pants keeps
# filename for jar and tar the same except for extension.
topology_jar = os.path.basename(topology_tar).replace(
".tar.gz", "").replace(".tar", "") + ".jar"
extra_jars = [
os.path.join(tmpdir, "heron-instance.jar"),
os.path.join(tmpdir, topology_jar),
os.path.join(tmpdir, "*"),
os.path.join(tmpdir, "libs/*")
]
lib_jars = config.get_heron_libs(jars.topology_jars())
# Now execute the class
return heron_class(class_name, lib_jars, extra_jars, arguments,
java_defines)
def launch_a_topology(cl_args, tmp_dir, topology_file, topology_defn_file, topology_name):
'''
Launch a topology given topology jar, its definition file and configurations
:param cl_args:
:param tmp_dir:
:param topology_file:
:param topology_defn_file:
:return:
'''
# get the normalized path for topology.tar.gz
topology_pkg_path = config.normalized_class_path(os.path.join(tmp_dir, 'topology.tar.gz'))
# get the release yaml file
release_yaml_file = config.get_heron_release_file()
# create a tar package with the cluster configuration and generated config files
config_path = cl_args['config_path']
tar_pkg_files = [topology_file, topology_defn_file]
generated_config_files = [release_yaml_file, cl_args['override_config_file']]
config.create_tar(topology_pkg_path, tar_pkg_files, config_path, generated_config_files)
# pass the args to submitter main
args = [
"--cluster", cl_args['cluster'],
"--role", cl_args['role'],
"--environment", cl_args['environ'],
"--heron_home", config.get_heron_dir(),
"--config_path", config_path,
"--override_config_file", cl_args['override_config_file'],
"--release_file", release_yaml_file,
"--topology_package", topology_pkg_path,
"--topology_defn", topology_defn_file,
"--topology_bin", topology_file # pex file if pex specified
]
if Log.getEffectiveLevel() == logging.DEBUG:
args.append("--verbose")
lib_jars = config.get_heron_libs(
jars.scheduler_jars() + jars.uploader_jars() + jars.statemgr_jars() + jars.packing_jars()
)
extra_jars = cl_args['extra_launch_classpath'].split(':')
# invoke the submitter to submit and launch the topology
main_class = 'com.twitter.heron.scheduler.SubmitterMain'
resp = execute.heron_class(
class_name=main_class,
lib_jars=lib_jars,
extra_jars=extra_jars,
args=args,
java_defines=[])
err_context = "Failed to launch topology '%s'" % topology_name
succ_context = "Successfully launched topology '%s'" % topology_name
resp.add_context(err_context, succ_context)
return resp
def run(command, cl_args, action, extra_args=[], extra_lib_jars=[]):
'''
helper function to take action on topologies
:param command:
:param cl_args:
:param action: description of action taken
:return:
'''
topology_name = cl_args['topology-name']
new_args = [
"--cluster",
cl_args['cluster'],
"--role",
cl_args['role'],
"--environment",
cl_args['environ'],
"--submit_user",
cl_args['submit_user'],
"--heron_home",
config.get_heron_dir(),
"--config_path",
cl_args['config_path'],
"--override_config_file",
cl_args['override_config_file'],
"--release_file",
config.get_heron_release_file(),
"--topology_name",
topology_name,
"--command",
command,
]
new_args += extra_args
lib_jars = config.get_heron_libs(jars.scheduler_jars() +
jars.statemgr_jars())
lib_jars += extra_lib_jars
if Log.getEffectiveLevel() == logging.DEBUG:
new_args.append("--verbose")
# invoke the runtime manager to kill the topology
result = execute.heron_class(
'com.twitter.heron.scheduler.RuntimeManagerMain',
lib_jars,
extra_jars=[],
args=new_args)
err_msg = "Failed to %s: %s" % (action, topology_name)
succ_msg = "Successfully %s: %s" % (action, topology_name)
result.add_context(err_msg, succ_msg)
return result
def run(command, parser, cl_args, unknown_args, action):
'''
helper function to take action on topologies
:param command:
:param parser:
:param cl_args:
:param unknown_args:
:param action: description of action taken
:return:
'''
try:
topology_name = cl_args['topology-name']
new_args = [
"--cluster",
cl_args['cluster'],
"--role",
cl_args['role'],
"--environment",
cl_args['environ'],
"--heron_home",
config.get_heron_dir(),
"--config_path",
cl_args['config_path'],
"--override_config_file",
cl_args['override_config_file'],
"--release_file",
config.get_heron_release_file(),
"--topology_name",
topology_name,
"--command",
command,
]
if Log.getEffectiveLevel() == logging.DEBUG:
new_args.append("--verbose")
lib_jars = config.get_heron_libs(jars.scheduler_jars() +
jars.statemgr_jars())
# invoke the runtime manager to kill the topology
execute.heron_class('com.twitter.heron.scheduler.RuntimeManagerMain',
lib_jars,
extra_jars=[],
args=new_args)
except Exception:
Log.error('Failed to %s \'%s\'' % (action, topology_name))
return False
Log.info('Successfully %s \'%s\'' % (action, topology_name))
return True
def launch_a_topology(cl_args, tmp_dir, topology_file, topology_defn_file):
'''
Launch a topology given topology jar, its definition file and configurations
:param cl_args:
:param tmp_dir:
:param topology_file:
:param topology_defn_file:
:return:
'''
# get the normalized path for topology.tar.gz
topology_pkg_path = config.normalized_class_path(os.path.join(tmp_dir, 'topology.tar.gz'))
# get the release yaml file
release_yaml_file = config.get_heron_release_file()
# create a tar package with the cluster configuration and generated config files
config_path = cl_args['config_path']
tar_pkg_files = [topology_file, topology_defn_file]
generated_config_files = [release_yaml_file, cl_args['override_config_file']]
config.create_tar(topology_pkg_path, tar_pkg_files, config_path, generated_config_files)
# pass the args to submitter main
args = [
"--cluster", cl_args['cluster'],
"--role", cl_args['role'],
"--environment", cl_args['environ'],
"--heron_home", config.get_heron_dir(),
"--config_path", config_path,
"--override_config_file", cl_args['override_config_file'],
"--release_file", release_yaml_file,
"--topology_package", topology_pkg_path,
"--topology_defn", topology_defn_file,
"--topology_bin", topology_file # pex file if pex specified
]
if Log.getEffectiveLevel() == logging.DEBUG:
args.append("--verbose")
lib_jars = config.get_heron_libs(
jars.scheduler_jars() + jars.uploader_jars() + jars.statemgr_jars() + jars.packing_jars()
)
# invoke the submitter to submit and launch the topology
execute.heron_class(
'com.twitter.heron.scheduler.SubmitterMain',
lib_jars,
extra_jars=[],
args=args,
java_defines=[]
)
def run(command, parser, cl_args, unknown_args):
""" run the update command """
topology_name = cl_args['topology-name']
try:
new_args = [
"--cluster",
cl_args['cluster'],
"--role",
cl_args['role'],
"--environment",
cl_args['environ'],
"--heron_home",
config.get_heron_dir(),
"--config_path",
cl_args['config_path'],
"--override_config_file",
cl_args['override_config_file'],
"--release_file",
config.get_heron_release_file(),
"--topology_name",
topology_name,
"--command",
command,
"--component_parallelism",
','.join(cl_args['component_parallelism']),
]
if Log.getEffectiveLevel() == logging.DEBUG:
new_args.append("--verbose")
lib_jars = config.get_heron_libs(jars.scheduler_jars() +
jars.statemgr_jars() +
jars.packing_jars())
# invoke the runtime manager to kill the topology
execute.heron_class('com.twitter.heron.scheduler.RuntimeManagerMain',
lib_jars,
extra_jars=[],
args=new_args)
except Exception as ex:
Log.error('Failed to update topology \'%s\': %s', topology_name,
traceback.format_exc(ex))
return False
Log.info('Successfully updated topology \'%s\'' % topology_name)
return True
def run(command, parser, cl_args, unknown_args, action):
'''
helper function to take action on topologies
:param command:
:param parser:
:param cl_args:
:param unknown_args:
:param action: description of action taken
:return:
'''
try:
topology_name = cl_args['topology-name']
new_args = [
"--cluster", cl_args['cluster'],
"--role", cl_args['role'],
"--environment", cl_args['environ'],
"--heron_home", config.get_heron_dir(),
"--config_path", cl_args['config_path'],
"--override_config_file", cl_args['override_config_file'],
"--release_file", config.get_heron_release_file(),
"--topology_name", topology_name,
"--command", command,
]
if Log.getEffectiveLevel() == logging.DEBUG:
new_args.append("--verbose")
lib_jars = config.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars())
# invoke the runtime manager to kill the topology
execute.heron_class(
'com.twitter.heron.scheduler.RuntimeManagerMain',
lib_jars,
extra_jars=[],
args=new_args
)
except Exception:
Log.error('Failed to %s \'%s\'' % (action, topology_name))
return False
Log.info('Successfully executed %s \'%s\'' % (action, topology_name))
return True
def run_direct(command, cl_args, action, extra_args=[], extra_lib_jars=[]):
'''
helper function to take action on topologies
:param command:
:param cl_args:
:param action: description of action taken
:return:
'''
topology_name = cl_args['topology-name']
new_args = [
"--cluster", cl_args['cluster'],
"--role", cl_args['role'],
"--environment", cl_args['environ'],
"--submit_user", cl_args['submit_user'],
"--heron_home", config.get_heron_dir(),
"--config_path", cl_args['config_path'],
"--override_config_file", cl_args['override_config_file'],
"--release_file", config.get_heron_release_file(),
"--topology_name", topology_name,
"--command", command,
]
new_args += extra_args
lib_jars = config.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars())
lib_jars += extra_lib_jars
if Log.getEffectiveLevel() == logging.DEBUG:
new_args.append("--verbose")
# invoke the runtime manager to kill the topology
result = execute.heron_class(
'com.twitter.heron.scheduler.RuntimeManagerMain',
lib_jars,
extra_jars=[],
args=new_args
)
err_msg = "Failed to %s: %s" % (action, topology_name)
succ_msg = "Successfully %s: %s" % (action, topology_name)
result.add_context(err_msg, succ_msg)
return result
def run(command, parser, cl_args, unknown_args):
'''
:param command:
:param parser:
:param cl_args:
:param unknown_args:
:return:
'''
topology_name = cl_args['topology-name']
try:
container_id = cl_args['container-id']
new_args = [
"--cluster", cl_args['cluster'],
"--role", cl_args['role'],
"--environment", cl_args['environ'],
"--heron_home", config.get_heron_dir(),
"--config_path", cl_args['config_path'],
"--override_config_file", cl_args['override_config_file'],
"--release_file", config.get_heron_release_file(),
"--topology_name", topology_name,
"--command", command,
"--container_id", str(container_id)
]
lib_jars = config.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars())
# invoke the runtime manager to kill the topology
execute.heron_class(
'com.twitter.heron.scheduler.RuntimeManagerMain',
lib_jars,
extra_jars=[],
args=new_args
)
except Exception as ex:
Log.debug(traceback.format_exc(ex))
Log.error('Failed to restart topology \'%s\'' % topology_name)
return False
Log.info('Successfully restarted topology \'%s\'' % topology_name)
return True
def submit_fatjar(cl_args, unknown_args, tmp_dir):
'''
We use the packer to make a package for the jar and dump it
to a well-known location. We then run the main method of class
with the specified arguments. We pass arguments as an environment variable HERON_OPTIONS.
This will run the jar file with the topology_class_name. The submitter
inside will write out the topology defn file to a location that
we specify. Then we write the topology defn file to a well known
location. We then write to appropriate places in zookeeper
and launch the scheduler jobs
:param cl_args:
:param unknown_args:
:param tmp_dir:
:return:
'''
# execute main of the topology to create the topology definition
topology_file = cl_args['topology-file-name']
main_class = cl_args['topology-class-name']
res = execute.heron_class(
class_name=main_class,
lib_jars=config.get_heron_libs(jars.topology_jars()),
extra_jars=[topology_file],
args=tuple(unknown_args),
java_defines=cl_args['topology_main_jvm_property'])
result.render(res)
if not res.is_successful():
err_context = "Failed to create topology definition \
file when executing class '%s' of file '%s'" % (main_class,
topology_file)
res.add_context(err_context)
return res
results = launch_topologies(cl_args, topology_file, tmp_dir)
return results
def submit_fatjar(cl_args, unknown_args, tmp_dir):
'''
We use the packer to make a package for the jar and dump it
to a well-known location. We then run the main method of class
with the specified arguments. We pass arguments as an environment variable HERON_OPTIONS.
This will run the jar file with the topology_class_name. The submitter
inside will write out the topology defn file to a location that
we specify. Then we write the topology defn file to a well known
location. We then write to appropriate places in zookeeper
and launch the scheduler jobs
:param cl_args:
:param unknown_args:
:param tmp_dir:
:return:
'''
# execute main of the topology to create the topology definition
topology_file = cl_args['topology-file-name']
try:
execute.heron_class(
cl_args['topology-class-name'],
config.get_heron_libs(jars.topology_jars()),
extra_jars=[topology_file],
args=tuple(unknown_args),
java_defines=cl_args['topology_main_jvm_property'])
except Exception as ex:
Log.debug(traceback.format_exc(ex))
Log.error("Unable to execute topology main class")
return False
try:
launch_topologies(cl_args, topology_file, tmp_dir)
except Exception as ex:
return False
finally:
shutil.rmtree(tmp_dir)
return True
def submit_fatjar(cl_args, unknown_args, tmp_dir):
'''
We use the packer to make a package for the jar and dump it
to a well-known location. We then run the main method of class
with the specified arguments. We pass arguments as an environment variable HERON_OPTIONS.
This will run the jar file with the topology_class_name. The submitter
inside will write out the topology defn file to a location that
we specify. Then we write the topology defn file to a well known
location. We then write to appropriate places in zookeeper
and launch the scheduler jobs
:param cl_args:
:param unknown_args:
:param tmp_dir:
:return:
'''
# execute main of the topology to create the topology definition
topology_file = cl_args['topology-file-name']
main_class = cl_args['topology-class-name']
res = execute.heron_class(
class_name=main_class,
lib_jars=config.get_heron_libs(jars.topology_jars()),
extra_jars=[topology_file],
args=tuple(unknown_args),
java_defines=cl_args['topology_main_jvm_property'])
result.render(res)
if not res.is_successful():
err_context = ("Failed to create topology definition " \
"file when executing class '%s' of file '%s'") % (main_class, topology_file)
res.add_context(err_context)
return res
results = launch_topologies(cl_args, topology_file, tmp_dir)
return results
def run(command, parser, cl_args, unknown_args):
'''
:param command:
:param parser:
:param cl_args:
:param unknown_args:
:return:
'''
topology_name = cl_args['topology-name']
try:
container_id = cl_args['container-id']
new_args = [
"--cluster", cl_args['cluster'], "--role", cl_args['role'],
"--environment", cl_args['environ'], "--heron_home",
config.get_heron_dir(), "--config_path", cl_args['config_path'],
"--override_config_file", cl_args['override_config_file'],
"--release_file",
config.get_heron_release_file(), "--topology_name", topology_name,
"--command", command, "--container_id",
str(container_id)
]
lib_jars = config.get_heron_libs(jars.scheduler_jars() +
jars.statemgr_jars())
# invoke the runtime manager to kill the topology
execute.heron_class('com.twitter.heron.scheduler.RuntimeManagerMain',
lib_jars,
extra_jars=[],
args=new_args)
except Exception as ex:
Log.debug(traceback.format_exc(ex))
Log.error('Failed to restart topology \'%s\'' % topology_name)
return False
Log.info('Successfully restarted topology \'%s\'' % topology_name)
return True
def run(command, parser, cl_args, unknown_args):
""" run the update command """
topology_name = cl_args['topology-name']
try:
new_args = [
"--cluster", cl_args['cluster'],
"--role", cl_args['role'],
"--environment", cl_args['environ'],
"--heron_home", config.get_heron_dir(),
"--config_path", cl_args['config_path'],
"--override_config_file", cl_args['override_config_file'],
"--release_file", config.get_heron_release_file(),
"--topology_name", topology_name,
"--command", command,
"--component_parallelism", ','.join(cl_args['component_parallelism']),
]
if Log.getEffectiveLevel() == logging.DEBUG:
new_args.append("--verbose")
lib_jars = config.get_heron_libs(
jars.scheduler_jars() + jars.statemgr_jars() + jars.packing_jars()
)
# invoke the runtime manager to kill the topology
execute.heron_class(
'com.twitter.heron.scheduler.RuntimeManagerMain',
lib_jars,
extra_jars=[],
args=new_args
)
except Exception as ex:
Log.error('Failed to update topology \'%s\': %s', topology_name, traceback.format_exc(ex))
return False
Log.info('Successfully updated topology \'%s\'' % topology_name)
return True
def launch_a_topology(cl_args, tmp_dir, topology_file, topology_defn_file,
topology_name):
'''
Launch a topology given topology jar, its definition file and configurations
:param cl_args:
:param tmp_dir:
:param topology_file:
:param topology_defn_file:
:return:
'''
# get the normalized path for topology.tar.gz
topology_pkg_path = config.normalized_class_path(
os.path.join(tmp_dir, 'topology.tar.gz'))
# get the release yaml file
release_yaml_file = config.get_heron_release_file()
# create a tar package with the cluster configuration and generated config files
config_path = cl_args['config_path']
tar_pkg_files = [topology_file, topology_defn_file]
generated_config_files = [
release_yaml_file, cl_args['override_config_file']
]
config.create_tar(topology_pkg_path, tar_pkg_files, config_path,
generated_config_files)
# pass the args to submitter main
args = [
"--cluster",
cl_args['cluster'],
"--role",
cl_args['role'],
"--environment",
cl_args['environ'],
"--submit_user",
cl_args['submit_user'],
"--heron_home",
config.get_heron_dir(),
"--config_path",
config_path,
"--override_config_file",
cl_args['override_config_file'],
"--release_file",
release_yaml_file,
"--topology_package",
topology_pkg_path,
"--topology_defn",
topology_defn_file,
"--topology_bin",
os.path.basename(topology_file) # pex file if pex specified
]
if Log.getEffectiveLevel() == logging.DEBUG:
args.append("--verbose")
if cl_args["dry_run"]:
args.append("--dry_run")
if "dry_run_format" in cl_args:
args += ["--dry_run_format", cl_args["dry_run_format"]]
lib_jars = config.get_heron_libs(jars.scheduler_jars() +
jars.uploader_jars() +
jars.statemgr_jars() +
jars.packing_jars())
extra_jars = cl_args['extra_launch_classpath'].split(':')
# invoke the submitter to submit and launch the topology
main_class = 'com.twitter.heron.scheduler.SubmitterMain'
res = execute.heron_class(class_name=main_class,
lib_jars=lib_jars,
extra_jars=extra_jars,
args=args,
java_defines=[])
err_context = "Failed to launch topology '%s'" % topology_name
if cl_args["dry_run"]:
err_context += " in dry-run mode"
succ_context = "Successfully launched topology '%s'" % topology_name
if cl_args["dry_run"]:
succ_context += " in dry-run mode"
res.add_context(err_context, succ_context)
return res
