def submit_fatjar(cl_args, unknown_args, tmp_dir):
# execute main of the topology to create the topology definition
topology_file = cl_args['topology-file-name']
try:
execute.heron_class(
cl_args['topology-class-name'],
utils.get_heron_libs(jars.topology_jars()),
extra_jars = [topology_file],
args = tuple(unknown_args),
javaDefines = cl_args['javaDefines'])
except Exception as ex:
Log.error("Unable to execute topology main class")
return False
try:
launch_topologies(cl_args, topology_file, tmp_dir)
except Exception as ex:
return False
finally:
shutil.rmtree(tmp_dir)
return True
def heron_tar(class_name, topology_tar, arguments, tmpdir_root, javaDefines):
# Extract tar to a tmp folder.
tmpdir = tempfile.mkdtemp(dir=tmpdir_root, prefix='tmp')
with contextlib.closing(tarfile.open(topology_tar)) as tar:
tar.extractall(path=tmpdir)
# A tar generated by pants has all dependency jars under libs/
# in addition to the topology jar at top level. Pants keeps
# filename for jar and tar the same except for extension.
# TODO: We need to extract heron-core to separate folder.
topology_jar = os.path.basename(topology_tar).replace(
".tar.gz", "").replace(".tar", "") + ".jar"
extra_jars = [
os.path.join(tmpdir, "heron-instance.jar"),
os.path.join(tmpdir, topology_jar),
os.path.join(tmpdir, "*"),
os.path.join(tmpdir, "libs/*")
]
lib_jars = utils.get_heron_libs(jars.topology_jars())
# Now execute the class
heron_class(class_name, lib_jars, extra_jars, arguments, javaDefines)
def heron_tar(class_name, topology_tar, arguments, tmpdir_root, javaDefines):
# Extract tar to a tmp folder.
tmpdir = tempfile.mkdtemp(dir=tmpdir_root, prefix='tmp')
with contextlib.closing(tarfile.open(topology_tar)) as tar:
tar.extractall(path=tmpdir)
# A tar generated by pants has all dependency jars under libs/
# in addition to the topology jar at top level. Pants keeps
# filename for jar and tar the same except for extension.
# TODO: We need to extract heron-core to separate folder.
topology_jar = os.path.basename(topology_tar).replace(".tar.gz", "").replace(".tar", "") + ".jar"
extra_jars = [
os.path.join(tmpdir, "heron-instance.jar"),
os.path.join(tmpdir, topology_jar),
os.path.join(tmpdir, "*"),
os.path.join(tmpdir, "libs/*")
]
lib_jars = utils.get_heron_libs(jars.topology_jars())
# Now execute the class
heron_class(class_name, lib_jars, extra_jars, arguments, javaDefines)
def run(command, parser, cl_args, unknown_args):
try:
topology_name = cl_args['topology-name']
new_args = [
"--cluster",
cl_args['cluster'],
"--role",
cl_args['role'],
"--environment",
cl_args['environ'],
"--heron_home",
utils.get_heron_dir(),
"--config_path",
cl_args['config_path'],
"--override_config_file",
cl_args['override_config_file'],
"--release_file",
utils.get_heron_release_file(),
"--topology_name",
topology_name,
"--command",
command,
]
if opts.verbose():
new_args.append("--verbose")
lib_jars = utils.get_heron_libs(jars.scheduler_jars() +
jars.statemgr_jars())
# invoke the runtime manager to kill the topology
execute.heron_class('com.twitter.heron.scheduler.RuntimeManagerMain',
lib_jars,
extra_jars=[],
args=new_args)
except Exception as ex:
print 'Error: %s' % str(ex)
Log.error('Failed to activate topology \'%s\'' % topology_name)
return False
Log.info('Successfully activated topology \'%s\'' % topology_name)
return True
def launch_a_topology(cl_args, tmp_dir, topology_file, topology_defn_file):
# get the normalized path for topology.tar.gz
topology_pkg_path = utils.normalized_class_path(os.path.join(tmp_dir, 'topology.tar.gz'))
# get the release yaml file
release_yaml_file = utils.get_heron_release_file()
# create a tar package with the cluster configuration and generated config files
config_path = cl_args['config_path']
tar_pkg_files = [topology_file, topology_defn_file]
generated_config_files = [release_yaml_file, cl_args['override_config_file']]
utils.create_tar(topology_pkg_path, tar_pkg_files, config_path, generated_config_files)
# pass the args to submitter main
args = [
"--cluster", cl_args['cluster'],
"--role", cl_args['role'],
"--environment", cl_args['environ'],
"--heron_home", utils.get_heron_dir(),
"--config_path", config_path,
"--override_config_file", cl_args['override_config_file'],
"--release_file", release_yaml_file,
"--topology_package", topology_pkg_path,
"--topology_defn", topology_defn_file,
"--topology_jar", topology_file
]
if opts.verbose():
args.append("--verbose")
lib_jars = utils.get_heron_libs(
jars.scheduler_jars() + jars.uploader_jars() + jars.statemgr_jars() + jars.packing_jars()
)
# invoke the submitter to submit and launch the topology
execute.heron_class(
'com.twitter.heron.scheduler.SubmitterMain',
lib_jars,
extra_jars=[],
args = args,
javaDefines = cl_args['javaDefines']
)
def run(command, parser, cl_args, unknown_args):
try:
topology_name = cl_args["topology-name"]
new_args = [
"--cluster",
cl_args["cluster"],
"--role",
cl_args["role"],
"--environment",
cl_args["environ"],
"--heron_home",
utils.get_heron_dir(),
"--config_path",
cl_args["config_path"],
"--override_config_file",
cl_args["override_config_file"],
"--release_file",
utils.get_heron_release_file(),
"--topology_name",
topology_name,
"--command",
command,
]
if opts.verbose():
new_args.append("--verbose")
lib_jars = utils.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars())
# invoke the runtime manager to kill the topology
execute.heron_class("com.twitter.heron.scheduler.RuntimeManagerMain", lib_jars, extra_jars=[], args=new_args)
except Exception as ex:
Log.error("Failed to kill topology '%s'" % topology_name)
return False
Log.info("Successfully killed topology '%s'" % topology_name)
return True
def run(command, parser, cl_args, unknown_args):
try:
topology_name = cl_args['topology-name']
new_args = [
"--cluster", cl_args['cluster'],
"--role", cl_args['role'],
"--environment", cl_args['environ'],
"--heron_home", utils.get_heron_dir(),
"--config_path", cl_args['config_path'],
"--override_config_file", cl_args['override_config_file'],
"--release_file", utils.get_heron_release_file(),
"--topology_name", topology_name,
"--command", command,
]
if opts.verbose():
new_args.append("--verbose")
lib_jars = utils.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars())
# invoke the runtime manager to kill the topology
execute.heron_class(
'com.twitter.heron.scheduler.RuntimeManagerMain',
lib_jars,
extra_jars=[],
args= new_args
)
except Exception as ex:
print 'Error: %s' % str(ex)
Log.error('Failed to deactivate topology \'%s\'' % topology_name)
return False
Log.info('Successfully deactivated topology \'%s\'' % topology_name)
return True