def heron_class(class_name, lib_jars, extra_jars=[], args=[], javaDefines=[]):
# Format all java -D options that need to be passed while running
# the class locally.
javaOpts = map(lambda opt: '-D' + opt, javaDefines)
# Construct the command line for the sub process to run
# Because of the way Python execute works,
# the java opts must be passed as part of the list
all_args = [
utils.get_java_path(), "-client", "-Xmx1g", opts.get_heron_config()] + \
javaOpts + \
["-cp", utils.get_classpath(lib_jars + extra_jars),
]
all_args += [class_name] + list(args)
# print the verbose message
if opts.verbose():
print('$> %s' % ' '.join(all_args))
# invoke the command with subprocess and print error message, if any
if not opts.trace_execution():
status = subprocess.call(all_args)
if status != 0:
err_str = "User main failed with status %d. Bailing out..." % status
raise RuntimeError(err_str)
def heron_class(class_name, lib_jars, extra_jars=[], args=[], javaDefines=[]):
# Format all java -D options that need to be passed while running
# the class locally.
javaOpts = map(lambda opt: '-D' + opt, javaDefines)
# Construct the command line for the sub process to run
# Because of the way Python execute works,
# the java opts must be passed as part of the list
all_args = [
utils.get_java_path(), "-client", "-Xmx1g", opts.get_heron_config()] + \
javaOpts + \
["-cp", utils.get_classpath(lib_jars + extra_jars),
]
all_args += [class_name] + list(args)
# print the verbose message
if opts.verbose():
print('$> %s' % ' '.join(all_args))
# invoke the command with subprocess and print error message, if any
if not opts.trace_execution():
status = subprocess.call(all_args)
if status != 0:
err_str = "User main failed with status %d. Bailing out..." % status
raise RuntimeError(err_str)
def main():
'''
Run the command
:return:
'''
# verify if the environment variables are correctly set
check_environment()
# create the argument parser
parser = create_parser()
# if no argument is provided, print help and exit
if len(sys.argv[1:]) == 0:
parser.print_help()
return 0
# insert the boolean values for some of the options
sys.argv = parse.insert_bool_values(sys.argv)
# parse the args
args, unknown_args = parser.parse_known_args()
command_line_args = vars(args)
try:
if command_line_args['verbose']:
opts.set_verbose()
if command_line_args['trace_execution']:
opts.set_trace_execution()
except:
pass
# command to be execute
command = command_line_args['subcommand']
# file resources to be cleaned when exit
files = []
if command != 'help' and command != 'version':
command_line_args = extract_common_args(command, parser, command_line_args)
# bail out if args are empty
if not command_line_args:
return 1
# register dirs cleanup function during exit
files.append(command_line_args['override_config_file'])
atexit.register(cleanup, files)
# print the input parameters, if verbose is enabled
if opts.verbose():
print command_line_args
start = time.time()
retcode = run(command, parser, command_line_args, unknown_args)
end = time.time()
if command != 'help':
sys.stdout.flush()
Log.info('Elapsed time: %.3fs.' % (end - start))
return 0 if retcode else 1
def main():
# verify if the environment variables are correctly set
check_environment()
# create the argument parser
parser = create_parser()
# if no argument is provided, print help and exit
if len(sys.argv[1:]) == 0:
parser.print_help()
return 0
# insert the boolean values for some of the options
sys.argv = parse.insert_bool_values(sys.argv)
# parse the args
args, unknown_args = parser.parse_known_args()
command_line_args = vars(args)
try:
if command_line_args['verbose']:
opts.set_verbose()
if command_line_args['trace_execution']:
opts.set_trace_execution()
except:
pass
# command to be execute
command = command_line_args['subcommand']
# file resources to be cleaned when exit
files = []
if command != 'help' and command != 'version':
command_line_args = extract_common_args(command, parser,
command_line_args)
# bail out if args are empty
if not command_line_args:
return 1
# register dirs cleanup function during exit
files.append(command_line_args['override_config_file'])
atexit.register(cleanup, files)
# print the input parameters, if verbose is enabled
if opts.verbose():
print command_line_args
start = time.time()
retcode = run(command, parser, command_line_args, unknown_args)
end = time.time()
if command != 'help':
sys.stdout.flush()
Log.info('Elapsed time: %.3fs.' % (end - start))
return 0 if retcode == True else 1
def run(command, parser, cl_args, unknown_args, action):
'''
helper function to take action on topologies
:param command:
:param parser:
:param cl_args:
:param unknown_args:
:param action: description of action taken
:return:
'''
try:
topology_name = cl_args['topology-name']
new_args = [
"--cluster",
cl_args['cluster'],
"--role",
cl_args['role'],
"--environment",
cl_args['environ'],
"--heron_home",
utils.get_heron_dir(),
"--config_path",
cl_args['config_path'],
"--override_config_file",
cl_args['override_config_file'],
"--release_file",
utils.get_heron_release_file(),
"--topology_name",
topology_name,
"--command",
command,
]
if opts.verbose():
new_args.append("--verbose")
lib_jars = utils.get_heron_libs(jars.scheduler_jars() +
jars.statemgr_jars())
# invoke the runtime manager to kill the topology
execute.heron_class('com.twitter.heron.scheduler.RuntimeManagerMain',
lib_jars,
extra_jars=[],
args=new_args)
except Exception:
Log.error('Failed to %s \'%s\'' % (action, topology_name))
return False
Log.info('Successfully %s \'%s\'' % (action, topology_name))
return True
def launch_a_topology(cl_args, tmp_dir, topology_file, topology_defn_file):
'''
Launch a topology given topology jar, its definition file and configurations
:param cl_args:
:param tmp_dir:
:param topology_file:
:param topology_defn_file:
:return:
'''
# get the normalized path for topology.tar.gz
topology_pkg_path = utils.normalized_class_path(os.path.join(tmp_dir, 'topology.tar.gz'))
# get the release yaml file
release_yaml_file = utils.get_heron_release_file()
# create a tar package with the cluster configuration and generated config files
config_path = cl_args['config_path']
tar_pkg_files = [topology_file, topology_defn_file]
generated_config_files = [release_yaml_file, cl_args['override_config_file']]
utils.create_tar(topology_pkg_path, tar_pkg_files, config_path, generated_config_files)
# pass the args to submitter main
args = [
"--cluster", cl_args['cluster'],
"--role", cl_args['role'],
"--environment", cl_args['environ'],
"--heron_home", utils.get_heron_dir(),
"--config_path", config_path,
"--override_config_file", cl_args['override_config_file'],
"--release_file", release_yaml_file,
"--topology_package", topology_pkg_path,
"--topology_defn", topology_defn_file,
"--topology_jar", topology_file
]
if opts.verbose():
args.append("--verbose")
lib_jars = utils.get_heron_libs(
jars.scheduler_jars() + jars.uploader_jars() + jars.statemgr_jars() + jars.packing_jars()
)
# invoke the submitter to submit and launch the topology
execute.heron_class(
'com.twitter.heron.scheduler.SubmitterMain',
lib_jars,
extra_jars=[],
args=args,
java_defines=[]
)
def launch_a_topology(cl_args, tmp_dir, topology_file, topology_defn_file):
'''
Launch a topology given topology jar, its definition file and configurations
:param cl_args:
:param tmp_dir:
:param topology_file:
:param topology_defn_file:
:return:
'''
# get the normalized path for topology.tar.gz
topology_pkg_path = utils.normalized_class_path(
os.path.join(tmp_dir, 'topology.tar.gz'))
# get the release yaml file
release_yaml_file = utils.get_heron_release_file()
# create a tar package with the cluster configuration and generated config files
config_path = cl_args['config_path']
tar_pkg_files = [topology_file, topology_defn_file]
generated_config_files = [
release_yaml_file, cl_args['override_config_file']
]
utils.create_tar(topology_pkg_path, tar_pkg_files, config_path,
generated_config_files)
# pass the args to submitter main
args = [
"--cluster", cl_args['cluster'], "--role", cl_args['role'],
"--environment", cl_args['environ'], "--heron_home",
utils.get_heron_dir(), "--config_path", config_path,
"--override_config_file", cl_args['override_config_file'],
"--release_file", release_yaml_file, "--topology_package",
topology_pkg_path, "--topology_defn", topology_defn_file,
"--topology_jar", topology_file
]
if opts.verbose():
args.append("--verbose")
lib_jars = utils.get_heron_libs(jars.scheduler_jars() +
jars.uploader_jars() +
jars.statemgr_jars() + jars.packing_jars())
# invoke the submitter to submit and launch the topology
execute.heron_class('com.twitter.heron.scheduler.SubmitterMain',
lib_jars,
extra_jars=[],
args=args,
java_defines=[])
def heron_class(class_name,
lib_jars,
extra_jars=None,
args=None,
java_defines=None):
'''
Execute a heron class given the args and the jars needed for class path
:param class_name:
:param lib_jars:
:param extra_jars:
:param args:
:param javaDefines:
:return:
'''
# default optional params to empty list if not provided
if extra_jars is None:
extra_jars = []
if args is None:
args = []
if java_defines is None:
java_defines = []
# Format all java -D options that need to be passed while running
# the class locally.
java_opts = ['-D' + opt for opt in java_defines]
# Construct the command line for the sub process to run
# Because of the way Python execute works,
# the java opts must be passed as part of the list
all_args = [utils.get_java_path(), "-client", "-Xmx1g", opts.get_heron_config()] + \
java_opts + \
["-cp", utils.get_classpath(lib_jars + extra_jars)]
all_args += [class_name] + list(args)
# print the verbose message
if opts.verbose():
print '$> %s' % ' '.join(all_args)
# invoke the command with subprocess and print error message, if any
if not opts.trace_execution():
status = subprocess.call(all_args)
if status != 0:
err_str = "User main failed with status %d. Bailing out..." % status
raise RuntimeError(err_str)
def run(command, parser, cl_args, unknown_args):
try:
topology_name = cl_args['topology-name']
new_args = [
"--cluster",
cl_args['cluster'],
"--role",
cl_args['role'],
"--environment",
cl_args['environ'],
"--heron_home",
utils.get_heron_dir(),
"--config_path",
cl_args['config_path'],
"--override_config_file",
cl_args['override_config_file'],
"--release_file",
utils.get_heron_release_file(),
"--topology_name",
topology_name,
"--command",
command,
]
if opts.verbose():
new_args.append("--verbose")
lib_jars = utils.get_heron_libs(jars.scheduler_jars() +
jars.statemgr_jars())
# invoke the runtime manager to kill the topology
execute.heron_class('com.twitter.heron.scheduler.RuntimeManagerMain',
lib_jars,
extra_jars=[],
args=new_args)
except Exception as ex:
print 'Error: %s' % str(ex)
Log.error('Failed to activate topology \'%s\'' % topology_name)
return False
Log.info('Successfully activated topology \'%s\'' % topology_name)
return True
def run(command, parser, cl_args, unknown_args, action):
'''
helper function to take action on topologies
:param command:
:param parser:
:param cl_args:
:param unknown_args:
:param action: description of action taken
:return:
'''
try:
topology_name = cl_args['topology-name']
new_args = [
"--cluster", cl_args['cluster'],
"--role", cl_args['role'],
"--environment", cl_args['environ'],
"--heron_home", utils.get_heron_dir(),
"--config_path", cl_args['config_path'],
"--override_config_file", cl_args['override_config_file'],
"--release_file", utils.get_heron_release_file(),
"--topology_name", topology_name,
"--command", command,
]
if opts.verbose():
new_args.append("--verbose")
lib_jars = utils.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars())
# invoke the runtime manager to kill the topology
execute.heron_class(
'com.twitter.heron.scheduler.RuntimeManagerMain',
lib_jars,
extra_jars=[],
args=new_args
)
except Exception:
Log.error('Failed to %s \'%s\'' % (action, topology_name))
return False
Log.info('Successfully %s \'%s\'' % (action, topology_name))
return True
def heron_class(class_name, lib_jars, extra_jars=None, args=None, java_defines=None):
'''
Execute a heron class given the args and the jars needed for class path
:param class_name:
:param lib_jars:
:param extra_jars:
:param args:
:param javaDefines:
:return:
'''
# default optional params to empty list if not provided
if extra_jars is None:
extra_jars = []
if args is None:
args = []
if java_defines is None:
java_defines = []
# Format all java -D options that need to be passed while running
# the class locally.
java_opts = ['-D' + opt for opt in java_defines]
# Construct the command line for the sub process to run
# Because of the way Python execute works,
# the java opts must be passed as part of the list
all_args = [utils.get_java_path(), "-client", "-Xmx1g", opts.get_heron_config()] + \
java_opts + \
["-cp", utils.get_classpath(lib_jars + extra_jars)]
all_args += [class_name] + list(args)
# print the verbose message
if opts.verbose():
print '$> %s' % ' '.join(all_args)
# invoke the command with subprocess and print error message, if any
if not opts.trace_execution():
status = subprocess.call(all_args)
if status != 0:
err_str = "User main failed with status %d. Bailing out..." % status
raise RuntimeError(err_str)
def run(command, parser, cl_args, unknown_args):
try:
topology_name = cl_args["topology-name"]
new_args = [
"--cluster",
cl_args["cluster"],
"--role",
cl_args["role"],
"--environment",
cl_args["environ"],
"--heron_home",
utils.get_heron_dir(),
"--config_path",
cl_args["config_path"],
"--override_config_file",
cl_args["override_config_file"],
"--release_file",
utils.get_heron_release_file(),
"--topology_name",
topology_name,
"--command",
command,
]
if opts.verbose():
new_args.append("--verbose")
lib_jars = utils.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars())
# invoke the runtime manager to kill the topology
execute.heron_class("com.twitter.heron.scheduler.RuntimeManagerMain", lib_jars, extra_jars=[], args=new_args)
except Exception as ex:
Log.error("Failed to kill topology '%s'" % topology_name)
return False
Log.info("Successfully killed topology '%s'" % topology_name)
return True
def run(command, parser, cl_args, unknown_args):
try:
topology_name = cl_args['topology-name']
new_args = [
"--cluster", cl_args['cluster'],
"--role", cl_args['role'],
"--environment", cl_args['environ'],
"--heron_home", utils.get_heron_dir(),
"--config_path", cl_args['config_path'],
"--override_config_file", cl_args['override_config_file'],
"--release_file", utils.get_heron_release_file(),
"--topology_name", topology_name,
"--command", command,
]
if opts.verbose():
new_args.append("--verbose")
lib_jars = utils.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars())
# invoke the runtime manager to kill the topology
execute.heron_class(
'com.twitter.heron.scheduler.RuntimeManagerMain',
lib_jars,
extra_jars=[],
args= new_args
)
except Exception as ex:
print 'Error: %s' % str(ex)
Log.error('Failed to deactivate topology \'%s\'' % topology_name)
return False
Log.info('Successfully deactivated topology \'%s\'' % topology_name)
return True