def submit_tar(cl_args, unknown_args, tmp_dir):
'''
Extract and execute the java files inside the tar and then add topology
definition file created by running submitTopology
We use the packer to make a package for the tar and dump it
to a well-known location. We then run the main method of class
with the specified arguments. We pass arguments as an environment variable HERON_OPTIONS.
This will run the jar file with the topology class name.
The submitter inside will write out the topology defn file to a location
that we specify. Then we write the topology defn file to a well known
packer location. We then write to appropriate places in zookeeper
and launch the aurora jobs
:param cl_args:
:param unknown_args:
:param tmp_dir:
:return:
'''
# execute main of the topology to create the topology definition
topology_file = cl_args['topology-file-name']
java_defines = cl_args['topology_main_jvm_property']
main_class = cl_args['topology-class-name']
res = execute.heron_tar(main_class, topology_file, tuple(unknown_args),
tmp_dir, java_defines)
result.render(res)
if not result.is_successful(res):
err_context = ("Failed to create topology definition " \
"file when executing class '%s' of file '%s'") % (main_class, topology_file)
res.add_context(err_context)
return res
return launch_topologies(cl_args, topology_file, tmp_dir)
def main():
'''
Run the command
:return:
'''
# verify if the environment variables are correctly set
check_environment()
# create the argument parser
parser = create_parser()
# if no argument is provided, print help and exit
if len(sys.argv[1:]) == 0:
parser.print_help()
return 0
# insert the boolean values for some of the options
sys.argv = config.insert_bool_values(sys.argv)
try:
# parse the args
args, unknown_args = parser.parse_known_args()
except ValueError as ex:
Log.error("Error while parsing arguments: %s", str(ex))
Log.debug(traceback.format_exc())
sys.exit(1)
command_line_args = vars(args)
# command to be execute
command = command_line_args['subcommand']
if command not in ('help', 'version'):
log.set_logging_level(command_line_args)
command_line_args = extract_common_args(command, parser,
command_line_args)
# bail out if args are empty
if not command_line_args:
return 1
# register dirs cleanup function during exit
cleaned_up_files.append(command_line_args['override_config_file'])
atexit.register(cleanup, cleaned_up_files)
# print the input parameters, if verbose is enabled
Log.debug(command_line_args)
start = time.time()
results = run(command, parser, command_line_args, unknown_args)
if command not in ('help', 'version'):
result.render(results)
end = time.time()
if command not in ('help', 'version'):
sys.stdout.flush()
Log.debug('Elapsed time: %.3fs.', (end - start))
return 0 if result.is_successful(results) else 1
def main():
'''
Run the command
:return:
'''
# verify if the environment variables are correctly set
check_environment()
# create the argument parser
parser = create_parser()
# if no argument is provided, print help and exit
if len(sys.argv[1:]) == 0:
parser.print_help()
return 0
# insert the boolean values for some of the options
sys.argv = config.insert_bool_values(sys.argv)
try:
# parse the args
args, unknown_args = parser.parse_known_args()
except ValueError as ex:
Log.error("Error while parsing arguments: %s", str(ex))
Log.debug(traceback.format_exc())
sys.exit(1)
command_line_args = vars(args)
# command to be execute
command = command_line_args['subcommand']
if command not in ('help', 'version'):
log.set_logging_level(command_line_args)
command_line_args = extract_common_args(command, parser, command_line_args)
# bail out if args are empty
if not command_line_args:
return 1
# register dirs cleanup function during exit
cleaned_up_files.append(command_line_args['override_config_file'])
atexit.register(cleanup, cleaned_up_files)
# print the input parameters, if verbose is enabled
Log.debug(command_line_args)
start = time.time()
results = run(command, parser, command_line_args, unknown_args)
if command not in ('help', 'version'):
result.render(results)
end = time.time()
if command not in ('help', 'version'):
sys.stdout.flush()
Log.info('Elapsed time: %.3fs.', (end - start))
return 0 if result.isAllSuccessful(results) else 1
def submit_cpp(cl_args, unknown_args, tmp_dir):
# execute main of the topology to create the topology definition
topology_file = cl_args['topology-file-name']
topology_binary_name = cl_args['topology-class-name']
res = execute.heron_cpp(topology_binary_name, tuple(unknown_args))
result.render(res)
if not result.is_successful(res):
err_context = ("Failed to create topology definition " \
"file when executing cpp binary '%s'") % (topology_binary_name)
res.add_context(err_context)
return res
return launch_topologies(cl_args, topology_file, tmp_dir)
def submit_cpp(cl_args, unknown_args, tmp_dir):
# execute main of the topology to create the topology definition
topology_file = cl_args['topology-file-name']
topology_binary_name = cl_args['topology-class-name']
res = execute.heron_cpp(topology_binary_name, tuple(unknown_args))
result.render(res)
if not result.is_successful(res):
err_context = ("Failed to create topology definition " \
"file when executing cpp binary '%s'") % (topology_binary_name)
res.add_context(err_context)
return res
return launch_topologies(cl_args, topology_file, tmp_dir)
def submit_fatjar(cl_args, unknown_args, tmp_dir):
'''
We use the packer to make a package for the jar and dump it
to a well-known location. We then run the main method of class
with the specified arguments. We pass arguments as an environment variable HERON_OPTIONS.
This will run the jar file with the topology_class_name. The submitter
inside will write out the topology defn file to a location that
we specify. Then we write the topology defn file to a well known
location. We then write to appropriate places in zookeeper
and launch the scheduler jobs
:param cl_args:
:param unknown_args:
:param tmp_dir:
:return:
'''
# execute main of the topology to create the topology definition
topology_file = cl_args['topology-file-name']
main_class = cl_args['topology-class-name']
res = execute.heron_class(
class_name=main_class,
lib_jars=config.get_heron_libs(jars.topology_jars()),
extra_jars=[topology_file],
args=tuple(unknown_args),
java_defines=cl_args['topology_main_jvm_property'])
result.render(res)
if not res.is_successful():
err_context = "Failed to create topology definition \
file when executing class '%s' of file '%s'" % (main_class,
topology_file)
res.add_context(err_context)
return res
results = launch_topologies(cl_args, topology_file, tmp_dir)
return results
def submit_tar(cl_args, unknown_args, tmp_dir):
'''
Extract and execute the java files inside the tar and then add topology
definition file created by running submitTopology
We use the packer to make a package for the tar and dump it
to a well-known location. We then run the main method of class
with the specified arguments. We pass arguments as an environment variable HERON_OPTIONS.
This will run the jar file with the topology class name.
The submitter inside will write out the topology defn file to a location
that we specify. Then we write the topology defn file to a well known
packer location. We then write to appropriate places in zookeeper
and launch the aurora jobs
:param cl_args:
:param unknown_args:
:param tmp_dir:
:return:
'''
# execute main of the topology to create the topology definition
topology_file = cl_args['topology-file-name']
java_defines = cl_args['topology_main_jvm_property']
main_class = cl_args['topology-class-name']
res = execute.heron_tar(
main_class,
topology_file,
tuple(unknown_args),
tmp_dir,
java_defines)
result.render(res)
if not res.is_successful():
err_context = ("Failed to create topology definition " \
"file when executing class '%s' of file '%s'") % (main_class, topology_file)
res.add_context(err_context)
return res
return launch_topologies(cl_args, topology_file, tmp_dir)
def submit_fatjar(cl_args, unknown_args, tmp_dir):
'''
We use the packer to make a package for the jar and dump it
to a well-known location. We then run the main method of class
with the specified arguments. We pass arguments as an environment variable HERON_OPTIONS.
This will run the jar file with the topology_class_name. The submitter
inside will write out the topology defn file to a location that
we specify. Then we write the topology defn file to a well known
location. We then write to appropriate places in zookeeper
and launch the scheduler jobs
:param cl_args:
:param unknown_args:
:param tmp_dir:
:return:
'''
# execute main of the topology to create the topology definition
topology_file = cl_args['topology-file-name']
main_class = cl_args['topology-class-name']
res = execute.heron_class(
class_name=main_class,
lib_jars=config.get_heron_libs(jars.topology_jars()),
extra_jars=[topology_file],
args=tuple(unknown_args),
java_defines=cl_args['topology_main_jvm_property'])
result.render(res)
if not res.is_successful():
err_context = ("Failed to create topology definition " \
"file when executing class '%s' of file '%s'") % (main_class, topology_file)
res.add_context(err_context)
return res
results = launch_topologies(cl_args, topology_file, tmp_dir)
return results
def execute(handlers, local_commands):
'''
Run the command
:return:
'''
# verify if the environment variables are correctly set
check_environment()
# create the argument parser
parser = create_parser(handlers)
# if no argument is provided, print help and exit
if len(sys.argv[1:]) == 0:
parser.print_help()
return 0
# insert the boolean values for some of the options
sys.argv = config.insert_bool_values(sys.argv)
try:
# parse the args
args, unknown_args = parser.parse_known_args()
except ValueError as ex:
Log.error("Error while parsing arguments: %s", str(ex))
Log.debug(traceback.format_exc())
sys.exit(1)
command_line_args = vars(args)
# set log level
log.set_logging_level(command_line_args)
Log.debug("Input Command Line Args: %s", command_line_args)
# command to be execute
command = command_line_args['subcommand']
is_local_command = command in local_commands
if command == 'version':
results = run(handlers, command, parser, command_line_args, unknown_args)
return 0 if result.is_successful(results) else 1
if not is_local_command:
log.set_logging_level(command_line_args)
Log.debug("Input Command Line Args: %s", command_line_args)
# determine the mode of deployment
command_line_args = extract_common_args(command, parser, command_line_args)
command_line_args = deployment_mode(command, parser, command_line_args)
# bail out if args are empty
if not command_line_args:
return 1
# register dirs cleanup function during exit
if command_line_args['deploy_mode'] == config.DIRECT_MODE and command != "version":
cleaned_up_files.append(command_line_args['override_config_file'])
atexit.register(cleanup, cleaned_up_files)
# print the input parameters, if verbose is enabled
Log.debug("Processed Command Line Args: %s", command_line_args)
start = time.time()
results = run(handlers, command, parser, command_line_args, unknown_args)
if not is_local_command:
result.render(results)
end = time.time()
if not is_local_command:
sys.stdout.flush()
Log.debug('Elapsed time: %.3fs.', (end - start))
return 0 if result.is_successful(results) else 1