def run(command, parser, cl_args, unknown_args):
'''
Submits the topology to the scheduler
* Depending on the topology file name extension, we treat the file as a
fatjar (if the ext is .jar) or a tar file (if the ext is .tar/.tar.gz).
* We upload the topology file to the packer, update zookeeper and launch
scheduler jobs representing that topology
* You can see your topology in Heron UI
:param command:
:param parser:
:param cl_args:
:param unknown_args:
:return:
'''
# get the topology file name
topology_file = cl_args['topology-file-name']
# check to see if the topology file exists
if not os.path.isfile(topology_file):
Log.error("Topology jar|tar|pex file %s does not exist" %
topology_file)
return False
# check if it is a valid file type
jar_type = topology_file.endswith(".jar")
tar_type = topology_file.endswith(".tar") or topology_file.endswith(
".tar.gz")
pex_type = topology_file.endswith(".pex")
if not jar_type and not tar_type and not pex_type:
Log.error(
"Unknown file type. Please use .tar or .tar.gz or .jar or .pex file"
)
return False
# create a temporary directory for topology definition file
tmp_dir = tempfile.mkdtemp()
# if topology needs to be launched in deactivated state, do it so
if cl_args['deploy_deactivated']:
initial_state = topology_pb2.TopologyState.Name(topology_pb2.PAUSED)
else:
initial_state = topology_pb2.TopologyState.Name(topology_pb2.RUNNING)
# set the tmp dir and deactivated state in global options
opts.set_config('cmdline.topologydefn.tmpdirectory', tmp_dir)
opts.set_config('cmdline.topology.initial.state', initial_state)
# check the extension of the file name to see if it is tar/jar file.
if jar_type:
return submit_fatjar(cl_args, unknown_args, tmp_dir)
elif tar_type:
return submit_tar(cl_args, unknown_args, tmp_dir)
elif pex_type:
return submit_pex(cl_args, unknown_args, tmp_dir)
return False
def test_many_opts(self):
opts.clear_config()
for k in range(1, 100):
key = "key-%d" % (k)
value = "value-%d" % (k)
opts.set_config(key, value)
for k in range(1, 100):
key = "key-%d" % (k)
value = "value-%d" % (k)
self.assertEqual(value, opts.get_config(key))
def run(command, parser, cl_args, unknown_args):
'''
Submits the topology to the scheduler
* Depending on the topology file name extension, we treat the file as a
fatjar (if the ext is .jar) or a tar file (if the ext is .tar/.tar.gz).
* We upload the topology file to the packer, update zookeeper and launch
scheduler jobs representing that topology
* You can see your topology in Heron UI
:param command:
:param parser:
:param cl_args:
:param unknown_args:
:return:
'''
# get the topology file name
topology_file = cl_args['topology-file-name']
# check to see if the topology file exists
if not os.path.isfile(topology_file):
Log.error("Topology jar|tar|pex file %s does not exist" % topology_file)
return False
# check if it is a valid file type
jar_type = topology_file.endswith(".jar")
tar_type = topology_file.endswith(".tar") or topology_file.endswith(".tar.gz")
pex_type = topology_file.endswith(".pex")
if not jar_type and not tar_type and not pex_type:
Log.error("Unknown file type. Please use .tar or .tar.gz or .jar or .pex file")
return False
# create a temporary directory for topology definition file
tmp_dir = tempfile.mkdtemp()
# if topology needs to be launched in deactivated state, do it so
if cl_args['deploy_deactivated']:
initial_state = topology_pb2.TopologyState.Name(topology_pb2.PAUSED)
else:
initial_state = topology_pb2.TopologyState.Name(topology_pb2.RUNNING)
# set the tmp dir and deactivated state in global options
opts.set_config('cmdline.topologydefn.tmpdirectory', tmp_dir)
opts.set_config('cmdline.topology.initial.state', initial_state)
# check the extension of the file name to see if it is tar/jar file.
if jar_type:
return submit_fatjar(cl_args, unknown_args, tmp_dir)
elif tar_type:
return submit_tar(cl_args, unknown_args, tmp_dir)
elif pex_type:
return submit_pex(cl_args, unknown_args, tmp_dir)
return False
def run(command, parser, cl_args, unknown_args):
# get the topology file name
topology_file = cl_args['topology-file-name']
# check to see if the topology file exists
if not os.path.isfile(topology_file):
Log.error("Topology jar|tar file %s does not exist" % topology_file)
return False
# check if it is a valid file type
jar_type = topology_file.endswith(".jar")
tar_type = topology_file.endswith(".tar") or topology_file.endswith(".tar.gz")
if not jar_type and not tar_type:
Log.error("Unknown file type. Please use .tar or .tar.gz or .jar file")
return False
# create a temporary directory for topology definition file
tmp_dir = tempfile.mkdtemp()
# if topology needs to be launched in deactivated state, do it so
if cl_args['deploy_deactivated']:
initial_state = topology_pb2.TopologyState.Name(topology_pb2.PAUSED)
else:
initial_state = topology_pb2.TopologyState.Name(topology_pb2.RUNNING)
# set the tmp dir and deactivated state in global options
opts.set_config('cmdline.topologydefn.tmpdirectory', tmp_dir)
opts.set_config('cmdline.topology.initial.state', initial_state)
# check the extension of the file name to see if it is tar/jar file.
if jar_type:
return submit_fatjar(cl_args, unknown_args, tmp_dir)
elif tar_type:
return submit_tar(cl_args, unknown_args, tmp_dir)
return False
def heron_pex(topology_pex, topology_class_name, args=None):
Log.debug("Importing %s from %s" % (topology_class_name, topology_pex))
if topology_class_name == '-':
# loading topology by running its main method (if __name__ == "__main__")
heron_env = os.environ.copy()
heron_env['HERON_OPTIONS'] = opts.get_heron_config()
cmd = [topology_pex]
if args is not None:
cmd.extend(args)
Log.debug('$> %s' % ' '.join(cmd))
Log.debug('Heron options: %s' % str(heron_env['HERON_OPTIONS']))
# invoke the command with subprocess and print error message, if any
status = subprocess.call(cmd, env=heron_env)
if status != 0:
err_str = "Topology failed to be loaded from the given pex, with status: %d. Bailing out..." \
% status
raise RuntimeError(err_str)
else:
try:
# loading topology from Topology's subclass (no main method)
# to support specifying the name of topology
Log.debug("args: %s" % args)
if args is not None and isinstance(args, (list, tuple)) and len(args) > 0:
opts.set_config('cmdline.topology.name', args[0])
os.environ["HERON_OPTIONS"] = opts.get_heron_config()
Log.debug("Heron options: %s" % os.environ["HERON_OPTIONS"])
pex_loader.load_pex(topology_pex)
topology_class = pex_loader.import_and_get_class(topology_pex, topology_class_name)
topology_class.write()
except Exception:
Log.debug(traceback.format_exc())
err_str = "Topology failed to be loaded from the given pex. Bailing out..."
raise RuntimeError(err_str)
def test_clear_opts(self):
opts.clear_config()
opts.set_config('YY', 'yy')
self.assertEqual('yy', opts.get_config('YY'))
opts.clear_config()
self.assertEqual(None, opts.get_config('YY'))
def test_non_exist_key(self):
opts.clear_config()
opts.set_config('XX', 'xx')
self.assertEqual(None, opts.get_config('YY'))
def test_two_opts(self):
opts.clear_config()
opts.set_config('XX', 'xx')
opts.set_config('YY', 'yy')
self.assertEqual('xx', opts.get_config('XX'))
self.assertEqual('yy', opts.get_config('YY'))
def test_one_opt(self):
opts.clear_config()
opts.set_config('XX', 'xx')
self.assertEqual('xx', opts.get_config('XX'))