def submit_topology(ctx, jar, name, klass, args):
ctx.logger.info("Obtaining topology jar '{}'".format(jar))
local_jar = utils.obtain_resource(ctx, jar)
ctx.logger.info("Topology jar stored as '{}'".format(local_jar))
ctx.logger.info("Preparing topology configuration")
cfg = ctx.node.properties["configuration"].copy()
cfg.update(ctx.instance.runtime_properties.get("configuration", {}))
cfg_file = _write_tmp_configuration(cfg)
ctx.logger.info("Configuration stored in '{}'".format(cfg_file))
ctx.logger.info("Submitting '{}' as '{}'".format(local_jar, name))
subprocess.call([
"storm", "--config", cfg_file, "jar", local_jar, klass, name
] + args)
os.unlink(cfg_file)
ctx.logger.info("Retrieving topology id for '{}'".format(name))
nimbus_ip = ctx.instance.host_ip
url = "http://{}:8080/api/v1/topology/summary".format(nimbus_ip)
topology_id = _get_topology_id(url, name)
ctx.logger.info("Topology id for '{}' is '{}'".format(name, topology_id))
if topology_id is None:
msg = "Topology '{}' failed to start properly".format(name)
raise NonRecoverableError(msg)
ctx.instance.runtime_properties["topology_id"] = topology_id
def download_resources(ctx, resource_pairs):
for destination_key, source in resource_pairs:
if source is None or source == "":
ctx.logger.info("Skipping key '{}'".format(destination_key))
location = None
else:
ctx.logger.info("Downloading '{}'".format(source))
location = utils.obtain_resource(ctx, source)
ctx.instance.runtime_properties[destination_key] = location
def submit(ctx, jar, name, klass, args):
ctx.logger.info("Obtaining application jar '{}'".format(jar))
local_jar = utils.obtain_resource(ctx, jar)
ctx.logger.info("Application jar stored as '{}'".format(local_jar))
ctx.logger.info("Submitting '{}' as '{}'".format(local_jar, name))
base_cmd = ["spark-submit", "--deploy-mode", "client", "--class", klass,
local_jar, name]
ctx.logger.info("COMMAND: {}".format(base_cmd + args))
proc, log = utils.call(base_cmd + args, run_in_background=True)
ctx.logger.info("LOG FILE: {}".format(log))
def run_script(ctx, script, arguments, resources, language):
supported_langs = {"bash", "python"}
if language not in supported_langs:
msg = "Unknown language: {}. Available languages: {}."
raise NonRecoverableError(
msg.format(language, ", ".join(supported_langs)))
workdir = tempfile.mkdtemp()
ctx.logger.info("Created working directory {}".format(workdir))
ctx.logger.info("Getting '{}' script".format(script))
local_script = utils.obtain_resource(ctx,
script,
dir=workdir,
keep_name=True)
cmd = map(str, [language, local_script] + arguments)
ctx.logger.info("Getting resources".format(script))
for res in resources:
utils.obtain_resource(ctx, res, dir=workdir, keep_name=True)
ctx.logger.info("Running command: {}".format(" ".join(cmd)))
proc = subprocess.Popen(cmd,
stdin=open(os.devnull, "r"),
cwd=workdir,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
for line in iter(proc.stdout.readline, ""):
ctx.logger.info(line.strip())
proc.wait()
shutil.rmtree(workdir)
if proc.returncode != 0:
msg = "Script terminated with non-zero ({}) status."
raise NonRecoverableError(msg.format(proc.returncode))
ctx.instance.runtime_properties["ip"] = ctx.instance.host_ip
ctx.instance.runtime_properties["fqdn"] = utils.get_fqdn()
def create_image(ctx, auth, env, image):
ctx.logger.info("Downloading image data")
img_file = utils.obtain_resource(ctx, image)
ctx.logger.info("Creating image")
client = _get_client(auth)
with open(img_file, "rb") as fd:
img = client.image.upload_image(
container_format=ctx.node.properties["container_format"],
disk_format=ctx.node.properties["disk_format"],
data=fd,
)
_set_id(ctx, img.id)
def submit(ctx, jar, name, klass, args):
ctx.logger.info("Obtaining application jar '{}'".format(jar))
local_jar = utils.obtain_resource(ctx, jar)
ctx.logger.info("Application jar stored as '{}'".format(local_jar))
ctx.logger.info("Submitting '{}' as '{}'".format(local_jar, name))
base_cmd = [
"spark-submit", "--deploy-mode", "client", "--class", klass, local_jar,
name
]
ctx.logger.info("COMMAND: {}".format(base_cmd + args))
proc, log = utils.call(base_cmd + args, run_in_background=True)
ctx.logger.info("LOG FILE: {}".format(log))
def run_script(ctx, script, arguments, resources, language):
supported_langs = {"bash", "python"}
if language not in supported_langs:
msg = "Unknown language: {}. Available languages: {}."
raise NonRecoverableError(
msg.format(language, ", ".join(supported_langs))
)
workdir = tempfile.mkdtemp()
ctx.logger.info("Created working directory {}".format(workdir))
ctx.logger.info("Getting '{}' script".format(script))
local_script = utils.obtain_resource(ctx, script, dir=workdir,
keep_name=True)
cmd = map(str, [language, local_script] + arguments)
ctx.logger.info("Getting resources".format(script))
for res in resources:
utils.obtain_resource(ctx, res, dir=workdir, keep_name=True)
ctx.logger.info("Running command: {}".format(" ".join(cmd)))
proc = subprocess.Popen(cmd, stdin=open(os.devnull, "r"), cwd=workdir,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
for line in iter(proc.stdout.readline, ""):
ctx.logger.info(line.strip())
proc.wait()
shutil.rmtree(workdir)
if proc.returncode != 0:
msg = "Script terminated with non-zero ({}) status."
raise NonRecoverableError(msg.format(proc.returncode))
ctx.instance.runtime_properties["ip"] = ctx.instance.host_ip
ctx.instance.runtime_properties["fqdn"] = utils.get_fqdn()
def submit_topology(ctx, jar, name, klass, args):
ctx.logger.info("Obtaining topology jar '{}'".format(jar))
local_jar = utils.obtain_resource(ctx, jar)
ctx.logger.info("Topology jar stored as '{}'".format(local_jar))
ctx.logger.info("Preparing topology configuration")
cfg = ctx.node.properties["configuration"].copy()
cfg.update(ctx.instance.runtime_properties.get("configuration", {}))
config_args = _prepare_config_args(cfg)
cmd = ["storm"] + config_args + ["jar", local_jar, klass, name] + args
ctx.logger.info("Executing '{}'".format(" ".join(cmd)))
subprocess.call(cmd)
ctx.logger.info("Retrieving topology id for '{}'".format(name))
nimbus_ip = ctx.instance.host_ip
url = "http://{}:8080/api/v1/topology/summary".format(nimbus_ip)
topology_id = _get_topology_id(url, name)
ctx.logger.info("Topology id for '{}' is '{}'".format(name, topology_id))
if topology_id is None:
msg = "Topology '{}' failed to start properly".format(name)
raise NonRecoverableError(msg)
ctx.instance.runtime_properties["topology_id"] = topology_id
def download_resource(ctx, source, destination_key):
ctx.logger.info("Downloading '{}'".format(source))
location = utils.obtain_resource(ctx, source)
ctx.instance.runtime_properties[destination_key] = location
