def submit_topology(ctx, jar, name, klass, args): ctx.logger.info("Obtaining topology jar '{}'".format(jar)) local_jar = utils.obtain_resource(ctx, jar) ctx.logger.info("Topology jar stored as '{}'".format(local_jar)) ctx.logger.info("Preparing topology configuration") cfg = ctx.node.properties["configuration"].copy() cfg.update(ctx.instance.runtime_properties.get("configuration", {})) cfg_file = _write_tmp_configuration(cfg) ctx.logger.info("Configuration stored in '{}'".format(cfg_file)) ctx.logger.info("Submitting '{}' as '{}'".format(local_jar, name)) subprocess.call([ "storm", "--config", cfg_file, "jar", local_jar, klass, name ] + args) os.unlink(cfg_file) ctx.logger.info("Retrieving topology id for '{}'".format(name)) nimbus_ip = ctx.instance.host_ip url = "http://{}:8080/api/v1/topology/summary".format(nimbus_ip) topology_id = _get_topology_id(url, name) ctx.logger.info("Topology id for '{}' is '{}'".format(name, topology_id)) if topology_id is None: msg = "Topology '{}' failed to start properly".format(name) raise NonRecoverableError(msg) ctx.instance.runtime_properties["topology_id"] = topology_id
def download_resources(ctx, resource_pairs): for destination_key, source in resource_pairs: if source is None or source == "": ctx.logger.info("Skipping key '{}'".format(destination_key)) location = None else: ctx.logger.info("Downloading '{}'".format(source)) location = utils.obtain_resource(ctx, source) ctx.instance.runtime_properties[destination_key] = location
def submit(ctx, jar, name, klass, args): ctx.logger.info("Obtaining application jar '{}'".format(jar)) local_jar = utils.obtain_resource(ctx, jar) ctx.logger.info("Application jar stored as '{}'".format(local_jar)) ctx.logger.info("Submitting '{}' as '{}'".format(local_jar, name)) base_cmd = ["spark-submit", "--deploy-mode", "client", "--class", klass, local_jar, name] ctx.logger.info("COMMAND: {}".format(base_cmd + args)) proc, log = utils.call(base_cmd + args, run_in_background=True) ctx.logger.info("LOG FILE: {}".format(log))
def run_script(ctx, script, arguments, resources, language): supported_langs = {"bash", "python"} if language not in supported_langs: msg = "Unknown language: {}. Available languages: {}." raise NonRecoverableError( msg.format(language, ", ".join(supported_langs))) workdir = tempfile.mkdtemp() ctx.logger.info("Created working directory {}".format(workdir)) ctx.logger.info("Getting '{}' script".format(script)) local_script = utils.obtain_resource(ctx, script, dir=workdir, keep_name=True) cmd = map(str, [language, local_script] + arguments) ctx.logger.info("Getting resources".format(script)) for res in resources: utils.obtain_resource(ctx, res, dir=workdir, keep_name=True) ctx.logger.info("Running command: {}".format(" ".join(cmd))) proc = subprocess.Popen(cmd, stdin=open(os.devnull, "r"), cwd=workdir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) for line in iter(proc.stdout.readline, ""): ctx.logger.info(line.strip()) proc.wait() shutil.rmtree(workdir) if proc.returncode != 0: msg = "Script terminated with non-zero ({}) status." raise NonRecoverableError(msg.format(proc.returncode)) ctx.instance.runtime_properties["ip"] = ctx.instance.host_ip ctx.instance.runtime_properties["fqdn"] = utils.get_fqdn()
def create_image(ctx, auth, env, image): ctx.logger.info("Downloading image data") img_file = utils.obtain_resource(ctx, image) ctx.logger.info("Creating image") client = _get_client(auth) with open(img_file, "rb") as fd: img = client.image.upload_image( container_format=ctx.node.properties["container_format"], disk_format=ctx.node.properties["disk_format"], data=fd, ) _set_id(ctx, img.id)
def submit(ctx, jar, name, klass, args): ctx.logger.info("Obtaining application jar '{}'".format(jar)) local_jar = utils.obtain_resource(ctx, jar) ctx.logger.info("Application jar stored as '{}'".format(local_jar)) ctx.logger.info("Submitting '{}' as '{}'".format(local_jar, name)) base_cmd = [ "spark-submit", "--deploy-mode", "client", "--class", klass, local_jar, name ] ctx.logger.info("COMMAND: {}".format(base_cmd + args)) proc, log = utils.call(base_cmd + args, run_in_background=True) ctx.logger.info("LOG FILE: {}".format(log))
def run_script(ctx, script, arguments, resources, language): supported_langs = {"bash", "python"} if language not in supported_langs: msg = "Unknown language: {}. Available languages: {}." raise NonRecoverableError( msg.format(language, ", ".join(supported_langs)) ) workdir = tempfile.mkdtemp() ctx.logger.info("Created working directory {}".format(workdir)) ctx.logger.info("Getting '{}' script".format(script)) local_script = utils.obtain_resource(ctx, script, dir=workdir, keep_name=True) cmd = map(str, [language, local_script] + arguments) ctx.logger.info("Getting resources".format(script)) for res in resources: utils.obtain_resource(ctx, res, dir=workdir, keep_name=True) ctx.logger.info("Running command: {}".format(" ".join(cmd))) proc = subprocess.Popen(cmd, stdin=open(os.devnull, "r"), cwd=workdir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) for line in iter(proc.stdout.readline, ""): ctx.logger.info(line.strip()) proc.wait() shutil.rmtree(workdir) if proc.returncode != 0: msg = "Script terminated with non-zero ({}) status." raise NonRecoverableError(msg.format(proc.returncode)) ctx.instance.runtime_properties["ip"] = ctx.instance.host_ip ctx.instance.runtime_properties["fqdn"] = utils.get_fqdn()
def submit_topology(ctx, jar, name, klass, args): ctx.logger.info("Obtaining topology jar '{}'".format(jar)) local_jar = utils.obtain_resource(ctx, jar) ctx.logger.info("Topology jar stored as '{}'".format(local_jar)) ctx.logger.info("Preparing topology configuration") cfg = ctx.node.properties["configuration"].copy() cfg.update(ctx.instance.runtime_properties.get("configuration", {})) config_args = _prepare_config_args(cfg) cmd = ["storm"] + config_args + ["jar", local_jar, klass, name] + args ctx.logger.info("Executing '{}'".format(" ".join(cmd))) subprocess.call(cmd) ctx.logger.info("Retrieving topology id for '{}'".format(name)) nimbus_ip = ctx.instance.host_ip url = "http://{}:8080/api/v1/topology/summary".format(nimbus_ip) topology_id = _get_topology_id(url, name) ctx.logger.info("Topology id for '{}' is '{}'".format(name, topology_id)) if topology_id is None: msg = "Topology '{}' failed to start properly".format(name) raise NonRecoverableError(msg) ctx.instance.runtime_properties["topology_id"] = topology_id
def download_resource(ctx, source, destination_key): ctx.logger.info("Downloading '{}'".format(source)) location = utils.obtain_resource(ctx, source) ctx.instance.runtime_properties[destination_key] = location