def cli_run(args): try: argdict = args.__dict__ destination = argdict['destination'] nm = NuleculeManager(app_spec=argdict['app_spec'], destination=destination, cli_answers=argdict['cli_answers'], answers_file=argdict['answers']) nm.run(**argdict) # Clean up the files if the user asked us to. Otherwise # notify the user where they can manage the application if destination and destination.lower() == 'none': Utils.rm_dir(nm.app_path) else: print_app_location(nm.app_path) sys.exit(0) except DockerException as e: logger.error(e) sys.exit(1) except NuleculeException as e: logger.error(e) sys.exit(1) except Exception as e: logger.error(e, exc_info=True) sys.exit(1)
def extract(self, image, src, dest, namespace, update=True): """ Extract contents of a container image from 'src' in container to 'dest' in host. Args: image (str): Name of container image src (str): Source path in container dest (str): Destination path in host update (bool): Update existing destination, if True """ if os.path.exists(dest) and not update: return cleaned_image_name = Utils.sanitizeName(image) pod_name = '{}-{}'.format(cleaned_image_name, Utils.getUniqueUUID()) container_name = cleaned_image_name # Pull (if needed) image and bring up a container from it # with 'sleep 3600' entrypoint, just to extract content from it artifact = { 'apiVersion': 'v1', 'kind': 'Pod', 'metadata': { 'name': pod_name }, 'spec': { 'containers': [ { 'image': image, 'command': [ 'sleep', '3600' ], 'imagePullPolicy': 'IfNotPresent', 'name': container_name } ], 'restartPolicy': 'Always' } } self.create(artifact, namespace) try: self._wait_till_pod_runs(namespace, pod_name, timeout=300) # Archive content from the container and dump it to tmpfile tmpfile = '/tmp/atomicapp-{pod}.tar.gz'.format(pod=pod_name) self._execute( namespace, pod_name, container_name, 'tar -cz --directory {} ./'.format('/' + src), outfile=tmpfile ) finally: # Delete created pod self.delete(artifact, namespace) # Extract archive data tar = tarfile.open(tmpfile, 'r:gz') tar.extractall(dest)
def cli_genanswers(args): argdict = args.__dict__ nm = NuleculeManager(app_spec=argdict['app_spec'], destination='none') nm.genanswers(**argdict) Utils.rm_dir(nm.app_path) # clean up files sys.exit(0)
def _callCli(self, path): cmd = [self.cli, "--config=%s" % self.config_file, "create", "-f", path] if self.dryrun: logger.info("Calling: %s", " ".join(cmd)) else: Utils.run_cmd(cmd, checkexitcode=True)
def init(self): self.namespace = "default" self.k8s_manifests = [] logger.debug("Given config: %s", self.config) if self.config.get("namespace"): self.namespace = self.config.get("namespace") logger.info("Using namespace %s", self.namespace) if self.container: self.kubectl = self._find_kubectl(Utils.getRoot()) kube_conf_path = "/etc/kubernetes" host_kube_conf_path = os.path.join(Utils.getRoot(), kube_conf_path.lstrip("/")) if not os.path.exists(kube_conf_path) and os.path.exists(host_kube_conf_path): if self.dryrun: logger.info("DRY-RUN: link %s from %s" % (kube_conf_path, host_kube_conf_path)) else: os.symlink(host_kube_conf_path, kube_conf_path) else: self.kubectl = self._find_kubectl() if not self.dryrun: if not os.access(self.kubectl, os.X_OK): raise ProviderFailedException("Command: " + self.kubectl + " not found") # Check if Kubernetes config file is accessible, but only # if one was provided by the user; config file is optional. if self.config_file: self.checkConfigFile()
def init(self): self.namespace = "default" self.k8s_manifests = [] logger.debug("Given config: %s", self.config) if self.config.get("namespace"): self.namespace = self.config.get("namespace") logger.info("Using namespace %s", self.namespace) if self.container: self.kubectl = self._find_kubectl(Utils.getRoot()) kube_conf_path = "/etc/kubernetes" host_kube_conf_path = Utils.get_real_abspath(kube_conf_path) if not os.path.exists(kube_conf_path) and os.path.exists(host_kube_conf_path): if self.dryrun: logger.info("DRY-RUN: link %s from %s" % (kube_conf_path, host_kube_conf_path)) else: os.symlink(host_kube_conf_path, kube_conf_path) else: self.kubectl = self._find_kubectl() if not self.dryrun: if not os.access(self.kubectl, os.X_OK): raise ProviderFailedException("Command: " + self.kubectl + " not found") # Check if Kubernetes config file is accessible, but only # if one was provided by the user; config file is optional. if self.config_file: self.checkConfigFile()
def init(self): self.namespace = "default" self.k8s_manifests = [] logger.debug("Given config: %s", self.config) if self.config.get("namespace"): self.namespace = self.config.get("namespace") logger.info("Using namespace %s", self.namespace) if self.container: self.kubectl = self._find_kubectl(Utils.getRoot()) kube_conf_path = "/etc/kubernetes" if not os.path.exists(kube_conf_path): if self.dryrun: logger.info("DRY-RUN: link %s from %s%s" % (kube_conf_path, HOST_DIR, kube_conf_path)) else: os.symlink( os.path.join(Utils.getRoot(), kube_conf_path.lstrip("/")), kube_conf_path) else: self.kubectl = self._find_kubectl() if not self.dryrun: if not os.access(self.kubectl, os.X_OK): raise ProviderFailedException("Command: " + self.kubectl + " not found") # Check if Kubernetes config file is accessible self.checkConfigFile()
def __init__(self, app_spec, destination=None, answers_file=None): """ init function for NuleculeManager. Sets a few instance variables. Args: app_spec: either a path to an unpacked nulecule app or a container image name where a nulecule can be found destination: where to unpack a nulecule to if it isn't local """ self.answers = copy.deepcopy(DEFAULT_ANSWERS) self.answers_format = None self.answers_file = None # The path to an answer file self.app_path = None # The path where the app resides or will reside self.image = None # The container image to pull the app from # Adjust app_spec, destination, and answer file paths if absolute. if os.path.isabs(app_spec): app_spec = os.path.join(Utils.getRoot(), app_spec.lstrip('/')) if destination and os.path.isabs(destination): destination = os.path.join(Utils.getRoot(), destination.lstrip('/')) if answers_file and os.path.isabs(answers_file): answers_file = os.path.join(Utils.getRoot(), answers_file.lstrip('/')) # Determine if the user passed us an image or a path to an app if not os.path.exists(app_spec): self.image = app_spec else: self.app_path = app_spec # Doesn't make sense to provide an app path and destination if self.app_path and destination: raise NuleculeException( "You can't provide a local path and destination.") # If the user provided an image, make sure we have a destination if self.image: if destination: self.app_path = destination else: self.app_path = Utils.getNewAppCacheDir(self.image) logger.debug("NuleculeManager init app_path: %s", self.app_path) logger.debug("NuleculeManager init image: %s", self.image) # Set where the main nulecule file should be self.main_file = os.path.join(self.app_path, MAIN_FILE) # If user provided a path to answers then make sure it exists. If they # didn't provide one then use the one in the app dir if it exists. if answers_file: self.answers_file = answers_file if not os.path.isfile(self.answers_file): raise NuleculeException( "Path for answers doesn't exist: %s" % self.answers_file) else: if os.path.isfile(os.path.join(self.app_path, ANSWERS_FILE)): self.answers_file = os.path.join(self.app_path, ANSWERS_FILE)
def __init__(self, app_spec, destination=None, answers_file=None): """ init function for NuleculeManager. Sets a few instance variables. Args: app_spec: either a path to an unpacked nulecule app or a container image name where a nulecule can be found destination: where to unpack a nulecule to if it isn't local """ self.answers = copy.deepcopy(DEFAULT_ANSWERS) self.answers_format = None self.answers_file = None # The path to an answer file self.app_path = None # The path where the app resides or will reside self.image = None # The container image to pull the app from # Adjust app_spec, destination, and answer file paths if absolute. if os.path.isabs(app_spec): app_spec = os.path.join(Utils.getRoot(), app_spec.lstrip('/')) if destination and os.path.isabs(destination): destination = os.path.join(Utils.getRoot(), destination.lstrip('/')) if answers_file and os.path.isabs(answers_file): answers_file = os.path.join(Utils.getRoot(), answers_file.lstrip('/')) # Determine if the user passed us an image or a path to an app if not os.path.exists(app_spec): self.image = app_spec else: self.app_path = app_spec # Doesn't make sense to provide an app path and destination if self.app_path and destination: raise NuleculeException( "You can't provide a local path and destination.") # If the user provided an image, make sure we have a destination if self.image: if destination: self.app_path = destination else: self.app_path = Utils.getNewAppCacheDir(self.image) logger.debug("NuleculeManager init app_path: %s", self.app_path) logger.debug("NuleculeManager init image: %s", self.image) # Set where the main nulecule file should be self.main_file = os.path.join(self.app_path, MAIN_FILE) # If user provided a path to answers then make sure it exists. If they # didn't provide one then use the one in the app dir if it exists. if answers_file: self.answers_file = answers_file if not os.path.isfile(self.answers_file): raise NuleculeException("Path for answers doesn't exist: %s" % self.answers_file) else: if os.path.isfile(os.path.join(self.app_path, ANSWERS_FILE)): self.answers_file = os.path.join(self.app_path, ANSWERS_FILE)
def run(self): cmdline = sys.argv[1:] # Grab args from cmdline # If we are running in an openshift pod (via `oc new-app`) then # there is no cmdline but we want to default to "atomicapp run". # In this case copy files to cwd and use the working directory. if Utils.running_on_openshift(): cmdline = 'run -v --dest=none /{}'.format(APP_ENT_PATH).split() # We want to be able to place options anywhere on the command # line. We have added all global options to each subparser, # but subparsers require all options to be after the 'action' # keyword. In order to handle this we just need to figure out # what subparser will be used and move it's keyword to the front # of the line. # NOTE: Also allow "mode" to override 'action' if specified args, _ = self.parser.parse_known_args(cmdline) cmdline.remove(args.action) # Remove 'action' from the cmdline if args.mode: args.action = args.mode # Allow mode to override 'action' cmdline.insert(0, args.action) # Place 'action' at front logger.info("Action/Mode Selected is: %s" % args.action) # Finally, parse args and give error if necessary args = self.parser.parse_args(cmdline) # Set logging level if args.verbose: set_logging(level=logging.DEBUG) elif args.quiet: set_logging(level=logging.WARNING) else: set_logging(level=logging.INFO) lock = LockFile(os.path.join(Utils.getRoot(), LOCK_FILE)) try: lock.acquire(timeout=-1) args.func(args) except AttributeError: if hasattr(args, 'func'): raise else: self.parser.print_help() except KeyboardInterrupt: pass except AlreadyLocked: logger.error("Could not proceed - there is probably another instance of Atomic App running on this machine.") except Exception as ex: if args.verbose: raise else: logger.error("Exception caught: %s", repr(ex)) logger.error( "Run the command again with -v option to get more information.") finally: if lock.i_am_locking(): lock.release()
def init(self): self.oc_artifacts = {} logger.debug("Given config: %s", self.config) if self.config.get("namespace"): self.namespace = self.config.get("namespace") logger.info("Using namespace %s", self.namespace) self._process_artifacts() if self.dryrun: return ''' Config_file: If a config_file has been provided, use the configuration from the file and load the associated generated file. If a config_file exists (--provider-config) use that. Params: If any provider specific parameters have been provided, load the configuration through the answers.conf file .kube/config: If no config file or params are provided by user then try to find and use a config file at the default location. no config at all: If no .kube/config file can be found then try to connect to the default unauthenticated http://localhost:8080/api end-point. ''' default_config_loc = os.path.join(Utils.getRoot(), Utils.getUserHome().strip('/'), '.kube/config') if self.config_file: logger.debug("Provider configuration provided") self.api = Client(KubeConfig.from_file(self.config_file), "openshift") elif self._check_required_params(): logger.debug("Generating .kube/config from given parameters") self.api = Client(self._from_required_params(), "openshift") elif os.path.isfile(default_config_loc): logger.debug( ".kube/config exists, using default configuration file") self.api = Client(KubeConfig.from_file(default_config_loc), "openshift") else: self.config["provider-api"] = OC_DEFAULT_API self.api = Client(self._from_required_params(), "openshift") self._check_namespaces()
def cli_genanswers(args): try: argdict = args.__dict__ nm = NuleculeManager(app_spec=argdict['app_spec'], destination='none') nm.genanswers(**argdict) Utils.rm_dir(nm.app_path) # clean up files sys.exit(0) except NuleculeException as e: logger.error(e) sys.exit(1) except Exception as e: logger.error(e, exc_info=True) sys.exit(1)
def init(self): self.k8s_artifacts = {} logger.debug("Given config: %s", self.config) if self.config.get("namespace"): self.namespace = self.config.get("namespace") logger.info("Using namespace %s", self.namespace) self._process_artifacts() if self.dryrun: return ''' Config_file: If a config_file has been provided, use the configuration from the file and load the associated generated file. If a config_file exists (--provider-config) use that. Params: If any provider specific parameters have been provided, load the configuration through the answers.conf file .kube/config: If no config file or params are provided by user then try to find and use a config file at the default location. no config at all: If no .kube/config file can be found then try to connect to the default unauthenticated http://localhost:8080/api end-point. ''' default_config_loc = os.path.join( Utils.getRoot(), Utils.getUserHome().strip('/'), '.kube/config') if self.config_file: logger.debug("Provider configuration provided") self.api = Client(KubeConfig.from_file(self.config_file), "kubernetes") elif self._check_required_params(): logger.debug("Generating .kube/config from given parameters") self.api = Client(self._from_required_params(), "kubernetes") elif os.path.isfile(default_config_loc): logger.debug(".kube/config exists, using default configuration file") self.api = Client(KubeConfig.from_file(default_config_loc), "kubernetes") else: self.config["provider-api"] = K8S_DEFAULT_API self.api = Client(self._from_required_params(), "kubernetes") # Check if the namespace that the app is being deployed to is available self._check_namespaces()
def cli_fetch(args): argdict = args.__dict__ destination = argdict['destination'] nm = NuleculeManager(app_spec=argdict['app_spec'], destination=destination, cli_answers=argdict['cli_answers'], answers_file=argdict['answers']) nm.fetch(**argdict) # Clean up the files if the user asked us to. Otherwise # notify the user where they can manage the application if destination and destination.lower() == 'none': Utils.rm_dir(nm.app_path) else: print_app_location(nm.app_path) sys.exit(0)
def _process_answers(self): """ Processes answer files to load data from them and then merges any cli provided answers into the config. NOTE: This function should be called once on startup and then once more after the application has been extracted, but only if answers file wasn't found on the first invocation. The idea is to allow for people to embed an answers file in the application if they want, which won't be available until after extraction. Returns: None """ # If the user didn't provide an answers file then check the app # dir to see if one exists. if not self.answers_file: f = os.path.join(self.app_path, ANSWERS_FILE) if os.path.isfile(f): self.answers_file = f # At this point if we have an answers file, load it if self.answers_file: self.answers = Utils.loadAnswers(self.answers_file) # If there is answers data from the cli then merge it in now if self.cli_answers: for k, v in self.cli_answers.iteritems(): self.answers[GLOBAL_CONF][k] = v
def load_config(self, config=None, ask=False, skip_asking=False): """ Load config data. Sets the loaded config data to self.config. Args: config (dict): Initial config data ask (bool): When True, ask for values for a param from user even if the param has a default value skip_asking (bool): When True, skip asking for values for params with missing values and set the value as None Returns: None """ config = config or DEFAULT_ANSWERS for param in self.params: value = config.get(self.namespace, {}).get(param['name']) or \ config.get(GLOBAL_CONF, {}).get(param['name']) if value is None and (ask or ( not skip_asking and param.get('default') is None)): value = Utils.askFor(param['name'], param) elif value is None: value = param.get('default') if config.get(self.namespace) is None: config[self.namespace] = {} config[self.namespace][param['name']] = value self.config = config
def get_pod_status(self, namespace, pod): """ Get pod status. Args: namespace (str): Openshift namespace pod (str): Pod name Returns: Status of pod (str) Raises: ProviderFailedException when unable to fetch Pod status. """ args = { 'namespace': namespace, 'pod': pod, 'access_token': self.access_token } url = urljoin( self.kubernetes_api, 'namespaces/{namespace}/pods/{pod}?' 'access_token={access_token}'.format(**args)) (status_code, return_data) = \ Utils.make_rest_request("get", url, verify=self._requests_tls_verify()) if status_code != 200: raise ProviderFailedException( 'Could not fetch status for pod: {namespace}/{pod}'.format( namespace=namespace, pod=pod)) return return_data['status']['phase'].lower()
def load_config(self, config, ask=False, skip_asking=False): """ Load config data. Sets the loaded config data to self.config. Args: config (dict): Initial config data ask (bool): When True, ask for values for a param from user even if the param has a default value skip_asking (bool): When True, skip asking for values for params with missing values and set the value as None Returns: None """ self.config = config for param in self.params: value = config.get(param[NAME_KEY], scope=self.namespace, ignore_sources=['defaults']) if value is None: if ask or (not skip_asking and param.get(DEFAULTNAME_KEY) is None): cockpit_logger.info( "%s is missing in answers.conf." % param[NAME_KEY]) value = config.get(param[NAME_KEY], scope=self.namespace) \ or Utils.askFor(param[NAME_KEY], param, self.namespace) else: value = param.get(DEFAULTNAME_KEY) config.set(param[NAME_KEY], value, source='runtime', scope=self.namespace)
def load_components(self, nodeps=False, dryrun=False): """ Load components for the Nulecule application. Sets a list of NuleculeComponent instances to self.components. Args: nodeps (bool): When True, do not external dependencies of a Nulecule component dryrun (bool): When True, do not make any change to the host system Returns: None """ components = [] for node in self.graph: node_name = node[NAME_KEY] source = Utils.getSourceImage(node) component = NuleculeComponent( node_name, self.basepath, source, node.get(PARAMS_KEY), node.get(ARTIFACTS_KEY), self.config) component.load(nodeps, dryrun) components.append(component) self.components = components
def test_connection(self): """ Test connection to OpenShift server Raises: ProviderFailedException - Invalid SSL/TLS certificate """ logger.debug("Testing connection to OpenShift server") if self.provider_ca and not os.path.exists(self.provider_ca): raise ProviderFailedException("Unable to find CA path %s" % self.provider_ca) try: (status_code, return_data) = \ Utils.make_rest_request("get", self.openshift_api, verify=self._requests_tls_verify()) except SSLError as e: if self.provider_tls_verify: msg = "SSL/TLS ERROR: invalid certificate. " \ "Add certificate of correct Certificate Authority providing" \ " `%s` or you can disable SSL/TLS verification by `%s=False`" \ % (PROVIDER_CA_KEY, PROVIDER_TLS_VERIFY_KEY) raise ProviderFailedException(msg) else: # this shouldn't happen raise ProviderFailedException(e.message)
def load_config(self, config, ask=False, skip_asking=False): """ Load config data. Sets the loaded config data to self.config. Args: config (dict): Initial config data ask (bool): When True, ask for values for a param from user even if the param has a default value skip_asking (bool): When True, skip asking for values for params with missing values and set the value as None Returns: None """ self.config = config for param in self.params: value = config.get(param[NAME_KEY], scope=self.namespace, ignore_sources=["defaults"]) if value is None: if ask or (not skip_asking and param.get(DEFAULTNAME_KEY) is None): cockpit_logger.info("%s is missing in answers.conf." % param[NAME_KEY]) value = config.get(param[NAME_KEY], scope=self.namespace) or Utils.askFor( param[NAME_KEY], param, self.namespace ) else: value = param.get(DEFAULTNAME_KEY) config.set(param[NAME_KEY], value, source="runtime", scope=self.namespace)
def install(self, nodeps=False, update=False, dryrun=False, answers_format=ANSWERS_FILE_SAMPLE_FORMAT, **kwargs): """ Installs (unpacks) a Nulecule application from a Nulecule image to a target path. Args: answers (dict or str): Answers data or local path to answers file nodeps (bool): Install the nulecule application without installing external dependencies update (bool): Pull requisite Nulecule image and install or update already installed Nulecule application dryrun (bool): Do not make any change to the host system if True answers_format (str): File format for writing sample answers file kwargs (dict): Extra keyword arguments Returns: None """ if self.answers_file: self.answers = Utils.loadAnswers(self.answers_file) self.answers_format = answers_format or ANSWERS_FILE_SAMPLE_FORMAT # Call unpack. If the app doesn't exist it will be pulled. If # it does exist it will be just be loaded and returned self.nulecule = self.unpack(update, dryrun, config=self.answers) self.nulecule.load_config(config=self.nulecule.config, skip_asking=True) runtime_answers = self._get_runtime_answers(self.nulecule.config, None) # write sample answers file self._write_answers(os.path.join(self.app_path, ANSWERS_FILE_SAMPLE), runtime_answers, answers_format)
def get_artifact_paths_for_provider(self, provider_key): """ Get artifact file paths of a Nulecule component for a provider. Args: provider_key (str): Provider name Returns: list: A list of artifact paths. """ artifact_paths = [] artifacts = self.artifacts.get(provider_key) for artifact in artifacts: # Convert dict if the Nulecule file references "resource" if isinstance(artifact, dict) and artifact.get(RESOURCE_KEY): artifact = artifact[RESOURCE_KEY] logger.debug("Resource xpath added: %s" % artifact) # Sanitize the file structure if isinstance(artifact, basestring): path = Utils.sanitizePath(artifact) path = os.path.join(self.basepath, path) \ if path[0] != '/' else path artifact_paths.extend(self._get_artifact_paths_for_path(path)) # Inherit if inherit name is referenced elif isinstance(artifact, dict) and artifact.get(INHERIT_KEY) and \ isinstance(artifact.get(INHERIT_KEY), list): for inherited_provider_key in artifact.get(INHERIT_KEY): artifact_paths.extend( self.get_artifact_paths_for_provider( inherited_provider_key)) else: logger.error('Invalid artifact file') return artifact_paths
def unpack(cls, image, dest, config=None, namespace=GLOBAL_CONF, nodeps=False, dryrun=False, update=False): """ Pull and extracts a docker image to the specified path, and loads the Nulecule application from the path. Args: image (str): A Docker image name. dest (str): Destination path where Nulecule data from Docker image should be extracted. config (dict): Dictionary, config data for Nulecule application. namespace (str): Namespace for Nulecule application. nodeps (bool): Don't pull external Nulecule dependencies when True. update (bool): Don't update contents of destination directory if False, else update it. Returns: A Nulecule instance, or None in case of dry run. """ logger.info('Unpacking image: %s to %s' % (image, dest)) if Utils.running_on_openshift(): # pass general config data containing provider specific data # to Openshift provider op = OpenShiftProvider(config.get('general', {}), './', False) op.artifacts = [] op.init() op.extract(image, APP_ENT_PATH, dest, update) else: docker_handler = DockerHandler(dryrun=dryrun) docker_handler.pull(image) docker_handler.extract(image, APP_ENT_PATH, dest, update) return cls.load_from_path( dest, config=config, namespace=namespace, nodeps=nodeps, dryrun=dryrun, update=update)
def load_components(self, nodeps=False, dryrun=False): """ Load components for the Nulecule application. Sets a list of NuleculeComponent instances to self.components. Args: nodeps (bool): When True, do not external dependencies of a Nulecule component dryrun (bool): When True, do not make any change to the host system Returns: None """ components = [] for node in self.graph: node_name = node[NAME_KEY] source = Utils.getSourceImage(node) component = NuleculeComponent( self._get_component_namespace(node_name), self.basepath, source, node.get(PARAMS_KEY), node.get(ARTIFACTS_KEY), self.config) component.load(nodeps, dryrun) components.append(component) self.components = components
def run(self): self.set_arguments() args = self.parser.parse_args() if args.verbose: set_logging(level=logging.DEBUG) elif args.quiet: set_logging(level=logging.WARNING) else: set_logging(level=logging.INFO) lock = LockFile(os.path.join(Utils.getRoot(), LOCK_FILE)) try: lock.acquire(timeout=-1) args.func(args) except AttributeError: if hasattr(args, 'func'): raise else: self.parser.print_help() except KeyboardInterrupt: pass except AlreadyLocked: logger.error("Could not proceed - there is probably another instance of Atomic App running on this machine.") except Exception as ex: if args.verbose: raise else: logger.error("Exception caught: %s", repr(ex)) logger.error( "Run the command again with -v option to get more information.") finally: if lock.i_am_locking(): lock.release()
def scale(self, url, replicas): """ Scale ReplicationControllers or DeploymentConfig Args: url (str): full url for artifact replicas (int): number of replicas scale to """ patch = [{ "op": "replace", "path": "/spec/replicas", "value": replicas }] (status_code, return_data) = \ Utils.make_rest_request("patch", url, data=patch, verify=self._requests_tls_verify()) if status_code == 200: logger.info("Successfully scaled to %s replicas", replicas) else: msg = "%s %s" % (status_code, return_data) logger.error(msg) raise ProviderFailedException(msg)
def persistent_storage(self, graph, action): """ Actions are either: run, stop or uninstall as per the Requirements class Curently run is the only function implemented for k8s persistent storage """ logger.debug("Persistent storage enabled! Running action: %s" % action) if action not in ['run']: logger.warning( "%s action is not available for provider %s. Doing nothing." % (action, self.key)) return self._check_persistent_volumes() # Get the path of the persistent storage yaml file includes in /external # Plug the information from the graph into the persistent storage file base_path = os.path.dirname(os.path.realpath(__file__)) template_path = os.path.join(base_path, 'external/kubernetes/persistent_storage.yaml') with open(template_path, 'r') as f: content = f.read() template = Template(content) rendered_template = template.safe_substitute(graph) tmp_file = Utils.getTmpFile(rendered_template, '.yaml') # Pass the .yaml file and execute if action is "run": cmd = [self.kubectl, "create", "-f", tmp_file, "--namespace=%s" % self.namespace] if self.config_file: cmd.append("--kubeconfig=%s" % self.config_file) self._call(cmd) os.unlink(tmp_file)
def stop(self): """ Undeploys the app by given resource manifests. Undeploy operation deletes Marathon apps from cluster. """ for artifact in self.marathon_artifacts: url = urlparse.urljoin( self.marathon_api, "apps/%s" % artifact["id"]) if self.dryrun: logger.info("DRY-RUN: %s", url) continue logger.debug("Deleting appid: %s", artifact["id"]) (status_code, return_data) = \ Utils.make_rest_request("delete", url, data=artifact) if status_code == 200: logger.info( "Marathon app %s sucessfully deleted.", artifact["id"]) else: msg = "Error deleting app: %s, Marathon API response %s - %s" % ( artifact["id"], status_code, return_data) logger.error(msg) raise ProviderFailedException(msg)
def get_artifact_paths_for_provider(self, provider_key): """ Get artifact file paths of a Nulecule component for a provider. Args: provider_key (str): Provider name Returns: list: A list of artifact paths. """ artifact_paths = [] artifacts = self.artifacts.get(provider_key) for artifact in artifacts: # Convert dict if the Nulecule file references "resource" if isinstance(artifact, dict) and artifact.get(RESOURCE_KEY): artifact = artifact[RESOURCE_KEY] logger.debug("Resource xpath added: %s" % artifact) # Sanitize the file structure if isinstance(artifact, basestring): path = Utils.sanitizePath(artifact) path = os.path.join(self.basepath, path) if path[0] != "/" else path artifact_paths.extend(self._get_artifact_paths_for_path(path)) # Inherit if inherit name is referenced elif isinstance(artifact, dict) and artifact.get("inherit") and isinstance(artifact.get("inherit"), list): for inherited_provider_key in artifact.get("inherit"): artifact_paths.extend(self.get_artifact_paths_for_provider(inherited_provider_key)) else: logger.error("Invalid artifact file") return artifact_paths
def deploy(self): logger.info("Deploying to provider: Docker") for container in self._get_containers(): if re.match( "%s_+%s+_+[a-zA-Z0-9]{12}" % (self.default_name, self.namespace), container): raise ProviderFailedException( "Namespace with name %s already deployed in Docker" % self.namespace) for artifact in self.artifacts: artifact_path = os.path.join(self.path, artifact) label_run = None with open(artifact_path, "r") as fp: label_run = fp.read().strip() run_args = label_run.split() # If --name is provided, do not re-name due to potential linking of containers. Warn user instead. # Else use namespace provided within answers.conf if '--name' in run_args: logger.info( "WARNING: Using --name provided within artifact file.") else: run_args.insert( run_args.index('run') + 1, "--name=%s_%s_%s" % (self.default_name, self.namespace, Utils.getUniqueUUID())) cmd = run_args if self.dryrun: logger.info("DRY-RUN: %s", " ".join(cmd)) else: subprocess.check_call(cmd)
def load_config(self, config, ask=False, skip_asking=False): """ Load config data. Sets the loaded config data to self.config. Args: config (dict): Initial config data ask (bool): When True, ask for values for a param from user even if the param has a default value skip_asking (bool): When True, skip asking for values for params with missing values and set the value as None Returns: None """ for param in self.params: value = config.get(self.namespace, {}).get(param[NAME_KEY]) or config.get(GLOBAL_CONF, {}).get( param[NAME_KEY] ) if value is None and (ask or (not skip_asking and param.get(DEFAULTNAME_KEY) is None)): cockpit_logger.info("%s is missing in answers.conf." % param[NAME_KEY]) value = Utils.askFor(param[NAME_KEY], param, self.namespace) elif value is None: value = param.get(DEFAULTNAME_KEY) if config.get(self.namespace) is None: config[self.namespace] = {} config[self.namespace][param[NAME_KEY]] = value self.config = config
def load_config(self, config, ask=False, skip_asking=False): """ Load config data. Sets the loaded config data to self.config. Args: config (dict): Initial config data ask (bool): When True, ask for values for a param from user even if the param has a default value skip_asking (bool): When True, skip asking for values for params with missing values and set the value as None Returns: None """ for param in self.params: value = config.get(self.namespace, {}).get(param[NAME_KEY]) or \ config.get(GLOBAL_CONF, {}).get(param[NAME_KEY]) if value is None and (ask or (not skip_asking and param.get(DEFAULTNAME_KEY) is None)): cockpit_logger.info("%s is missing in answers.conf." % param[NAME_KEY]) value = Utils.askFor(param[NAME_KEY], param) elif value is None: value = param.get(DEFAULTNAME_KEY) if config.get(self.namespace) is None: config[self.namespace] = {} config[self.namespace][param[NAME_KEY]] = value self.config = config
def run(self, cli_provider, answers_output, ask, answers_format=ANSWERS_FILE_SAMPLE_FORMAT, **kwargs): """ Runs a Nulecule application from a local path or a Nulecule image name. Args: answers (dict or str): Answers data or local path to answers file cli_provider (str): Provider to use to run the Nulecule application answers_output (str): Path to file to export runtime answers data to ask (bool): Ask for values for params with default values from user, if True answers_format (str): File format for writing sample answers file kwargs (dict): Extra keyword arguments Returns: None """ if self.answers_file: self.answers = Utils.loadAnswers(self.answers_file) self.answers_format = answers_format or ANSWERS_FILE_SAMPLE_FORMAT dryrun = kwargs.get('dryrun') or False # Call unpack. If the app doesn't exist it will be pulled. If # it does exist it will be just be loaded and returned self.nulecule = self.unpack(dryrun=dryrun, config=self.answers) # Unless otherwise specified with CLI arguments we will # default to the first provider available providers = Utils.getSupportedProviders(self.app_path) if cli_provider is None and len(providers) == 1: self.answers[GLOBAL_CONF][PROVIDER_KEY] = providers[0] self.nulecule.load_config(config=self.nulecule.config, ask=ask) self.nulecule.render(cli_provider, dryrun) self.nulecule.run(cli_provider, dryrun) runtime_answers = self._get_runtime_answers( self.nulecule.config, cli_provider) self._write_answers( os.path.join(self.app_path, ANSWERS_RUNTIME_FILE), runtime_answers, self.answers_format) if answers_output: self._write_answers(answers_output, runtime_answers, self.answers_format)
def extract_nulecule_data(self, image, source, dest, update=False): """ Extract the Nulecule contents from a container into a destination directory. Args: image (str): Docker image name source (str): Source directory in Docker image to copy from dest (str): Path to destination directory on host update (bool): Update destination directory if it exists when True Returns: None """ logger.info( 'Extracting Nulecule data from image %s to %s' % (image, dest)) if self.dryrun: return # Create a temporary directory for extraction tmpdir = '/tmp/nulecule-{}'.format(uuid.uuid1()) self.extract_files(image, source=source, dest=tmpdir) # If the application already exists locally then need to # make sure the local app id is the same as the one requested # on the command line. mainfile = os.path.join(dest, MAIN_FILE) tmpmainfile = os.path.join(tmpdir, MAIN_FILE) if os.path.exists(mainfile): existing_id = Utils.getAppId(mainfile) new_id = Utils.getAppId(tmpmainfile) cockpit_logger.info("Loading app_id %s" % new_id) if existing_id != new_id: raise NuleculeException( "Existing app (%s) and requested app (%s) differ" % (existing_id, new_id)) # If app exists and no update requested then move on if update: logger.info("App exists locally. Performing update...") else: logger.info("App exists locally and no update requested") return # Copy files from tmpdir into place logger.debug('Copying nulecule data from %s to %s' % (tmpdir, dest)) Utils.copy_dir(tmpdir, dest, update) # Clean up tmpdir logger.debug('Removing tmp dir: %s' % tmpdir) Utils.rm_dir(tmpdir) # Set the proper permissions on the extracted folder Utils.setFileOwnerGroup(dest)
def extract_files(self, image, source, dest): """ Extracts a directory/file in a Docker image to a specified destination. Args: image (str): Docker image name source (str): Source directory in Docker image to copy from dest (str): Path to destination directory on host Returns: None """ logger.info( 'Copying files from image %s:%s to %s' % (image, source, dest)) if self.dryrun: return # Create a dummy container in order to retrieve the file(s) run_cmd = [ self.docker_cli, 'create', '--entrypoint', '/bin/true', image] logger.debug('Creating docker container: %s' % ' '.join(run_cmd)) container_id = subprocess.check_output(run_cmd).strip() # Copy files out of dummy container to the destination directory cp_cmd = [self.docker_cli, 'cp', '%s:/%s' % (container_id, source), dest] logger.debug( 'Copying data from docker container: %s' % ' '.join(cp_cmd)) try: subprocess.check_output(cp_cmd, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: raise DockerException('Copying data from docker container failed: %s. \n%s' % (cp_cmd, e.output)) # Clean up dummy container rm_cmd = [self.docker_cli, 'rm', '-f', container_id] logger.debug('Removing docker container: %s' % ' '.join(rm_cmd)) try: subprocess.check_output(rm_cmd) except subprocess.CalledProcessError as e: raise DockerException('Removing docker container failed: %s. \n%s' % (rm_cmd, e.output)) # Set the proper permissions on the extracted folder Utils.setFileOwnerGroup(dest)
def _write_answers(self, path, answers, answers_format): """ Write answers data to file. Args: path (str): path to answers file to write to answers (dict): Answers data answers_format (str): Format to use to dump answers data to file, e.g., json Returns: None """ logger.debug("Writing answers to file.") logger.debug("FILE: %s", path) logger.debug("ANSWERS: %s", answers) anymarkup.serialize_file(answers, path, format=answers_format) # Make sure that the permission of the file is set to the current user Utils.setFileOwnerGroup(path)
def init(self): self.namespace = DEFAULT_NAMESPACE self.default_name = DEFAULT_CONTAINER_NAME logger.debug("Given config: %s", self.config) if self.config.get("namespace"): self.namespace = self.config.get("namespace") logger.debug("Namespace: %s", self.namespace) if "image" in self.config: self.image = Utils.sanitizeName(self.config.get("image")) else: self.image = Utils.getUniqueUUID() logger.warning( "The artifact name has not been provided within Nulecule, using a UUID instead" ) logger.debug( "No image name found for artifact, using UUID %s in container name" % self.image) if self.dryrun: logger.info("DRY-RUN: Did not check Docker version compatibility") else: cmd_check = ["docker", "version"] try: docker_version = subprocess.check_output(cmd_check).split("\n") except Exception as ex: raise ProviderFailedException(ex) client = "" server = "" for line in docker_version: if line.startswith("Client API version"): client = line.split(":")[1] if line.startswith("Server API version"): server = line.split(":")[1] if client > server: msg = ( "Docker version in app image (%s) is higher than the one " "on host (%s). Please update your host." % (client, server)) raise ProviderFailedException(msg)
def extract_nulecule_data(self, image, source, dest, update=False): """ Extract the Nulecule contents from a container into a destination directory. Args: image (str): Docker image name source (str): Source directory in Docker image to copy from dest (str): Path to destination directory on host update (bool): Update destination directory if it exists when True Returns: None """ logger.info("Extracting Nulecule data from image %s to %s" % (image, dest)) if self.dryrun: return # Create a temporary directory for extraction tmpdir = "/tmp/nulecule-{}".format(uuid.uuid1()) self.extract_files(image, source=source, dest=tmpdir) # If the application already exists locally then need to # make sure the local app id is the same as the one requested # on the command line. mainfile = os.path.join(dest, MAIN_FILE) tmpmainfile = os.path.join(tmpdir, MAIN_FILE) if os.path.exists(mainfile): existing_id = Utils.getAppId(mainfile) new_id = Utils.getAppId(tmpmainfile) cockpit_logger.info("Loading app_id %s" % new_id) if existing_id != new_id: raise NuleculeException("Existing app (%s) and requested app (%s) differ" % (existing_id, new_id)) # If app exists and no update requested then move on if update: logger.info("App exists locally. Performing update...") else: logger.info("App exists locally and no update requested") return # Copy files from tmpdir into place logger.debug("Copying nulecule data from %s to %s" % (tmpdir, dest)) Utils.copy_dir(tmpdir, dest, update) # Clean up tmpdir logger.debug("Removing tmp dir: %s" % tmpdir) Utils.rm_dir(tmpdir) # Set the proper permissions on the extracted folder Utils.setFileOwnerGroup(dest)
def _processTemplate(self, path): cmd = [self.cli, "--config=%s" % self.config_file, "process", "-f", path] name = "config-%s" % os.path.basename(path) output_path = os.path.join(self.path, name) if self.cli and not self.dryrun: ec, stdout, stderr = Utils.run_cmd(cmd, checkexitcode=True) logger.debug("Writing processed template to %s", output_path) with open(output_path, "w") as fp: fp.write(stdout) return name
def _write_answers(self, path, answers, answers_format): """ Write answers data to file. Args: path (str): path to answers file to write to answers (dict): Answers data answers_format (str): Format to use to dump answers data to file, e.g., json Returns: None """ logger.debug("Writing answers to file.") logger.debug("FILE: %s", path) logger.debug("ANSWERS: %s", answers) logger.debug("ANSWERS FORMAT: %s", answers_format) anymarkup.serialize_file(answers, path, format=answers_format) # Make sure that the permission of the file is set to the current user Utils.setFileOwnerGroup(path)
def init(self): self.cli = find_executable(self.cli_str) if self.container and not self.cli: host_path = [] for path in os.environ.get("PATH").split(":"): host_path.append(os.path.join(Utils.getRoot(), path.lstrip("/"))) self.cli = find_binary(self.cli_str, path=":".join(host_path)) if not self.cli: # if run as non-root we need a symlink in the container os.symlink(os.path.join(Utils.getRoot(), "usr/bin/oc"), "/usr/bin/oc") self.cli = "/usr/bin/oc" if not self.dryrun: if not self.cli or not os.access(self.cli, os.X_OK): raise ProviderFailedException("Command %s not found" % self.cli) else: logger.debug("Using %s to run OpenShift commands.", self.cli) # Check if OpenShift config file is accessible self.checkConfigFile()
def init(self): self.cli = find_executable(self.cli_str) if self.container and not self.cli: host_path = [] for path in os.environ.get("PATH").split(":"): host_path.append(os.path.join(Utils.getRoot(), path.lstrip("/"))) self.cli = find_binary(self.cli_str, path=":".join(host_path)) if not self.cli: # if run as non-root we need a symlink in the container os.symlink(os.path.join(Utils.getRoot(), "usr/bin/oc"), "/usr/bin/oc") self.cli = "/usr/bin/oc" if not self.dryrun: if not self.cli or not os.access(self.cli, os.X_OK): raise ProviderFailedException("Command %s not found" % self.cli) else: logger.debug("Using %s to run OpenShift commands.", self.cli) # Check if the required OpenShift config file is accessible. self.checkConfigFile()
def _call(self, cmd): """Calls given command :arg cmd: Command to be called in a form of list :raises: Exception """ if self.dryrun: logger.info("DRY-RUN: %s", " ".join(cmd)) else: ec, stdout, stderr = Utils.run_cmd(cmd, checkexitcode=True) return stdout
def extract_files(self, image, source, dest): """ Extracts a directory/file in a Docker image to a specified destination. Args: image (str): Docker image name source (str): Source directory in Docker image to copy from dest (str): Path to destination directory on host Returns: None """ logger.info("Copying files from image %s:%s to %s" % (image, source, dest)) if self.dryrun: return # Create a dummy container in order to retrieve the file(s) run_cmd = [self.docker_cli, "create", "--entrypoint", "/bin/true", image] logger.debug("Creating docker container: %s" % " ".join(run_cmd)) container_id = subprocess.check_output(run_cmd).strip() # Copy files out of dummy container to the destination directory cp_cmd = [self.docker_cli, "cp", "%s:/%s" % (container_id, source), dest] logger.debug("Copying data from docker container: %s" % " ".join(cp_cmd)) try: subprocess.check_output(cp_cmd, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: raise DockerException("Copying data from docker container failed: %s. \n%s" % (cp_cmd, e.output)) # Clean up dummy container rm_cmd = [self.docker_cli, "rm", "-f", container_id] logger.debug("Removing docker container: %s" % " ".join(rm_cmd)) try: subprocess.check_output(rm_cmd) except subprocess.CalledProcessError as e: raise DockerException("Removing docker container failed: %s. \n%s" % (rm_cmd, e.output)) # Set the proper permissions on the extracted folder Utils.setFileOwnerGroup(dest)
def load_external_application(self, dryrun=False, update=False): """ Loads an external application for the NuleculeComponent. Args: dryrun (bool): When True, skips pulling an external application. update (bool): When True, it ignores an already pulled external application, and tries to pull the external application and update the existing one. Returns: A Nulecule instance or None """ nulecule = None external_app_path = os.path.join( self.basepath, EXTERNAL_APP_DIR, self.name) if os.path.isdir(external_app_path) and not update: logger.info( 'Found existing external application: %s ' 'Loading: ' % self.name) nulecule = Nulecule.load_from_path( external_app_path, dryrun=dryrun, update=update, namespace=self.namespace) elif not dryrun: logger.info('Pulling external application: %s' % self.name) nulecule = Nulecule.unpack( self.source, external_app_path, config=self.config, namespace=self.namespace, dryrun=dryrun, update=update ) # When pulling an external application, make sure that the # "external" folder is owned by the respective user extracting it # by providing the basepath of the extraction Utils.setFileOwnerGroup(self.basepath) self._app = nulecule cockpit_logger.info("Copied app successfully.")
def process_template(self, url, template): (status_code, return_data) = \ Utils.make_rest_request("post", url, verify=self._requests_tls_verify(), data=template) if status_code == 201: logger.info("template processed %s", template['metadata']['name']) logger.debug("processed template %s", return_data) return return_data['objects'] else: msg = "%s %s" % (status_code, return_data) logger.error(msg) raise ProviderFailedException(msg)
def deploy(self, url, artifact): (status_code, return_data) = \ Utils.make_rest_request("post", url, verify=self._requests_tls_verify(), data=artifact) if status_code == 201: logger.info("Object %s successfully deployed.", artifact['metadata']['name']) else: msg = "%s %s" % (status_code, return_data) logger.error(msg) # TODO: remove running components (issue: #428) raise ProviderFailedException(msg)
def _process_answers(self): """ Processes answer files to load data from them and then merges any cli provided answers into the config. NOTE: This function should be called once on startup and then once more after the application has been extracted, but only if answers file wasn't found on the first invocation. The idea is to allow for people to embed an answers file in the application if they want, which won't be available until after extraction. Returns: None """ app_path_answers = os.path.join(self.app_path, ANSWERS_FILE) # If the user didn't provide an answers file then check the app # dir to see if one exists. if not self.answers_file: if os.path.isfile(app_path_answers): self.answers_file = app_path_answers # At this point if we have an answers file, load it if self.answers_file: # If this is a url then download answers file to app directory if urlparse.urlparse(self.answers_file).scheme != "": logger.debug("Retrieving answers file from: {}".format( self.answers_file)) with open(app_path_answers, 'w+') as f: stream = urllib.urlopen(self.answers_file) f.write(stream.read()) self.answers_file = app_path_answers # Check to make sure the file exists if not os.path.isfile(self.answers_file): raise NuleculeException( "Provided answers file doesn't exist: {}".format( self.answers_file)) # Load answers self.answers = Utils.loadAnswers(self.answers_file) # If there is answers data from the cli then merge it in now if self.cli_answers: for k, v in self.cli_answers.iteritems(): self.answers[GLOBAL_CONF][k] = v
def run(self, answers, cli_provider, answers_output, ask, answers_format=ANSWERS_FILE_SAMPLE_FORMAT, **kwargs): """ Runs a Nulecule application from a local path or a Nulecule image name. Args: answers (dict or str): Answers data or local path to answers file cli_provider (str): Provider to use to run the Nulecule application answers_output (str): Path to file to export runtime answers data to ask (bool): Ask for values for params with default values from user, if True answers_format (str): File format for writing sample answers file kwargs (dict): Extra keyword arguments Returns: None """ self.answers = Utils.loadAnswers( answers or os.path.join(self.app_path, ANSWERS_FILE)) self.answers_format = answers_format or ANSWERS_FILE_SAMPLE_FORMAT dryrun = kwargs.get('dryrun') or False # Call unpack. If the app doesn't exist it will be pulled. If # it does exist it will be just be loaded and returned self.nulecule = self.unpack(dryrun=dryrun, config=self.answers) self.nulecule.load_config(config=self.nulecule.config, ask=ask) self.nulecule.render(cli_provider, dryrun) self.nulecule.run(cli_provider, dryrun) runtime_answers = self._get_runtime_answers(self.nulecule.config, cli_provider) self._write_answers(os.path.join(self.app_path, ANSWERS_RUNTIME_FILE), runtime_answers, self.answers_format, dryrun=dryrun) if answers_output: self._write_answers(answers_output, runtime_answers, self.answers_format, dryrun)
def stop(self, cli_provider, **kwargs): """ Stops a running Nulecule application. Args: cli_provider (str): Provider running the Nulecule application kwargs (dict): Extra keyword arguments """ self.answers = Utils.loadAnswers( os.path.join(self.app_path, ANSWERS_RUNTIME_FILE)) dryrun = kwargs.get('dryrun') or False self.nulecule = Nulecule.load_from_path(self.app_path, config=self.answers, dryrun=dryrun) self.nulecule.load_config(config=self.answers) self.nulecule.render(cli_provider, dryrun=dryrun) self.nulecule.stop(cli_provider, dryrun)