def get_slack_usernames_from_owners(owners_raw_urls, owners_aliases_raw_url, users, usergroup, user_key, ssl_verify=True, missing_user_log_method=logging.warning): all_slack_usernames = [] all_username_keys = [u[user_key] for u in users] owners_aliases = {} if owners_aliases_raw_url: r = get_raw_owners_content(owners_aliases_raw_url, ssl_verify=ssl_verify) try: content = anymarkup.parse(r.content, force_types=None) owners_aliases = content['aliases'] except (anymarkup.AnyMarkupError, KeyError): msg = "Could not parse data. Skipping owners_aliases file: {}" logging.warning(msg.format(owners_aliases_raw_url)) for owners_raw_url in owners_raw_urls or []: r = get_raw_owners_content(owners_raw_url, ssl_verify=ssl_verify) try: content = anymarkup.parse(r.content, force_types=None) owners = set() for value in content.values(): for user in value: if user in owners_aliases: for alias_user in owners_aliases[user]: owners.add(alias_user) else: owners.add(user) except (anymarkup.AnyMarkupError, KeyError): msg = "Could not parse data. Skipping owners file: {}" logging.warning(msg.format(owners_raw_url)) continue if not owners: continue slack_usernames = [get_slack_username(u) for u in users if u[user_key] in owners] not_found_users = [owner for owner in owners if owner not in all_username_keys] if not_found_users: msg = f'[{usergroup}] {user_key} not found in app-interface: ' + \ f'{not_found_users}' missing_user_log_method(msg) all_slack_usernames.extend(slack_usernames) return all_slack_usernames
def export_all(self): """ only kubernetes things for now """ # Resources to export. # Don't export Pods for now. # Exporting ReplicationControllers should be enough. # Ideally this should detect Pods that are not created by # ReplicationController and only export those. resources = ["replicationcontrollers", "persistentvolumeclaims", "services"] # output of this export is kind List args = ["export", ",".join(resources), "-o", "json"] ec, stdout, stderr = self._call_oc(args) objects = anymarkup.parse(stdout, format="json", force_types=None) image_infos = [] for o in objects["items"]: if o["kind"] == "ReplicationController": image_infos.extend(self.get_image_info(o)) for ii in image_infos: if ii["private"]: logger.warning("{kind} {name} has image that appears to be " "from local OpenShift registry!!".format(**ii)) return objects
def has_image(self, imageinfo): # TODO: it seems that the best non-auth way is to look at repository's tags, # e.g. registry.com/v1/repositories/<image>/tags url = ('http://' if self.insecure else 'https://') + self.url url = url + '/v1/repositories/{0}/tags'.format( imageinfo.name_str(registry=False, tag=False)) ret = False try: logger.debug('Polling %s to see if image %s exists', url, imageinfo.name_str()) r = requests.get(url) if r.status_code == 200: tags = anymarkup.parse(r.text) look_for_tag = imageinfo.imagename.tag or 'latest' if look_for_tag not in tags: raise AtomicappBuilderException( 'Image "{0}" exists, but it doesn\'t have tag "{1}"'.format( imageinfo.name_str(), look_for_tag) ) logger.debug('Image %s exists', imageinfo.name_str()) ret = True else: logger.debug('Image %s does not exist', imageinfo.name_str()) except requests.exceptions.SSLError as e: raise AtomicappBuilderException('SSL error while polling registry: {0}'.format(e)) except Exception as e: logger.debug( 'Image %s does not seem to exist, exception was: %s', imageinfo.name_str(), e) return ret
def _add_mvn_results(self, result_summary, anitya_mvn_names, version): def _compare_version(downstream, upstream): dv = downstream if 'redhat' in dv: # remove ".redhat-X" or "-redhat-X" suffix dv = dv[:dv.find('redhat')-1] if dv == upstream: return True else: return False downstream_rebuilds = [] for name in anitya_mvn_names: metadata_url = '{repo}/{pkg}/maven-metadata.xml'.format(repo=RH_MVN_GA_REPO, pkg=mvn_pkg_to_repo_path(name)) res = requests.get(metadata_url) if res.status_code != 200: self.log.info('Metadata for package {pkg} not found in {repo} (status {code})'. format(pkg=name, repo=RH_MVN_GA_REPO, code=res.status_code)) continue versions = anymarkup.parse(res.text)['metadata']['versioning']['versions']['version'] # make sure 'versions' is a list (it's a string if there is just one version) if not isinstance(versions, list): versions = [versions] self.log.info('Found versions {v} for package {p}'.format(v=versions, p=name)) for v in versions: if _compare_version(v, version): downstream_rebuilds.append(v) result_summary['rh_mvn_matched_versions'] = downstream_rebuilds if downstream_rebuilds: # For now, we don't distinguish products, we just use general "Middleware" # for all Maven artifacts result_summary['all_rhsm_product_names'].append('Middleware')
def export_all(self): """ only kubernetes things for now """ # Resources to export. # Don't export Pods for now. # Exporting ReplicationControllers should be enough. # Ideally this should detect Pods that are not created by # ReplicationController and only export those. resources = [ "replicationcontrollers", "persistentvolumeclaims", "services" ] # output of this export is kind List args = ["export", ",".join(resources), "-o", "json"] ec, stdout, stderr = self._call_oc(args) objects = anymarkup.parse(stdout, format="json", force_types=None) image_infos = [] for o in objects["items"]: if o["kind"] == "ReplicationController": image_infos.extend(self.get_image_info(o)) for ii in image_infos: if ii["private"]: logger.warning("{kind} {name} has image that appears to be " "from local OpenShift registry!!".format(**ii)) return objects
def read_cccp_index(self): if self.cccp_index_uri.startswith('file://'): file_to_read = self.cccp_index_uri[len('file://'):] self.cccp_index = anymarkup.parse_file(file_to_read) else: fetched = requests.get(self.cccp_index_uri) self.cccp_index = anymarkup.parse(fetched.text)
def fetch_provider_resource(path): gqlapi = gql.get_api() # get resource data try: resource = gqlapi.get_resource(path) except gql.GqlApiError as e: raise FetchResourceError(e.message) try: resource['body'] = anymarkup.parse(resource['content'], force_types=None) except anymarkup.AnyMarkupError: e_msg = "Could not parse data. Skipping resource: {}" raise FetchResourceError(e_msg.format(path)) openshift_resource = OR(resource['body']) try: openshift_resource.verify_valid_k8s_object() except (KeyError, TypeError) as e: k = e.__class__.__name__ e_msg = "Invalid data ({}). Skipping resource: {}" raise FetchResourceError(e_msg.format(k, path)) return openshift_resource
def get_packit_config_from_repo(sourcegit_project: GitProject, ref: str) -> Optional[PackageConfig]: for config_file_name in CONFIG_FILE_NAMES: try: config_file = sourcegit_project.get_file_content( path=config_file_name, ref=ref) logger.debug( f"Found a config file '{config_file_name}' " f"on ref '{ref}' " f"of the {sourcegit_project.full_repo_name} repository.") except FileNotFoundError: logger.debug( f"The config file '{config_file_name}' " f"not found on ref '{ref}' " f"of the {sourcegit_project.full_repo_name} repository.") continue try: loaded_config = anymarkup.parse(config_file) except Exception as ex: logger.error(f"Cannot load package config '{config_file_name}'.") raise Exception(f"Cannot load package config: {ex}.") return parse_loaded_config(loaded_config=loaded_config) return None
def get_data(self, exclude_urls=()): response = requests.get(self.url, headers=self.request_headers) entries = anymarkup.parse(response.text) data_list = [] for entry in entries['source']['vacancies']['vacancy']: if entry['url'] in exclude_urls: continue if not words_in_string( self.keywords, entry['description'] ) and not words_in_string( self.keywords, entry['job-name'] ): continue data_list.append({ 'url': entry['url'], 'source_datetime': datetime.strptime( entry['creation-date'][:19], self.pub_date_format ), 'text': entry['description'], 'title': entry['job-name'], }) return data_list
def get_slack_usernames_from_github_owners(github_owners, users): all_slack_usernames = [] for owners_file in github_owners or []: r = requests.get(owners_file) try: owners_file = anymarkup.parse( r.content, force_types=None ) github_users = [u for l in owners_file.values() for u in l] except (anymarkup.AnyMarkupError, KeyError): msg = "Could not parse data. Skipping owners file: {}" logging.warning(msg.format(owners_file)) continue if not github_users: continue slack_usernames = [get_slack_username(u) for u in users if u['github_username'] in github_users] if len(set(slack_usernames)) != len(set(github_users)): msg = ( 'found Slack usernames {} ' 'do not match all github usernames: {} ' '(hint: user is missing from app-interface)' ).format(slack_usernames, github_users) logging.warning(msg) else: all_slack_usernames.extend(slack_usernames) return all_slack_usernames
def fetch_openshift_resource(resource, parent): global _log_lock provider = resource['provider'] path = resource['path'] msg = "Fetching {}: {}".format(provider, path) _log_lock.acquire() logging.debug(msg) _log_lock.release() if provider == 'resource': validate_json = resource.get('validate_json') or False openshift_resource = \ fetch_provider_resource(path, validate_json=validate_json) elif provider == 'resource-template': tv = {} if resource['variables']: tv = anymarkup.parse(resource['variables'], force_types=None) tv['resource'] = resource tv['resource']['namespace'] = parent tt = resource['type'] tt = 'jinja2' if tt is None else tt if tt == 'jinja2': tfunc = process_jinja2_template elif tt == 'extracurlyjinja2': tfunc = process_extracurlyjinja2_template else: UnknownTemplateTypeError(tt) try: openshift_resource = fetch_provider_resource(path, tfunc=tfunc, tvars=tv) except Exception as e: msg = "could not render template at path {}\n{}".format(path, e) raise ResourceTemplateRenderError(msg) elif provider == 'vault-secret': version = resource['version'] rn = resource['name'] name = path.split('/')[-1] if rn is None else rn rl = resource['labels'] labels = {} if rl is None else json.loads(rl) ra = resource['annotations'] annotations = {} if ra is None else json.loads(ra) rt = resource['type'] type = 'Opaque' if rt is None else rt try: openshift_resource = \ fetch_provider_vault_secret(path, version, name, labels, annotations, type) except vault_client.SecretVersionNotFound as e: raise FetchVaultSecretError(e) elif provider == 'route': tls_path = resource['vault_tls_secret_path'] tls_version = resource['vault_tls_secret_version'] openshift_resource = fetch_provider_route(path, tls_path, tls_version) else: raise UnknownProviderError(provider) return openshift_resource
def fetch_schema(schema_url): if schema_url.startswith('http'): r = requests.get(schema_url) r.raise_for_status() schema = r.text return anymarkup.parse(schema, force_types=None) else: raise MissingSchemaFile(schema_url)
def loadArtifact(self, path): data = super(self.__class__, self).loadArtifact(path) self.template_data = anymarkup.parse(data, force_types=None) if "kind" in self.template_data and self.template_data["kind"].lower() == "template": if "parameters" in self.template_data: return anymarkup.serialize(self.template_data["parameters"], format="json") return data
def process_template(template, values): try: manifest = template % values return OR(anymarkup.parse(manifest, force_types=None), QONTRACT_INTEGRATION, QONTRACT_INTEGRATION_VERSION) except KeyError as e: raise ConstructResourceError( 'could not process template: missing key {}'.format(e))
def fetch_schema(schemas_root, schema_url): if schema_url.startswith('http'): r = requests.get(schema_url) r.raise_for_status() schema = r.text else: schema = fetch_schema_file(schemas_root, schema_url) return anymarkup.parse(schema, force_types=None)
def __init__(self, keyword_file=None, lemmatizer=False, stemmer=None): # pylint: disable=too-many-branches """Construct. :param keyword_file: a path to keyword file :param lemmatizer: lematizer instance to be used :param stemmer: stemmer instance to be used """ self._stemmer = stemmer or defaults.DEFAULT_STEMMER self._lemmatizer = lemmatizer or defaults.DEFAULT_LEMMATIZER if isinstance(keyword_file, str) or keyword_file is None: with open(keyword_file or self._DEFAULT_KEYWORD_FILE_PATH, 'r') as f: content = f.read() elif isinstance(keyword_file, io.TextIOBase): content = keyword_file.read() else: raise InvalidInputError("Unknown keyword file provided - %s" % (type(keyword_file))) self._keywords = anymarkup.parse(content) del content # make sure keywords are strings self._keywords = dict((str(keyword), val) for keyword, val in self._keywords.items()) # add missing default values for keyword in self._keywords.keys(): if self._keywords[keyword] is None: self._keywords[keyword] = {'synonyms': [], 'regexp': []} if self._keywords[keyword].get('synonyms') is None: self._keywords[keyword]['synonyms'] = [] # make sure synonyms are strings self._keywords[keyword]['synonyms'] = list(map(str, self._keywords[keyword]['synonyms'])) if self._keywords[keyword].get('regexp') is None: self._keywords[keyword]['regexp'] = [] for keyword, entry in self._keywords.items(): for idx, regexp in enumerate(entry['regexp']): entry['regexp'][idx] = re.compile(regexp) entry['synonyms'].append(keyword) for idx, synonym in enumerate(entry['synonyms']): for delim in [' ', '_', '-']: synonyms = synonym.split(delim) if self._lemmatizer: synonyms = [self._lemmatizer.lemmatize(t) for t in synonyms] if self._stemmer: synonyms = [self._stemmer.stem(t) for t in synonyms] new_synonym = delim.join(synonyms) if new_synonym != synonym: _logger.debug("Stemmed and lemmatized keyword synonym from '%s' to '%s' for keyword '%s'", synonym, new_synonym, keyword) entry['synonyms'][idx] = new_synonym
def saveArtifact(self, path, data): if self.template_data: if "kind" in self.template_data and self.template_data["kind"].lower() == "template": if "parameters" in self.template_data: passed_data = anymarkup.parse(data, force_types=None) self.template_data["parameters"] = passed_data data = anymarkup.serialize(self.template_data, format=os.path.splitext(path)[1].strip(".")) #FIXME super(self.__class__, self).saveArtifact(path, data)
def prepareOrder(self): for artifact in self.artifacts: data = None with open(os.path.join(self.path, artifact), "r") as fp: logger.debug(os.path.join(self.path, artifact)) data = anymarkup.parse(fp) if "kind" in data: self.kube_order[data["kind"].lower()] = artifact else: raise ProviderFailedException("Malformed kube file")
def loadArtifact(self, path): data = super(self.__class__, self).loadArtifact(path) self.template_data = anymarkup.parse(data, force_types=None) if "kind" in self.template_data and self.template_data["kind"].lower( ) == "template": if "parameters" in self.template_data: return anymarkup.serialize(self.template_data["parameters"], format="json") return data
def call(self, method, kwargs): method_handler = getattr(self.core, method) if kwargs: kwargs = anymarkup.parse(kwargs) else: kwargs = {} if method_handler: logger.debug("Calling method %s with args\n%s" % (method, kwargs)) method_handler(**kwargs)
def call(self, method, kwargs): method_handler = getattr(self.core, method) if kwargs: kwargs = anymarkup.parse(kwargs) else: kwargs = {} if method_handler: logger.debug("Calling method %s with args\n%s", method, kwargs) method_handler(**kwargs)
def load_from_path(cls, src, config=None, namespace=GLOBAL_CONF, nodeps=False, dryrun=False, update=False): """ Load a Nulecule application from a path in the source path itself, or in the specified destination path. Args: src (str): Path to load Nulecule application from. config (dict): Config data for Nulecule application. namespace (str): Namespace for Nulecule application. nodeps (bool): Do not pull external applications if True. dryrun (bool): Do not make any change to underlying host. update (bool): Update existing application if True, else reuse it. Returns: A Nulecule instance or None in case of some dry run (fetching an image). """ nulecule_path = os.path.join(src, MAIN_FILE) if os.path.exists(nulecule_path): with open(nulecule_path, 'r') as f: nulecule_data = f.read() else: raise NuleculeException( "No Nulecule file exists in directory: %s" % src) if dryrun and not os.path.exists(nulecule_path): raise NuleculeException( "Fetched Nulecule components are required to initiate dry-run. " "Please specify your app via atomicapp --dry-run /path/to/your-app" ) # By default, AnyMarkup converts all formats to YAML when parsing. # Thus the rescue works either on JSON or YAML. try: nulecule_data = anymarkup.parse(nulecule_data) except (yaml.parser.ParserError, AnyMarkupError), e: line = re.search('line (\d+)', str(e)).group(1) column = re.search('column (\d+)', str(e)).group(1) output = "" for i, l in enumerate(nulecule_data.splitlines()): if (i == int(line) - 1) or (i == int(line)) or (i == int(line) + 1): output += "%s %s\n" % (str(i), str(l)) raise NuleculeException( "Failure parsing %s file. Validation error on line %s, column %s:\n%s" % (nulecule_path, line, column, output))
def apply_pointers(self, content, params): """ Let's apply all the json pointers! Valid params in Nulecule: param1: - /spec/containers/0/ports/0/hostPort - /spec/containers/0/ports/0/hostPort2 or param1: - /spec/containers/0/ports/0/hostPort, /spec/containers/0/ports/0/hostPort2 Args: content (str): content of artifact file params (dict): list of params with pointers to replace in content Returns: str: content with replaced pointers Todo: In the future we need to change this to detect haml, yaml, etc as we add more providers Blocked by: github.com/bkabrda/anymarkup-core/blob/master/anymarkup_core/__init__.py#L393 """ obj = anymarkup.parse(content) if type(obj) != dict: logger.debug( "Artifact file not json/haml, assuming it's $VARIABLE substitution" ) return content if params is None: # Nothing to do here! return content for name, pointers in params.items(): if not pointers: logger.warning("Could not find pointer for %s" % name) continue for pointer in pointers: try: resolve_pointer(obj, pointer) set_pointer(obj, pointer, name) logger.debug("Replaced %s pointer with %s param" % (pointer, name)) except JsonPointerException: logger.debug("Error replacing %s with %s" % (pointer, name)) logger.debug("Artifact content: %s", obj) raise NuleculeException( "Error replacing pointer %s with %s." % (pointer, name)) return anymarkup.serialize(obj, format="json")
def export_project(self): """ Export configuration from Openshift for various providers Returns: A dict with keys as provider and value as artifacts corresponding to that provider. """ # Resources to export. # Don't export Pods for now. # Exporting ReplicationControllers should be enough. # Ideally this should detect Pods that are not created by # ReplicationController and only export those. # Order in resource list is significant! Object are exported in same # order as they are specified on command line and they will have same # order in Nulecule file also. # ImageStream is first as workaround for this https://github.com/openshift/origin/issues/4518 # But this workaround is going to work only after resolving # https://github.com/projectatomic/atomicapp/issues/669 all_artifacts = {} for provider in NULECULE_PROVIDERS: if provider == "kubernetes": resources = ["persistentVolumeClaim", "service", "replicationController"] elif provider == "openshift": resources = ["imageStream", "service", "persistentVolumeClaim", "replicationController", "deploymentConfig", "buildConfig"] # output of this export is kind List args = ["export", ",".join(resources), "-o", "json"] # if user has specified the selector append it to command if self.selector: args.extend(["-l", self.selector]) ec, stdout, stderr = self._call_oc(args) objects = anymarkup.parse(stdout, format="json", force_types=None) # convert OpenShift List to array if objects["kind"] == "List": artifacts = objects["items"] else: msg = "Output of `oc export` command is of diferent kind than 'List'" logger.critical(msg) raise Exception(msg) all_artifacts[provider] = artifacts ep = ExportedProject(artifacts=all_artifacts) return ep
def get_j_config(self): """ Retrieve a job config from Jenkins. """ logger.info("Getting %s job config", self.job_name) job_config = self.server.get_job_config(self.job_name) # Convert to YAML job_config_yaml = anymarkup.serialize(anymarkup.parse(job_config), 'yaml') with open(self.config_path, 'w') as f: f.write(job_config_yaml) logger.info(" %s job config has been saved to %s", self.job_name, self.config_path)
def convert(filename, from_format, to_format, interpolate): """Parses stdin or a file and converts to the specified format""" # Try to parse the file and output in the specified format data = anymarkup.parse(filename, format=from_format, interpolate=interpolate) serialized = anymarkup.serialize(data, format=to_format) click.echo(serialized) # Exit if all done sys.exit(0)
def saveArtifact(self, path, data): if self.template_data: if "kind" in self.template_data and self.template_data[ "kind"].lower() == "template": if "parameters" in self.template_data: passed_data = anymarkup.parse(data, force_types=None) self.template_data["parameters"] = passed_data data = anymarkup.serialize( self.template_data, format=os.path.splitext(path)[1].strip(".")) #FIXME super(self.__class__, self).saveArtifact(path, data)
def get_values(self, path): gqlapi = gql.get_api() try: raw_values = gqlapi.get_resource(path) except gql.GqlApiError as e: raise FetchResourceError(e.message) try: values = anymarkup.parse(raw_values['content'], force_types=None) except anymarkup.AnyMarkupError: e_msg = "Could not parse data. Skipping resource: {}" raise FetchResourceError(e_msg.format(path)) return values
def export_project(self): """ Export configuration from Openshift for various providers Returns: A dict with keys as provider and value as artifacts corresponding to that provider. """ # Resources to export. # Don't export Pods for now. # Exporting ReplicationControllers should be enough. # Ideally this should detect Pods that are not created by # ReplicationController and only export those. # Order in resource list is significant! Object are exported in same # order as they are specified on command line and they will have same # order in Nulecule file also. # ImageStream is first as workaround for this https://github.com/openshift/origin/issues/4518 # But this workaround is going to work only after resolving # https://github.com/projectatomic/atomicapp/issues/669 all_artifacts = {} for provider in NULECULE_PROVIDERS: if provider == "kubernetes": resources = [ "persistentVolumeClaim", "service", "replicationController" ] elif provider == "openshift": resources = [ "imageStream", "service", "persistentVolumeClaim", "replicationController", "deploymentConfig", "buildConfig" ] # output of this export is kind List args = ["export", ",".join(resources), "-o", "json"] # if user has specified the selector append it to command if self.selector: args.extend(["-l", self.selector]) ec, stdout, stderr = self._call_oc(args) objects = anymarkup.parse(stdout, format="json", force_types=None) # convert OpenShift List to array if objects["kind"] == "List": artifacts = objects["items"] else: msg = "Output of `oc export` command is of diferent kind than 'List'" logger.critical(msg) raise Exception(msg) all_artifacts[provider] = artifacts ep = ExportedProject(artifacts=all_artifacts) return ep
def _convert_config_file_to_json_dict(filename): """ Parses the provided file to a dictionary then converts to JSON then to dictionary. Needs to do a double trip because the anymarkup module autoconverts booleans and integers. :param filename: filename to convert :return: JSON data as a dictionary """ config = ConfigObj(filename, interpolation=False, list_values=False) dict_data = config.dict() json_data = json.dumps(dict_data) json_dict = anymarkup.parse(json_data) return json_dict
def _process_artifacts(self): """ Parse OpenShift manifests files and checks if manifest under process is valid. Reads self.artifacts and saves parsed artifacts to self.openshift_artifacts """ for artifact in self.artifacts: logger.debug("Processing artifact: %s", artifact) data = None with open(os.path.join(self.path, artifact), "r") as fp: data = anymarkup.parse(fp, force_types=None) self._process_artifact_data(artifact, data)
def pull_images(self, registry, username, password, only_internal=True): """ This pulls all images that are mentioned in artifact. Args: registry (str): url of exposed OpenShift Docker registry username (str): username for for OpenShift Docker registry password (str): password for OpenShift Docker registry only_internal (bool): if True only images that are in internal OpenShift Docker registry, otherwise pulls all images (default is True) """ logger.debug("Pulling images (only_internal: {}, registry:{}," " login:{}:{})".format(only_internal, registry, username, password)) docker_client = docker.Client(base_url='unix://var/run/docker.sock', version='auto') try: login_response = docker_client.login(username=username, password=password, registry=registry) logger.debug(login_response) except docker.errors.APIError as e: logger.critical(e) raise Exception(e) for image_info in self.images: if image_info["internal"]: image_info["image"] = utils.replace_registry_host( image_info["image"], registry) else: if only_internal: # we are exporting only internal images, skip this continue image = image_info["image"] logger.info("Pulling image {}".format(image)) for line in docker_client.pull(image, stream=True, insecure_registry=True): line_info = anymarkup.parse(line) if "progress" in line_info: # don't print progress information # showing status is enough for now continue elif "status" in line_info: logger.info(line_info["status"]) elif "errorDetail" in line_info: msg = line_info["errorDetail"]["message"] logger.critical(msg) raise Exception(msg)
def apply_pointers(self, content, params): """ Let's apply all the json pointers! Valid params in Nulecule: param1: - /spec/containers/0/ports/0/hostPort - /spec/containers/0/ports/0/hostPort2 or param1: - /spec/containers/0/ports/0/hostPort, /spec/containers/0/ports/0/hostPort2 Args: content (str): content of artifact file params (dict): list of params with pointers to replace in content Returns: str: content with replaced pointers Todo: In the future we need to change this to detect haml, yaml, etc as we add more providers Blocked by: github.com/bkabrda/anymarkup-core/blob/master/anymarkup_core/__init__.py#L393 """ obj = anymarkup.parse(content) if type(obj) != dict: logger.debug("Artifact file not json/haml, assuming it's $VARIABLE substitution") return content if params is None: # Nothing to do here! return content for name, pointers in params.items(): if not pointers: logger.warning("Could not find pointer for %s" % name) continue for pointer in pointers: try: resolve_pointer(obj, pointer) set_pointer(obj, pointer, name) logger.debug("Replaced %s pointer with %s param" % (pointer, name)) except JsonPointerException: logger.debug("Error replacing %s with %s" % (pointer, name)) logger.debug("Artifact content: %s", obj) raise NuleculeException("Error replacing pointer %s with %s." % (pointer, name)) return anymarkup.serialize(obj, format="json")
def _dependencies_from_pom_xml(): """ Extract dependencies from pom.xml in current directory :return: {"groupId:artifactId": "version"} """ solved = {} with open('pom.xml') as r: pom_dict = anymarkup.parse(r.read()) dependencies = pom_dict.get('project', {}).get('dependencies', {}).get('dependency', []) if not isinstance(dependencies, list): dependencies = [dependencies] for dependency in dependencies: name = "{}:{}".format(dependency['groupId'], dependency['artifactId']) solved[name] = dependency['version'] return solved
def _process_artifacts(self): """ Parse each Kubernetes file and convert said format into an Object for deployment. """ for artifact in self.artifacts: logger.debug("Processing artifact: %s", artifact) data = None # Open and parse the artifact data with open(os.path.join(self.path, artifact), "r") as fp: data = anymarkup.parse(fp, force_types=None) # Process said artifacts self._process_artifact_data(artifact, data)
def convert_to_yaml_without_none(txt): """ Convert any object to OrderDict without None value """ raw = anymarkup.parse(txt) raw = remove_none(raw) represent_dict_order = lambda self, data: self.represent_mapping( 'tag:yaml.org,2002:map', data.items()) # noqa yaml.add_representer(OrderedDict, represent_dict_order) txt = yaml.dump(raw, default_flow_style=False, width=10000, allow_unicode=True) return txt
def load_from_path(cls, src, config=None, namespace=GLOBAL_CONF, nodeps=False, dryrun=False, update=False): """ Load a Nulecule application from a path in the source path itself, or in the specified destination path. Args: src (str): Path to load Nulecule application from. config (atomicapp.nulecule.config.Config): Config data for Nulecule application. namespace (str): Namespace for Nulecule application. nodeps (bool): Do not pull external applications if True. dryrun (bool): Do not make any change to underlying host. update (bool): Update existing application if True, else reuse it. Returns: A Nulecule instance or None in case of some dry run (fetching an image). """ nulecule_path = os.path.join(src, MAIN_FILE) if os.path.exists(nulecule_path): with open(nulecule_path, 'r') as f: nulecule_data = f.read() else: raise NuleculeException("No Nulecule file exists in directory: %s" % src) if dryrun and not os.path.exists(nulecule_path): raise NuleculeException("Fetched Nulecule components are required to initiate dry-run. " "Please specify your app via atomicapp --dry-run /path/to/your-app") # By default, AnyMarkup converts all formats to YAML when parsing. # Thus the rescue works either on JSON or YAML. try: nulecule_data = anymarkup.parse(nulecule_data) except (yaml.parser.ParserError, AnyMarkupError), e: line = re.search('line (\d+)', str(e)).group(1) column = re.search('column (\d+)', str(e)).group(1) output = "" for i, l in enumerate(nulecule_data.splitlines()): if (i == int(line) - 1) or (i == int(line)) or (i == int(line) + 1): output += "%s %s\n" % (str(i), str(l)) raise NuleculeException("Failure parsing %s file. Validation error on line %s, column %s:\n%s" % (nulecule_path, line, column, output))
def get_data(self, exclude_urls=()): response = requests.get(self.url, headers=self.request_headers) entries = anymarkup.parse(response.text) data_list = [] for entry in entries['rss']['channel']['item']: if entry['link'] in exclude_urls: continue data_list.append({ 'url': entry['link'], 'source_datetime': datetime.strptime( entry['pubDate'], self.pub_date_format ), 'text': entry['description'], 'title': entry['title'], 'icon_url': entry.get('image', None) }) return data_list
def deploy(self): kube_order = OrderedDict([("service", None), ("rc", None), ("pod", None)]) #FIXME for artifact in self.artifacts: data = None with open(os.path.join(self.path, artifact), "r") as fp: logger.debug(os.path.join(self.path, artifact)) data = anymarkup.parse(fp) if "kind" in data: kube_order[data["kind"].lower()] = artifact else: raise ProviderFailedException("Malformed kube file") for artifact in kube_order: if not kube_order[artifact]: continue k8s_file = os.path.join(self.path, kube_order[artifact]) self._callK8s(k8s_file)
def export_project(self): """ only kubernetes things for now """ # Resources to export. # Don't export Pods for now. # Exporting ReplicationControllers should be enough. # Ideally this should detect Pods that are not created by # ReplicationController and only export those. resources = ["replicationcontrollers", "persistentvolumeclaims", "services"] # output of this export is kind List args = ["export", ",".join(resources), "-o", "json"] ec, stdout, stderr = self._call_oc(args) objects = anymarkup.parse(stdout, format="json", force_types=None) ep = ExportedProject(artifacts=objects) return ep
def javancss(self, source_path): """Run JavaNCSS tool http://www.kclee.de/clemens/java/javancss :param source_path: path to source codes :return normalized output """ javancss_path = os.path.join(os.environ['JAVANCSS_PATH'], 'bin', 'javancss') command = [javancss_path, '-all', '-xml', source_path] status, output, error = self._run_analyzer(command, json_output=False) if status != 0: self.log.warning("JavaNCSS tool reported some errors: %s", error) if output: output = anymarkup.parse("".join(output)) output = self._normalize_javancss_output(output) return output
def read_manifest(manifest_file): if manifest_file == '-': manifest_raw = sys.stdin.read() if not manifest_raw: raise argparse.ArgumentTypeError("Received empty data from STDIN") else: try: with open(manifest_file) as f: manifest_raw = f.read() except FileNotFoundError: raise argparse.ArgumentTypeError( "File not found: {}".format(manifest_file)) try: manifest = anymarkup.parse(manifest_raw, force_types=None) except anymarkup.AnyMarkupError: raise argparse.ArgumentTypeError( "Could not parse file: {}".format(manifest_file)) return manifest
def process_k8s_artifacts(self): """Processes Kubernetes manifests files and checks if manifest under process is valid. """ for artifact in self.artifacts: data = None with open(os.path.join(self.path, artifact), "r") as fp: logger.debug(os.path.join(self.path, artifact)) try: data = anymarkup.parse(fp) except Exception: msg = "Error processing %s artifcats, Error:" % os.path.join( self.path, artifact) printErrorStatus(msg) raise if "kind" in data: self.k8s_manifests.append((data["kind"].lower(), artifact)) else: apath = os.path.join(self.path, artifact) raise ProviderFailedException("Malformed kube file: %s" % apath)
def _parse_kubeconf(self, filename): """" Parse kubectl config file Args: filename (string): path to configuration file (e.g. ./kube/config) Returns: dict of parsed values from config Example of expected file format: apiVersion: v1 clusters: - cluster: server: https://10.1.2.2:8443 certificate-authority: path-to-ca.cert insecure-skip-tls-verify: false name: 10-1-2-2:8443 contexts: - context: cluster: 10-1-2-2:8443 namespace: test user: test-admin/10-1-2-2:8443 name: test/10-1-2-2:8443/test-admin current-context: test/10-1-2-2:8443/test-admin kind: Config preferences: {} users: - name: test-admin/10-1-2-2:8443 user: token: abcdefghijklmnopqrstuvwxyz0123456789ABCDEF """ logger.debug("Parsing %s", filename) with open(filename, 'r') as fp: kubecfg = anymarkup.parse(fp.read()) try: return self._parse_kubeconf_data(kubecfg) except ProviderFailedException: raise ProviderFailedException('Invalid %s' % filename)
def deploy(self): kube_order = OrderedDict([("service", None), ("rc", None), ("pod", None)]) #FIXME for artifact in self.artifacts: data = None artifact_path = os.path.join(self.path, artifact) with open(artifact_path, "r") as fp: data = anymarkup.parse(fp, force_types=None) if "kind" in data: if data["kind"].lower() == "template": logger.info("Processing template") artifact = self._processTemplate(artifact_path) kube_order[data["kind"].lower()] = artifact else: raise ProviderFailedException("Malformed artifact file") for artifact in kube_order: if not kube_order[artifact]: continue k8s_file = os.path.join(self.path, kube_order[artifact]) self._callCli(k8s_file)
def _process_artifacts(self): """ Parse and validate Marathon artifacts Parsed artifacts are saved to self.marathon_artifacts """ for artifact in self.artifacts: logger.debug("Procesesing artifact: %s", artifact) data = None with open(os.path.join(self.path, artifact), "r") as fp: try: data = anymarkup.parse(fp) logger.debug("Parsed artifact %s", data) # every marathon app has to have id. 'id' key is also used for showing messages if "id" not in data.keys(): msg = "Error processing %s artifact. There is no id" % artifact printErrorStatus(msg) raise ProviderFailedException(msg) except anymarkup.AnyMarkupError, e: msg = "Error processing artifact - %s" % e printErrorStatus(msg) raise ProviderFailedException(msg) self.marathon_artifacts.append(data)
def _process_artifacts(self): """ Parse and validate Marathon artifacts Parsed artifacts are saved to self.marathon_artifacts """ for artifact in self.artifacts: logger.debug("Procesesing artifact: %s", artifact) data = None with open(os.path.join(self.path, artifact), "r") as fp: try: # env variables in marathon artifacts have to be string:string # force_types=None respects types from json file data = anymarkup.parse(fp, force_types=None) logger.debug("Parsed artifact %s", data) # every marathon app has to have id. 'id' key is also used for showing messages if "id" not in data.keys(): msg = "Error processing %s artifact. There is no id" % artifact cockpit_logger.error(msg) raise ProviderFailedException(msg) except anymarkup.AnyMarkupError, e: msg = "Error processing artifact - %s" % e cockpit_logger.error(msg) raise ProviderFailedException(msg) self.marathon_artifacts.append(data)
def push_images(self, registry, username, password, only_internal=True): """ This pushes all images that are mentioned in artifact. Args: registry (str): url of registry username (str): username for docker registry. If None (don't autheticate to registry) password (str): password for docker registry only_internal (bool): if True only images that are in internal OpenShift Docker registry, otherwise pulls all images (default is True) """ logger.debug("pushing images to registry only_internal: {}, " "registry:{}, login:{}:{}".format(only_internal, registry, username, password)) docker_client = docker.Client(base_url='unix://var/run/docker.sock', version='auto') if username and password: try: login_response = docker_client.login(username=username, password=password, registry=registry) logger.debug(login_response) except docker.errors.APIError as e: logger.critical(e) raise Exception(e) for image_info in self.images: if only_internal and not image_info["internal"]: # skip this image continue image = image_info["image"] # new name of image (only replace registry part) name_new_registry = utils.replace_registry_host(image, registry) (new_name, new_name_tag, new_name_digest) = utils.parse_image_name( name_new_registry) if new_name_digest: # if this is image with define digest, use digest as tag # docker cannot push image without tag, and if images # is pulled with digest it doesn't have tag specified # if this is going to be used as tag, it cannot contain ':' tag = new_name_digest.replace(":", "") else: tag = new_name_tag new_full_name = "{}:{}".format(new_name, tag) image_info["image"] = new_full_name logger.info("Tagging image {} as {}".format(image, new_full_name)) try: tag_response = docker_client.tag(image, new_name, tag, force=True) if not tag_response: msg = "Error while tagging image" logger.critical(msg) raise Exception(msg) except docker.errors.APIError as e: logger.critical(e) raise Exception(e) logger.info("Pushing image {}".format(new_full_name)) for line in docker_client.push(new_full_name, stream=True): line_info = anymarkup.parse(line) if "progress" in line_info: # don't print progress information # showing status is enough for now continue elif "status" in line_info: logger.info(line_info["status"]) elif "errorDetail" in line_info: msg = line_info["errorDetail"]["message"] logger.critical(msg) raise Exception(msg)