def generate(self, location, output_location=INDEX_GEN_DEFAULT_OUTPUT_LOC): """ Generate an index.yaml with a provided directory location """ logger.info("Generating index.yaml from %s" % location) self.index = deepcopy(self.index_template) if not os.path.isdir(location): raise Exception("Location must be a directory") for f in os.listdir(location): nulecule_dir = os.path.join(location, f) if f.startswith("."): continue if os.path.isdir(nulecule_dir): try: index_info = self._nulecule_get_info(nulecule_dir) except NuleculeException as e: logger.warning("SKIPPING %s. %s" % (nulecule_dir, e)) continue index_info["path"] = f self.index["nulecules"].append(index_info) if len(index_info) > 0: anymarkup.serialize_file(self.index, output_location, format="yaml") logger.info("index.yaml generated")
def run(self): args = self.parser.parse_args() if args.debug: logger.setLevel(logging.DEBUG) if utils.in_container() and not os.path.isabs(args.output): msg = "If running inside container --output path has to be absolute" logger.critical(msg) raise Exception(msg) nulecule_dir = utils.get_path(os.path.abspath(args.output)) if os.path.exists(nulecule_dir): msg = "{} must not exist".format(nulecule_dir) logger.critical(msg) raise Exception(msg) artifacts_dir = os.path.join(nulecule_dir, "artifacts", "kubernetes") nulecule_file = os.path.join(nulecule_dir, "Nulecule") oc = OpenshiftClient(oc=args.oc, namespace=args.project, oc_config=args.oc_config) artifacts = oc.export_all() # remove ugly thing to do :-( # I don't know hot to get securityContext and Selinux # to work on k8s for now :-( artifacts = oc.remove_securityContext(artifacts) # list of artifact for Nulecule file nulecule_artifacts = [] os.makedirs(artifacts_dir) filepath = os.path.join(artifacts_dir, "artifacts.json") nulecule_artifacts.append("file://{}".format( os.path.relpath(filepath, nulecule_dir))) anymarkup.serialize_file(artifacts, filepath, format="json") nulecule = { "specversion": "0.0.2", "id": args.project, "metadata": { "name": args.project }, "graph": [{ "name": args.project, "artifacts": { "kubernetes": nulecule_artifacts } }] } anymarkup.serialize_file(nulecule, nulecule_file, format="yaml") logger.info("Nulecule application created in {}".format( utils.remove_path(nulecule_dir)))
def loadMainfileToSchema(self): nulecule = None for element in self.params.schema["elements"]: if element["name"] == "Atomicfile": nulecule = copy.deepcopy(element["contents"]) result = copy.deepcopy(nulecule) self.fillNulecule(nulecule, result, self.params.mainfile_data) anymarkup.serialize_file(result, "test.yaml") anymarkup.serialize_file(self._generateContents(result, ask_if_null=False), "test2.yaml")
def write_service_file(self, name, output=None): """ Writes service file to disk, either to original file name, or to a name given by output param """ service_file_cont = {"services": []} for n in self._service_files[self.services[name]["file"]]: dump = copy.deepcopy(self.services[n]) filename = dump["file"] del dump["file"] service_file_cont["services"].append(dump) if not output: output = self.services[name]["file"] anymarkup.serialize_file(service_file_cont, output) print("Services written to file %s." % output)
def run(self): args = self.parser.parse_args() if args.debug: logger.setLevel(logging.DEBUG) if utils.in_container() and not os.path.isabs(args.output): msg = "If running inside container --output path has to be absolute" logger.critical(msg) raise Exception(msg) nulecule_dir = utils.get_path(os.path.abspath(args.output)) if os.path.exists(nulecule_dir): msg = "{} must not exist".format(nulecule_dir) logger.critical(msg) raise Exception(msg) artifacts_dir = os.path.join(nulecule_dir, "artifacts", "kubernetes") nulecule_file = os.path.join(nulecule_dir, "Nulecule") oc = OpenshiftClient(oc=args.oc, namespace=args.project, oc_config=args.oc_config) artifacts = oc.export_all() # remove ugly thing to do :-( # I don't know hot to get securityContext and Selinux # to work on k8s for now :-( artifacts = oc.remove_securityContext(artifacts) # list of artifact for Nulecule file nulecule_artifacts = [] os.makedirs(artifacts_dir) filepath = os.path.join(artifacts_dir, "artifacts.json") nulecule_artifacts.append("file://{}".format(os.path.relpath( filepath, nulecule_dir))) anymarkup.serialize_file(artifacts, filepath, format="json") nulecule = {"specversion": "0.0.2", "id": args.project, "metadata": {"name": args.project}, "graph": [{"name": args.project, "artifacts": {"kubernetes": nulecule_artifacts}}]} anymarkup.serialize_file(nulecule, nulecule_file, format="yaml") logger.info("Nulecule application created in {}".format( utils.remove_path(nulecule_dir)))
def _write_answers(self, path, answers, answers_format): """ Write answers data to file. Args: path (str): path to answers file to write to answers (dict): Answers data answers_format (str): Format to use to dump answers data to file, e.g., json Returns: None """ logger.debug("Writing answers to file.") logger.debug("FILE: %s", path) logger.debug("ANSWERS: %s", answers) anymarkup.serialize_file(answers, path, format=answers_format)
def _print_result(result, output_file, fmt=None): if not output_file or output_file == '-': if fmt == 'yaml' or fmt == 'yml': print(yaml.dump(result)) elif fmt == 'json' or fmt is None: print(json_dumps(result)) else: raise ValueError("Unknown output format '%s'" % fmt) else: if fmt is None: # try to guess format by file extension extension = output_file.split('.')[-1] if extension in ('yaml', 'yml', 'json'): fmt = extension if extension != 'yml' else 'yaml' else: fmt = defaults.DEFAULT_OUTPUT_FORMAT _logger.debug("Serializing output to file '%s'", output_file) anymarkup.serialize_file(result, output_file, format=fmt)
def _write_answers(self, path, answers, answers_format, dryrun=False): """ Write answers data to file. Args: path (str): path to answers file to write to answers (dict): Answers data answers_format (str): Format to use to dump answers data to file, e.g., json dryrun (bool): Do not make any change to the host system, while True Returns: None """ if not dryrun: anymarkup.serialize_file(answers, path, format=answers_format) else: logger.info('ANSWERS: %s' % answers)
def _write_answers(self, path, answers, answers_format): """ Write answers data to file. Args: path (str): path to answers file to write to answers (dict): Answers data answers_format (str): Format to use to dump answers data to file, e.g., json Returns: None """ logger.debug("Writing answers to file.") logger.debug("FILE: %s", path) logger.debug("ANSWERS: %s", answers) anymarkup.serialize_file(answers, path, format=answers_format) # Make sure that the permission of the file is set to the current user Utils.setFileOwnerGroup(path)
def _write_answers(self, path, answers, answers_format): """ Write answers data to file. Args: path (str): path to answers file to write to answers (dict): Answers data answers_format (str): Format to use to dump answers data to file, e.g., json Returns: None """ logger.debug("Writing answers to file.") logger.debug("FILE: %s", path) logger.debug("ANSWERS: %s", answers) logger.debug("ANSWERS FORMAT: %s", answers_format) anymarkup.serialize_file(answers, path, format=answers_format) # Make sure that the permission of the file is set to the current user Utils.setFileOwnerGroup(path)
def _write_answers(self, path, answers, answers_format, dryrun=False): """ Write answers data to file. Args: path (str): path to answers file to write to answers (dict): Answers data answers_format (str): Format to use to dump answers data to file, e.g., json dryrun (bool): Do not make any change to the host system, while True Returns: None """ if not dryrun: anymarkup.serialize_file( answers, path, format=answers_format) else: logger.info('ANSWERS: %s' % answers)
def write_topology_to_yamls(topology: Topology, outdir): os.makedirs(outdir, exist_ok=True) for facility_name, facility_data in topology.data.items(): facility_dir = os.path.join(outdir, facility_name) os.makedirs(facility_dir, exist_ok=True) anymarkup.serialize_file({"ID": facility_data["ID"]}, os.path.join(facility_dir, "FACILITY.yaml")) for site_name, site_data in facility_data.items(): if site_name == "ID": continue site_dir = os.path.join(facility_dir, site_name) os.makedirs(site_dir, exist_ok=True) anymarkup.serialize_file({"ID": site_data["ID"]}, os.path.join(site_dir, "SITE.yaml")) for rg_name, rg_data in site_data.items(): if rg_name == "ID": continue anymarkup.serialize_file(rg_data, os.path.join(site_dir, rg_name)) # We want to sort this so write it out ourselves for filename, data in [("support-centers.yaml", topology.support_centers), ("services.yaml", topology.services)]: with open(os.path.join(outdir, filename), "w") as outfh: for name, id_ in sorted(data.items(), key=lambda x: x[1]): print("%s: %s" % (name, id_), file=outfh)
def save(self): anymarkup.serialize_file(self.config, self.path)
def run(self): args = self.parser.parse_args() if args.debug: logger.setLevel(logging.DEBUG) logger.debug("Running with arguments {}".format(args)) if utils.in_container() and not os.path.isabs(args.output): msg = "If running inside container --output path has to be absolute path" logger.critical(msg) raise Exception(msg) if not args.skip_push and args.export_images != 'none' and not args.registry_host: msg = "With --export-images you also need set --registry-host. If you don't want to push images to registry, you have to use --skip-push" logger.critical(msg) raise Exception(msg) # validate and parse --registry-login if args.registry_login is None: registry_user = None registry_password = None elif len(args.registry_login.split(":")) == 2: registry_user = args.registry_login.split(":")[0] registry_password = args.registry_login.split(":")[1] else: msg = "Invalid format of --registry-login. Use (username:password)" logger.critical(msg) raise Exception(msg) nulecule_dir = utils.get_path(args.output) if os.path.exists(nulecule_dir): msg = "{} must not exist".format(nulecule_dir) logger.critical(msg) raise Exception(msg) artifacts_dir = os.path.join(nulecule_dir, "artifacts") provider_paths = { provider: os.path.join(artifacts_dir, provider) for provider in NULECULE_PROVIDERS } nulecule_file = os.path.join(nulecule_dir, "Nulecule") oc = OpenshiftClient(oc=args.oc, namespace=args.project, oc_config=args.oc_config, selector=args.selector) # export project info from openshift exported_project = oc.export_project() # export images if args.export_images != "none": if args.export_images == "internal": only_internal = True elif args.export_images == "all": only_internal = False exported_project.pull_images(args.oc_registry_host, oc.get_username(), oc.get_token(), only_internal) # if registy-host is not set or skip-push is set do not perform push if args.registry_host and not args.skip_push: exported_project.push_images(args.registry_host, registry_user, registry_password, only_internal) exported_project.update_artifacts_images() provider_artifacts = {} for provider, path in provider_paths.items(): # list of artifact for Nulecule file nulecule_artifacts = [] os.makedirs(path) # create artifact files for artifact in exported_project.artifacts[provider]: if "name" in artifact["metadata"]: name = artifact["metadata"]["name"] else: name = "unknown" kind = artifact["kind"] filename = "{}-{}.json".format(name, kind) filepath = os.path.join(path, filename) if os.path.exists(filepath): filepath = utils.get_new_name(filepath) nulecule_artifacts.append("file://{}".format( os.path.relpath(filepath, nulecule_dir))) anymarkup.serialize_file(artifact, filepath, format="json") provider_artifacts[provider] = nulecule_artifacts nulecule = { "specversion": "0.0.2", "id": args.project, "metadata": { "name": args.project }, "graph": [{ "name": args.project, "artifacts": provider_artifacts }] } anymarkup.serialize_file(nulecule, nulecule_file, format="yaml") utils.generate_dockerfile(nulecule_dir, args.atomicapp_ver) logger.info("Nulecule application created in {}".format( utils.remove_path(nulecule_dir)))
def writeAnswers(self, path): anymarkup.serialize_file(self.answers_data, path, format='yaml')
def writeAnswers(self, path): logger.debug("writing %s to %s with format %s", self.answers_data, path, self.answer_file_format) anymarkup.serialize_file(self.answers_data, path, format=self.answer_file_format)
def to_xml_file(data, outfile): return anymarkup.serialize_file(data, outfile, "xml")
#!/usr/bin/env python3 """Split an rgsummary.xml file into a directory of xml files of the individual <ResourceGroup>s. """ import anymarkup import os import shutil import sys import xmltodict try: infile, outdir = sys.argv[1:] except IndexError: print("Usage: %s <infile> <outdir>" % sys.argv[0], file=sys.stderr) sys.exit(2) with open(infile, "r") as infh: # Use dict constructor to get rid of ordering parsed = xmltodict.parse(infh.read(), dict_constructor=dict) if not os.path.exists(outdir): os.makedirs(outdir) rgs = parsed["ResourceSummary"]["ResourceGroup"] for rg in rgs: anymarkup.serialize_file({"ResourceGroup": rg}, os.path.join(outdir, rg["GroupName"] + ".xml"), format="xml")
def run(self): args = self.parser.parse_args() if args.debug: logger.setLevel(logging.DEBUG) logger.debug("Running with arguments {}".format(args)) if utils.in_container() and not os.path.isabs(args.output): msg = "If running inside container --output path has to be absolute path" logger.critical(msg) raise Exception(msg) if not args.skip_push and args.export_images != 'none' and not args.registry_host: msg = "With --export-images you also need set --registry-host. If you don't want to push images to registry, you have to use --skip-push" logger.critical(msg) raise Exception(msg) # validate and parse --registry-login if args.registry_login is None: registry_user = None registry_password = None elif len(args.registry_login.split(":")) == 2: registry_user = args.registry_login.split(":")[0] registry_password = args.registry_login.split(":")[1] else: msg = "Invalid format of --registry-login. Use (username:password)" logger.critical(msg) raise Exception(msg) nulecule_dir = utils.get_path(args.output) if os.path.exists(nulecule_dir): msg = "{} must not exist".format(nulecule_dir) logger.critical(msg) raise Exception(msg) artifacts_dir = os.path.join(nulecule_dir, "artifacts") provider_paths = {provider: os.path.join(artifacts_dir, provider) for provider in NULECULE_PROVIDERS} nulecule_file = os.path.join(nulecule_dir, "Nulecule") oc = OpenshiftClient(oc=args.oc, namespace=args.project, oc_config=args.oc_config, selector=args.selector) # export project info from openshift exported_project = oc.export_project() # export images if args.export_images != "none": if args.export_images == "internal": only_internal = True elif args.export_images == "all": only_internal = False exported_project.pull_images(args.oc_registry_host, oc.get_username(), oc.get_token(), only_internal) # if registy-host is not set or skip-push is set do not perform push if args.registry_host and not args.skip_push: exported_project.push_images(args.registry_host, registry_user, registry_password, only_internal) exported_project.update_artifacts_images() provider_artifacts = {} for provider, path in provider_paths.items(): # list of artifact for Nulecule file nulecule_artifacts = [] os.makedirs(path) # create artifact files for artifact in exported_project.artifacts[provider]: if "name" in artifact["metadata"]: name = artifact["metadata"]["name"] else: name = "unknown" kind = artifact["kind"] filename = "{}-{}.json".format(name, kind) filepath = os.path.join(path, filename) if os.path.exists(filepath): filepath = utils.get_new_name(filepath) nulecule_artifacts.append("file://{}".format(os.path.relpath( filepath, nulecule_dir))) anymarkup.serialize_file(artifact, filepath, format="json") provider_artifacts[provider] = nulecule_artifacts nulecule = {"specversion": "0.0.2", "id": args.project, "metadata": {"name": args.project}, "graph": [{"name": args.project, "artifacts": provider_artifacts}]} anymarkup.serialize_file(nulecule, nulecule_file, format="yaml") utils.generate_dockerfile(nulecule_dir, args.atomicapp_ver) logger.info("Nulecule application created in {}".format( utils.remove_path(nulecule_dir)))
#!/usr/bin/env python3 """Split a VOSummary XML file into a directory of XML files of the individual <VO>s. """ import anymarkup import os import shutil import sys import xmltodict try: infile, outdir = sys.argv[1:] except IndexError: print("Usage: %s <infile> <outdir>" % sys.argv[0], file=sys.stderr) sys.exit(2) with open(infile, "r") as infh: # Use dict constructor to get rid of ordering parsed = xmltodict.parse(infh.read(), dict_constructor=dict) if not os.path.exists(outdir): os.makedirs(outdir) _vos = parsed["VOSummary"]["VO"] for vo in _vos: anymarkup.serialize_file({"VO": vo}, os.path.join(outdir, vo["Name"]+".xml"), format="xml")
def run(self): args = self.parser.parse_args() if args.debug: logger.setLevel(logging.DEBUG) logger.debug("Running with arguments {}".format(args)) if utils.in_container() and not os.path.isabs(args.output): msg = "If running inside container --output path has to be absolute path" logger.critical(msg) raise Exception(msg) if args.export_images != 'none' and not args.registry_host: msg = "With --export-images you need also set --registry-host" logger.critical(msg) raise Exception(msg) # validate and parse --registry-login if args.registry_login is None: registry_user = None registry_password = None elif len(args.registry_login.split(":")) == 2: registry_user = args.registry_login.split(":")[0] registry_password = args.registry_login.split(":")[1] else: msg = "Invalid format of --registry-login. Use (username:password)" logger.critical(msg) raise Exception(msg) nulecule_dir = utils.get_path(args.output) if os.path.exists(nulecule_dir): msg = "{} must not exist".format(nulecule_dir) logger.critical(msg) raise Exception(msg) artifacts_dir = os.path.join(nulecule_dir, "artifacts", "kubernetes") nulecule_file = os.path.join(nulecule_dir, "Nulecule") oc = OpenshiftClient(oc=args.oc, namespace=args.project, oc_config=args.oc_config) # export project info from openshift exported_project = oc.export_project() # export images if args.export_images != "none": if args.export_images == "internal": only_internal = True elif args.export_images == "all": only_internal = False exported_project.pull_images(args.oc_registry_host, oc.get_username(), oc.get_token(), only_internal) # if registy-host is not set do not perform push if args.registry_host: exported_project.push_images(args.registry_host, registry_user, registry_password, only_internal) exported_project.update_artifacts_images() artifacts = exported_project.artifacts # list of artifact for Nulecule file nulecule_artifacts = [] os.makedirs(artifacts_dir) filepath = os.path.join(artifacts_dir, "artifacts.json") nulecule_artifacts.append("file://{}".format(os.path.relpath( filepath, nulecule_dir))) anymarkup.serialize_file(artifacts, filepath, format="json") nulecule = {"specversion": "0.0.2", "id": args.project, "metadata": {"name": args.project}, "graph": [{"name": args.project, "artifacts": {"kubernetes": nulecule_artifacts}}]} anymarkup.serialize_file(nulecule, nulecule_file, format="yaml") utils.generate_dockerfile(nulecule_dir) logger.info("Nulecule application created in {}".format( utils.remove_path(nulecule_dir)))