def generate(self, location, output_location=INDEX_GEN_DEFAULT_OUTPUT_LOC):
"""
Generate an index.yaml with a provided directory location
"""
logger.info("Generating index.yaml from %s" % location)
self.index = deepcopy(self.index_template)
if not os.path.isdir(location):
raise Exception("Location must be a directory")
for f in os.listdir(location):
nulecule_dir = os.path.join(location, f)
if f.startswith("."):
continue
if os.path.isdir(nulecule_dir):
try:
index_info = self._nulecule_get_info(nulecule_dir)
except NuleculeException as e:
logger.warning("SKIPPING %s. %s" % (nulecule_dir, e))
continue
index_info["path"] = f
self.index["nulecules"].append(index_info)
if len(index_info) > 0:
anymarkup.serialize_file(self.index,
output_location,
format="yaml")
logger.info("index.yaml generated")
def generate(self, location, output_location=INDEX_GEN_DEFAULT_OUTPUT_LOC):
"""
Generate an index.yaml with a provided directory location
"""
logger.info("Generating index.yaml from %s" % location)
self.index = deepcopy(self.index_template)
if not os.path.isdir(location):
raise Exception("Location must be a directory")
for f in os.listdir(location):
nulecule_dir = os.path.join(location, f)
if f.startswith("."):
continue
if os.path.isdir(nulecule_dir):
try:
index_info = self._nulecule_get_info(nulecule_dir)
except NuleculeException as e:
logger.warning("SKIPPING %s. %s" %
(nulecule_dir, e))
continue
index_info["path"] = f
self.index["nulecules"].append(index_info)
if len(index_info) > 0:
anymarkup.serialize_file(self.index, output_location, format="yaml")
logger.info("index.yaml generated")
def run(self):
args = self.parser.parse_args()
if args.debug:
logger.setLevel(logging.DEBUG)
if utils.in_container() and not os.path.isabs(args.output):
msg = "If running inside container --output path has to be absolute"
logger.critical(msg)
raise Exception(msg)
nulecule_dir = utils.get_path(os.path.abspath(args.output))
if os.path.exists(nulecule_dir):
msg = "{} must not exist".format(nulecule_dir)
logger.critical(msg)
raise Exception(msg)
artifacts_dir = os.path.join(nulecule_dir, "artifacts", "kubernetes")
nulecule_file = os.path.join(nulecule_dir, "Nulecule")
oc = OpenshiftClient(oc=args.oc,
namespace=args.project,
oc_config=args.oc_config)
artifacts = oc.export_all()
# remove ugly thing to do :-(
# I don't know hot to get securityContext and Selinux
# to work on k8s for now :-(
artifacts = oc.remove_securityContext(artifacts)
# list of artifact for Nulecule file
nulecule_artifacts = []
os.makedirs(artifacts_dir)
filepath = os.path.join(artifacts_dir, "artifacts.json")
nulecule_artifacts.append("file://{}".format(
os.path.relpath(filepath, nulecule_dir)))
anymarkup.serialize_file(artifacts, filepath, format="json")
nulecule = {
"specversion":
"0.0.2",
"id":
args.project,
"metadata": {
"name": args.project
},
"graph": [{
"name": args.project,
"artifacts": {
"kubernetes": nulecule_artifacts
}
}]
}
anymarkup.serialize_file(nulecule, nulecule_file, format="yaml")
logger.info("Nulecule application created in {}".format(
utils.remove_path(nulecule_dir)))
def loadMainfileToSchema(self):
nulecule = None
for element in self.params.schema["elements"]:
if element["name"] == "Atomicfile":
nulecule = copy.deepcopy(element["contents"])
result = copy.deepcopy(nulecule)
self.fillNulecule(nulecule, result, self.params.mainfile_data)
anymarkup.serialize_file(result, "test.yaml")
anymarkup.serialize_file(self._generateContents(result, ask_if_null=False), "test2.yaml")
def loadMainfileToSchema(self):
nulecule = None
for element in self.params.schema["elements"]:
if element["name"] == "Atomicfile":
nulecule = copy.deepcopy(element["contents"])
result = copy.deepcopy(nulecule)
self.fillNulecule(nulecule, result, self.params.mainfile_data)
anymarkup.serialize_file(result, "test.yaml")
anymarkup.serialize_file(self._generateContents(result, ask_if_null=False), "test2.yaml")
def write_service_file(self, name, output=None):
""" Writes service file to disk, either to original file name, or to a name given by output param """
service_file_cont = {"services": []}
for n in self._service_files[self.services[name]["file"]]:
dump = copy.deepcopy(self.services[n])
filename = dump["file"]
del dump["file"]
service_file_cont["services"].append(dump)
if not output:
output = self.services[name]["file"]
anymarkup.serialize_file(service_file_cont, output)
print("Services written to file %s." % output)
def run(self):
args = self.parser.parse_args()
if args.debug:
logger.setLevel(logging.DEBUG)
if utils.in_container() and not os.path.isabs(args.output):
msg = "If running inside container --output path has to be absolute"
logger.critical(msg)
raise Exception(msg)
nulecule_dir = utils.get_path(os.path.abspath(args.output))
if os.path.exists(nulecule_dir):
msg = "{} must not exist".format(nulecule_dir)
logger.critical(msg)
raise Exception(msg)
artifacts_dir = os.path.join(nulecule_dir, "artifacts", "kubernetes")
nulecule_file = os.path.join(nulecule_dir, "Nulecule")
oc = OpenshiftClient(oc=args.oc,
namespace=args.project,
oc_config=args.oc_config)
artifacts = oc.export_all()
# remove ugly thing to do :-(
# I don't know hot to get securityContext and Selinux
# to work on k8s for now :-(
artifacts = oc.remove_securityContext(artifacts)
# list of artifact for Nulecule file
nulecule_artifacts = []
os.makedirs(artifacts_dir)
filepath = os.path.join(artifacts_dir, "artifacts.json")
nulecule_artifacts.append("file://{}".format(os.path.relpath(
filepath, nulecule_dir)))
anymarkup.serialize_file(artifacts, filepath, format="json")
nulecule = {"specversion": "0.0.2",
"id": args.project,
"metadata": {"name": args.project},
"graph": [{"name": args.project,
"artifacts":
{"kubernetes": nulecule_artifacts}}]}
anymarkup.serialize_file(nulecule, nulecule_file, format="yaml")
logger.info("Nulecule application created in {}".format(
utils.remove_path(nulecule_dir)))
def _write_answers(self, path, answers, answers_format):
"""
Write answers data to file.
Args:
path (str): path to answers file to write to
answers (dict): Answers data
answers_format (str): Format to use to dump answers data to file,
e.g., json
Returns:
None
"""
logger.debug("Writing answers to file.")
logger.debug("FILE: %s", path)
logger.debug("ANSWERS: %s", answers)
anymarkup.serialize_file(answers, path, format=answers_format)
def _write_answers(self, path, answers, answers_format):
"""
Write answers data to file.
Args:
path (str): path to answers file to write to
answers (dict): Answers data
answers_format (str): Format to use to dump answers data to file,
e.g., json
Returns:
None
"""
logger.debug("Writing answers to file.")
logger.debug("FILE: %s", path)
logger.debug("ANSWERS: %s", answers)
anymarkup.serialize_file(answers, path, format=answers_format)
def _print_result(result, output_file, fmt=None):
if not output_file or output_file == '-':
if fmt == 'yaml' or fmt == 'yml':
print(yaml.dump(result))
elif fmt == 'json' or fmt is None:
print(json_dumps(result))
else:
raise ValueError("Unknown output format '%s'" % fmt)
else:
if fmt is None: # try to guess format by file extension
extension = output_file.split('.')[-1]
if extension in ('yaml', 'yml', 'json'):
fmt = extension if extension != 'yml' else 'yaml'
else:
fmt = defaults.DEFAULT_OUTPUT_FORMAT
_logger.debug("Serializing output to file '%s'", output_file)
anymarkup.serialize_file(result, output_file, format=fmt)
def _write_answers(self, path, answers, answers_format, dryrun=False):
"""
Write answers data to file.
Args:
path (str): path to answers file to write to
answers (dict): Answers data
answers_format (str): Format to use to dump answers data to file,
e.g., json
dryrun (bool): Do not make any change to the host system,
while True
Returns:
None
"""
if not dryrun:
anymarkup.serialize_file(answers, path, format=answers_format)
else:
logger.info('ANSWERS: %s' % answers)
def _write_answers(self, path, answers, answers_format):
"""
Write answers data to file.
Args:
path (str): path to answers file to write to
answers (dict): Answers data
answers_format (str): Format to use to dump answers data to file,
e.g., json
Returns:
None
"""
logger.debug("Writing answers to file.")
logger.debug("FILE: %s", path)
logger.debug("ANSWERS: %s", answers)
anymarkup.serialize_file(answers, path, format=answers_format)
# Make sure that the permission of the file is set to the current user
Utils.setFileOwnerGroup(path)
def _write_answers(self, path, answers, answers_format):
"""
Write answers data to file.
Args:
path (str): path to answers file to write to
answers (dict): Answers data
answers_format (str): Format to use to dump answers data to file,
e.g., json
Returns:
None
"""
logger.debug("Writing answers to file.")
logger.debug("FILE: %s", path)
logger.debug("ANSWERS: %s", answers)
logger.debug("ANSWERS FORMAT: %s", answers_format)
anymarkup.serialize_file(answers, path, format=answers_format)
# Make sure that the permission of the file is set to the current user
Utils.setFileOwnerGroup(path)
def _write_answers(self, path, answers, answers_format, dryrun=False):
"""
Write answers data to file.
Args:
path (str): path to answers file to write to
answers (dict): Answers data
answers_format (str): Format to use to dump answers data to file,
e.g., json
dryrun (bool): Do not make any change to the host system,
while True
Returns:
None
"""
if not dryrun:
anymarkup.serialize_file(
answers, path, format=answers_format)
else:
logger.info('ANSWERS: %s' % answers)
def write_topology_to_yamls(topology: Topology, outdir):
os.makedirs(outdir, exist_ok=True)
for facility_name, facility_data in topology.data.items():
facility_dir = os.path.join(outdir, facility_name)
os.makedirs(facility_dir, exist_ok=True)
anymarkup.serialize_file({"ID": facility_data["ID"]},
os.path.join(facility_dir, "FACILITY.yaml"))
for site_name, site_data in facility_data.items():
if site_name == "ID": continue
site_dir = os.path.join(facility_dir, site_name)
os.makedirs(site_dir, exist_ok=True)
anymarkup.serialize_file({"ID": site_data["ID"]},
os.path.join(site_dir, "SITE.yaml"))
for rg_name, rg_data in site_data.items():
if rg_name == "ID": continue
anymarkup.serialize_file(rg_data,
os.path.join(site_dir, rg_name))
# We want to sort this so write it out ourselves
for filename, data in [("support-centers.yaml", topology.support_centers),
("services.yaml", topology.services)]:
with open(os.path.join(outdir, filename), "w") as outfh:
for name, id_ in sorted(data.items(), key=lambda x: x[1]):
print("%s: %s" % (name, id_), file=outfh)
def save(self):
anymarkup.serialize_file(self.config, self.path)
def run(self):
args = self.parser.parse_args()
if args.debug:
logger.setLevel(logging.DEBUG)
logger.debug("Running with arguments {}".format(args))
if utils.in_container() and not os.path.isabs(args.output):
msg = "If running inside container --output path has to be absolute path"
logger.critical(msg)
raise Exception(msg)
if not args.skip_push and args.export_images != 'none' and not args.registry_host:
msg = "With --export-images you also need set --registry-host. If you don't want to push images to registry, you have to use --skip-push"
logger.critical(msg)
raise Exception(msg)
# validate and parse --registry-login
if args.registry_login is None:
registry_user = None
registry_password = None
elif len(args.registry_login.split(":")) == 2:
registry_user = args.registry_login.split(":")[0]
registry_password = args.registry_login.split(":")[1]
else:
msg = "Invalid format of --registry-login. Use (username:password)"
logger.critical(msg)
raise Exception(msg)
nulecule_dir = utils.get_path(args.output)
if os.path.exists(nulecule_dir):
msg = "{} must not exist".format(nulecule_dir)
logger.critical(msg)
raise Exception(msg)
artifacts_dir = os.path.join(nulecule_dir, "artifacts")
provider_paths = {
provider: os.path.join(artifacts_dir, provider)
for provider in NULECULE_PROVIDERS
}
nulecule_file = os.path.join(nulecule_dir, "Nulecule")
oc = OpenshiftClient(oc=args.oc,
namespace=args.project,
oc_config=args.oc_config,
selector=args.selector)
# export project info from openshift
exported_project = oc.export_project()
# export images
if args.export_images != "none":
if args.export_images == "internal":
only_internal = True
elif args.export_images == "all":
only_internal = False
exported_project.pull_images(args.oc_registry_host,
oc.get_username(), oc.get_token(),
only_internal)
# if registy-host is not set or skip-push is set do not perform push
if args.registry_host and not args.skip_push:
exported_project.push_images(args.registry_host, registry_user,
registry_password, only_internal)
exported_project.update_artifacts_images()
provider_artifacts = {}
for provider, path in provider_paths.items():
# list of artifact for Nulecule file
nulecule_artifacts = []
os.makedirs(path)
# create artifact files
for artifact in exported_project.artifacts[provider]:
if "name" in artifact["metadata"]:
name = artifact["metadata"]["name"]
else:
name = "unknown"
kind = artifact["kind"]
filename = "{}-{}.json".format(name, kind)
filepath = os.path.join(path, filename)
if os.path.exists(filepath):
filepath = utils.get_new_name(filepath)
nulecule_artifacts.append("file://{}".format(
os.path.relpath(filepath, nulecule_dir)))
anymarkup.serialize_file(artifact, filepath, format="json")
provider_artifacts[provider] = nulecule_artifacts
nulecule = {
"specversion": "0.0.2",
"id": args.project,
"metadata": {
"name": args.project
},
"graph": [{
"name": args.project,
"artifacts": provider_artifacts
}]
}
anymarkup.serialize_file(nulecule, nulecule_file, format="yaml")
utils.generate_dockerfile(nulecule_dir, args.atomicapp_ver)
logger.info("Nulecule application created in {}".format(
utils.remove_path(nulecule_dir)))
def writeAnswers(self, path):
anymarkup.serialize_file(self.answers_data, path, format='yaml')
def writeAnswers(self, path):
logger.debug("writing %s to %s with format %s", self.answers_data,
path, self.answer_file_format)
anymarkup.serialize_file(self.answers_data,
path,
format=self.answer_file_format)
def to_xml_file(data, outfile):
return anymarkup.serialize_file(data, outfile, "xml")
def writeAnswers(self, path):
anymarkup.serialize_file(self.answers_data, path, format='yaml')
#!/usr/bin/env python3
"""Split an rgsummary.xml file into a directory of xml files of the
individual <ResourceGroup>s.
"""
import anymarkup
import os
import shutil
import sys
import xmltodict
try:
infile, outdir = sys.argv[1:]
except IndexError:
print("Usage: %s <infile> <outdir>" % sys.argv[0], file=sys.stderr)
sys.exit(2)
with open(infile, "r") as infh:
# Use dict constructor to get rid of ordering
parsed = xmltodict.parse(infh.read(), dict_constructor=dict)
if not os.path.exists(outdir):
os.makedirs(outdir)
rgs = parsed["ResourceSummary"]["ResourceGroup"]
for rg in rgs:
anymarkup.serialize_file({"ResourceGroup": rg},
os.path.join(outdir, rg["GroupName"] + ".xml"),
format="xml")
def run(self):
args = self.parser.parse_args()
if args.debug:
logger.setLevel(logging.DEBUG)
logger.debug("Running with arguments {}".format(args))
if utils.in_container() and not os.path.isabs(args.output):
msg = "If running inside container --output path has to be absolute path"
logger.critical(msg)
raise Exception(msg)
if not args.skip_push and args.export_images != 'none' and not args.registry_host:
msg = "With --export-images you also need set --registry-host. If you don't want to push images to registry, you have to use --skip-push"
logger.critical(msg)
raise Exception(msg)
# validate and parse --registry-login
if args.registry_login is None:
registry_user = None
registry_password = None
elif len(args.registry_login.split(":")) == 2:
registry_user = args.registry_login.split(":")[0]
registry_password = args.registry_login.split(":")[1]
else:
msg = "Invalid format of --registry-login. Use (username:password)"
logger.critical(msg)
raise Exception(msg)
nulecule_dir = utils.get_path(args.output)
if os.path.exists(nulecule_dir):
msg = "{} must not exist".format(nulecule_dir)
logger.critical(msg)
raise Exception(msg)
artifacts_dir = os.path.join(nulecule_dir, "artifacts")
provider_paths = {provider: os.path.join(artifacts_dir, provider)
for provider in NULECULE_PROVIDERS}
nulecule_file = os.path.join(nulecule_dir, "Nulecule")
oc = OpenshiftClient(oc=args.oc,
namespace=args.project,
oc_config=args.oc_config,
selector=args.selector)
# export project info from openshift
exported_project = oc.export_project()
# export images
if args.export_images != "none":
if args.export_images == "internal":
only_internal = True
elif args.export_images == "all":
only_internal = False
exported_project.pull_images(args.oc_registry_host,
oc.get_username(),
oc.get_token(),
only_internal)
# if registy-host is not set or skip-push is set do not perform push
if args.registry_host and not args.skip_push:
exported_project.push_images(args.registry_host,
registry_user,
registry_password,
only_internal)
exported_project.update_artifacts_images()
provider_artifacts = {}
for provider, path in provider_paths.items():
# list of artifact for Nulecule file
nulecule_artifacts = []
os.makedirs(path)
# create artifact files
for artifact in exported_project.artifacts[provider]:
if "name" in artifact["metadata"]:
name = artifact["metadata"]["name"]
else:
name = "unknown"
kind = artifact["kind"]
filename = "{}-{}.json".format(name, kind)
filepath = os.path.join(path, filename)
if os.path.exists(filepath):
filepath = utils.get_new_name(filepath)
nulecule_artifacts.append("file://{}".format(os.path.relpath(
filepath, nulecule_dir)))
anymarkup.serialize_file(artifact, filepath, format="json")
provider_artifacts[provider] = nulecule_artifacts
nulecule = {"specversion": "0.0.2",
"id": args.project,
"metadata": {"name": args.project},
"graph": [{"name": args.project,
"artifacts": provider_artifacts}]}
anymarkup.serialize_file(nulecule, nulecule_file, format="yaml")
utils.generate_dockerfile(nulecule_dir, args.atomicapp_ver)
logger.info("Nulecule application created in {}".format(
utils.remove_path(nulecule_dir)))
#!/usr/bin/env python3
"""Split a VOSummary XML file into a directory of XML files of the
individual <VO>s.
"""
import anymarkup
import os
import shutil
import sys
import xmltodict
try:
infile, outdir = sys.argv[1:]
except IndexError:
print("Usage: %s <infile> <outdir>" % sys.argv[0], file=sys.stderr)
sys.exit(2)
with open(infile, "r") as infh:
# Use dict constructor to get rid of ordering
parsed = xmltodict.parse(infh.read(), dict_constructor=dict)
if not os.path.exists(outdir):
os.makedirs(outdir)
_vos = parsed["VOSummary"]["VO"]
for vo in _vos:
anymarkup.serialize_file({"VO": vo}, os.path.join(outdir, vo["Name"]+".xml"), format="xml")
def writeAnswers(self, path):
logger.debug("writing %s to %s with format %s",
self.answers_data, path, self.answer_file_format)
anymarkup.serialize_file(self.answers_data, path, format=self.answer_file_format)
def run(self):
args = self.parser.parse_args()
if args.debug:
logger.setLevel(logging.DEBUG)
logger.debug("Running with arguments {}".format(args))
if utils.in_container() and not os.path.isabs(args.output):
msg = "If running inside container --output path has to be absolute path"
logger.critical(msg)
raise Exception(msg)
if args.export_images != 'none' and not args.registry_host:
msg = "With --export-images you need also set --registry-host"
logger.critical(msg)
raise Exception(msg)
# validate and parse --registry-login
if args.registry_login is None:
registry_user = None
registry_password = None
elif len(args.registry_login.split(":")) == 2:
registry_user = args.registry_login.split(":")[0]
registry_password = args.registry_login.split(":")[1]
else:
msg = "Invalid format of --registry-login. Use (username:password)"
logger.critical(msg)
raise Exception(msg)
nulecule_dir = utils.get_path(args.output)
if os.path.exists(nulecule_dir):
msg = "{} must not exist".format(nulecule_dir)
logger.critical(msg)
raise Exception(msg)
artifacts_dir = os.path.join(nulecule_dir, "artifacts", "kubernetes")
nulecule_file = os.path.join(nulecule_dir, "Nulecule")
oc = OpenshiftClient(oc=args.oc,
namespace=args.project,
oc_config=args.oc_config)
# export project info from openshift
exported_project = oc.export_project()
# export images
if args.export_images != "none":
if args.export_images == "internal":
only_internal = True
elif args.export_images == "all":
only_internal = False
exported_project.pull_images(args.oc_registry_host,
oc.get_username(),
oc.get_token(),
only_internal)
# if registy-host is not set do not perform push
if args.registry_host:
exported_project.push_images(args.registry_host,
registry_user, registry_password,
only_internal)
exported_project.update_artifacts_images()
artifacts = exported_project.artifacts
# list of artifact for Nulecule file
nulecule_artifacts = []
os.makedirs(artifacts_dir)
filepath = os.path.join(artifacts_dir, "artifacts.json")
nulecule_artifacts.append("file://{}".format(os.path.relpath(
filepath, nulecule_dir)))
anymarkup.serialize_file(artifacts, filepath, format="json")
nulecule = {"specversion": "0.0.2",
"id": args.project,
"metadata": {"name": args.project},
"graph": [{"name": args.project,
"artifacts":
{"kubernetes": nulecule_artifacts}}]}
anymarkup.serialize_file(nulecule, nulecule_file, format="yaml")
utils.generate_dockerfile(nulecule_dir)
logger.info("Nulecule application created in {}".format(
utils.remove_path(nulecule_dir)))
