def load_yaml(fname: str, round_trip: bool = False) -> JSON_TYPE:
"""Load a YAML file."""
if round_trip:
yaml = YAML(typ='rt')
yaml.preserve_quotes = True
else:
if not hasattr(ExtSafeConstructor, 'name'):
ExtSafeConstructor.name = fname
yaml = YAML(typ='safe')
yaml.Constructor = ExtSafeConstructor
try:
with open(fname, encoding='utf-8') as conf_file:
# If configuration file is empty YAML returns None
# We convert that to an empty dict
return yaml.load(conf_file) or OrderedDict()
except YAMLError as exc:
_LOGGER.error("YAML error in %s: %s", fname, exc)
raise HomeAssistantError(exc)
except UnicodeDecodeError as exc:
_LOGGER.error("Unable to read file %s: %s", fname, exc)
raise HomeAssistantError(exc)
is_url_or_abbrev,
DEFAULT_URL_ABBREVIATIONS_REPO,
calculate_cache_location_for_url,
)
from frutils.exceptions import FrklException
from frutils.frutils import auto_parse_string
from frutils.tasks.callback import load_callback
from frutils.tasks.tasks import Tasks
from ting.tings import TingTings
log = logging.getLogger("freckles")
yaml = YAML()
yaml.default_flow_style = False
yaml.preserve_quotes = True
yaml.width = 4096
def startup_housekeeping():
if not os.path.exists(FRECKLES_CONFIG_DIR):
os.makedirs(FRECKLES_CONFIG_DIR)
else:
if not os.path.isdir(os.path.realpath(FRECKLES_CONFIG_DIR)):
raise Exception(
"Freckles config location exists and is not a directory: '{}'".
format(FRECKLES_CONFIG_DIR))
if not os.path.exists(FRECKLES_SHARE_DIR):
os.makedirs(FRECKLES_SHARE_DIR)
from importlib.util import spec_from_file_location
from importlib.util import module_from_spec
import os
import sys
import json
import string
from io import StringIO, IOBase
from ruamel.yaml import YAML
# A shiny global ruamel.yaml obj with sane options (dumps should pass yamllint)
YM = YAML()
YM.indent(mapping=2, sequence=4, offset=2)
YM.explicit_start = True
YM.explicit_end = True
YM.allow_unicode = True
YM.preserve_quotes = True
# global formatting (any changes *should* propagate to later directives)
EOL = '\n' # line separator
RENDER_JS = False
##
# substitution
##
def sub_stream(strm, meta, method):
"""
substitute strings in strm
return (the same? or a new) stream with substituted values
"""
strm = strm or StringIO()
def extract_to_package_format(self) -> int:
"""Extracts the self.input yml file into several files according to the Demisto standard of the package format.
Returns:
int. status code for the operation.
"""
try:
output_path = self.get_output_path()
except ValueError as ex:
print_error(str(ex))
return 1
self.print_logs("Starting migration of: {} to dir: {}".format(self.input, output_path), log_color=LOG_COLORS.NATIVE)
os.makedirs(output_path, exist_ok=True)
base_name = os.path.basename(output_path) if not self.base_name else self.base_name
code_file = "{}/{}".format(output_path, base_name)
self.extract_code(code_file)
script = self.yml_data['script']
lang_type: str = script['type'] if self.file_type == 'integration' else self.yml_data['type']
code_file = f"{code_file}{TYPE_TO_EXTENSION[lang_type]}"
self.extract_image("{}/{}_image.png".format(output_path, base_name))
self.extract_long_description("{}/{}_description.md".format(output_path, base_name))
yaml_out = "{}/{}.yml".format(output_path, base_name)
self.print_logs("Creating yml file: {} ...".format(yaml_out), log_color=LOG_COLORS.NATIVE)
ryaml = YAML()
ryaml.preserve_quotes = True
with open(self.input, 'r') as yf:
yaml_obj = ryaml.load(yf)
script_obj = yaml_obj
if self.file_type == 'integration':
script_obj = yaml_obj['script']
if 'image' in yaml_obj:
del yaml_obj['image']
if 'detaileddescription' in yaml_obj:
del yaml_obj['detaileddescription']
script_obj['script'] = SingleQuotedScalarString('')
code_type = script_obj['type']
if code_type == TYPE_PWSH and not yaml_obj.get('fromversion'):
self.print_logs("Setting fromversion for PowerShell to: 5.5.0", log_color=LOG_COLORS.NATIVE)
yaml_obj['fromversion'] = "5.5.0"
with open(yaml_out, 'w') as yf:
ryaml.dump(yaml_obj, yf)
# check if there is a README and if found, set found_readme to True
found_readme = False
if self.readme:
yml_readme = os.path.splitext(self.input)[0] + '_README.md'
readme = output_path + '/README.md'
if os.path.exists(yml_readme):
found_readme = True
self.print_logs(f"Copying {readme} to {readme}", log_color=LOG_COLORS.NATIVE)
shutil.copy(yml_readme, readme)
else:
# open an empty file
with open(readme, 'w'):
pass
# Python code formatting and dev env setup
if code_type == TYPE_PYTHON:
if self.basic_fmt:
self.print_logs("Running autopep8 on file: {} ...".format(code_file), log_color=LOG_COLORS.NATIVE)
try:
subprocess.call(["autopep8", "-i", "--max-line-length", "130", code_file])
except FileNotFoundError:
self.print_logs("autopep8 skipped! It doesn't seem you have autopep8 installed.\n"
"Make sure to install it with: pip install autopep8.\n"
"Then run: autopep8 -i {}".format(code_file), LOG_COLORS.YELLOW)
if self.pipenv:
if self.basic_fmt:
self.print_logs("Running isort on file: {} ...".format(code_file), LOG_COLORS.NATIVE)
try:
subprocess.call(["isort", code_file])
except FileNotFoundError:
self.print_logs("isort skipped! It doesn't seem you have isort installed.\n"
"Make sure to install it with: pip install isort.\n"
"Then run: isort {}".format(code_file), LOG_COLORS.YELLOW)
self.print_logs("Detecting python version and setting up pipenv files ...", log_color=LOG_COLORS.NATIVE)
docker = get_all_docker_images(script_obj)[0]
py_ver = get_python_version(docker, self.config.log_verbose)
pip_env_dir = get_pipenv_dir(py_ver, self.config.envs_dirs_base)
self.print_logs("Copying pipenv files from: {}".format(pip_env_dir), log_color=LOG_COLORS.NATIVE)
shutil.copy("{}/Pipfile".format(pip_env_dir), output_path)
shutil.copy("{}/Pipfile.lock".format(pip_env_dir), output_path)
env = os.environ.copy()
env["PIPENV_IGNORE_VIRTUALENVS"] = "1"
try:
subprocess.call(["pipenv", "install", "--dev"], cwd=output_path, env=env)
self.print_logs("Installing all py requirements from docker: [{}] into pipenv".format(docker),
LOG_COLORS.NATIVE)
requirements = get_pip_requirements(docker)
fp = tempfile.NamedTemporaryFile(delete=False)
fp.write(requirements.encode('utf-8'))
fp.close()
try:
subprocess.check_call(["pipenv", "install", "-r", fp.name], cwd=output_path, env=env)
except Exception:
self.print_logs("Failed installing requirements in pipenv.\n "
"Please try installing manually after extract ends\n", LOG_COLORS.RED)
os.unlink(fp.name)
self.print_logs("Installing flake8 for linting", log_color=LOG_COLORS.NATIVE)
subprocess.call(["pipenv", "install", "--dev", "flake8"], cwd=output_path, env=env)
except FileNotFoundError as err:
self.print_logs("pipenv install skipped! It doesn't seem you have pipenv installed.\n"
"Make sure to install it with: pip3 install pipenv.\n"
f"Then run in the package dir: pipenv install --dev\n.Err: {err}", LOG_COLORS.YELLOW)
arg_path = os.path.relpath(output_path)
self.print_logs("\nCompleted: setting up package: {}\n".format(arg_path), LOG_COLORS.GREEN)
next_steps: str = "Next steps: \n" \
"* Install additional py packages for unit testing (if needed): cd {};" \
" pipenv install <package>\n".format(arg_path) if code_type == TYPE_PYTHON else ''
next_steps += "* Create unit tests\n" \
"* Check linting and unit tests by running: demisto-sdk lint -i {}\n".format(arg_path)
next_steps += "* When ready, remove from git the old yml and/or README and add the new package:\n" \
" git rm {}\n".format(self.input)
if found_readme:
next_steps += " git rm {}\n".format(os.path.splitext(self.input)[0] + '_README.md')
next_steps += " git add {}\n".format(arg_path)
self.print_logs(next_steps, log_color=LOG_COLORS.NATIVE)
else:
self.print_logs("Skipping pipenv and requirements installation - Note: no Pipfile will be created",
log_color=LOG_COLORS.YELLOW)
self.print_logs(f"Finished splitting the yml file - you can find the split results here: {output_path}",
log_color=LOG_COLORS.GREEN)
return 0
def pretty_format_yaml(argv=None):
# type: (typing.Optional[typing.List[typing.Text]]) -> int
parser = argparse.ArgumentParser()
parser.add_argument(
"--autofix",
action="store_true",
dest="autofix",
help="Automatically fixes encountered not-pretty-formatted files",
)
parser.add_argument(
"--indent",
type=int,
help=
("The number of indent spaces or a string to be used as delimiter"
' for indentation level e.g. 4 or "\t". Overrides offset, sequence, and mapping values if set.'
),
)
parser.add_argument(
"--mapping",
type=int,
default="2",
help="Indentation of mapping values (Default: 2)",
)
parser.add_argument(
"--offset",
type=int,
default="2",
help="Map items offset (Default: 2)",
)
parser.add_argument(
"--sequence",
type=int,
default="4",
help=
"Sequence indentation. Best to have sequence >= offset + 2 (Default: 4)",
)
parser.add_argument(
"--preserve-quotes",
action="store_true",
dest="preserve_quotes",
help="Keep existing string quoting",
)
parser.add_argument("filenames", nargs="*", help="Filenames to fix")
args = parser.parse_args(argv)
status = 0
yaml = YAML()
if args.indent:
yaml.indent = args.indent
else:
yaml.indent(mapping=args.mapping,
sequence=args.sequence,
offset=args.offset)
yaml.preserve_quotes = args.preserve_quotes
# Prevent ruamel.yaml to wrap yaml lines
yaml.width = maxsize
separator = "---\n"
for yaml_file in set(args.filenames):
with open(yaml_file) as input_file:
string_content = "".join(input_file.readlines())
# Split multi-document file into individual documents
#
# Not using yaml.load_all() because it reformats primitive (non-YAML) content. It removes
# newline characters.
separator_pattern = r"^---\s*\n"
original_docs = re.split(separator_pattern,
string_content,
flags=re.MULTILINE)
# A valid multi-document YAML file might starts with the separator.
# In this case the first document of original docs will be empty and should not be consdered
if string_content.startswith("---"):
original_docs = original_docs[1:]
pretty_docs = []
try:
for doc in original_docs:
content = _process_single_document(doc, yaml)
if content is not None:
pretty_docs.append(content)
# Start multi-doc file with separator
pretty_content = "" if len(pretty_docs) == 1 else separator
pretty_content += separator.join(pretty_docs)
if string_content != pretty_content:
print("File {} is not pretty-formatted".format(yaml_file))
if args.autofix:
print("Fixing file {}".format(yaml_file))
with io.open(yaml_file, "w",
encoding="UTF-8") as output_file:
output_file.write(text_type(pretty_content))
status = 1
except YAMLError: # pragma: no cover
print(
"Input File {} is not a valid YAML file, consider using check-yaml"
.format(yaml_file, ), )
return 1
return status
def exportPipelineToZip(pipelineId):
try:
usingS3 = (settings.DEFAULT_FILE_STORAGE ==
"gremlin.utils.storages.MediaRootS3Boto3Storage")
pipeline = Pipeline.objects.select_related('root_node').get(
pk=pipelineId)
print("Fetched pipeline obj...")
nodes = []
edges = []
scripts = {}
script_dict = {}
zip_bytes = io.BytesIO()
myYaml = io.StringIO()
yaml = YAML()
yaml.preserve_quotes = False
zip_file = zipfile.ZipFile(zip_bytes,
mode='w',
compression=zipfile.ZIP_DEFLATED)
print("Zip file created")
for node in PipelineNode.objects.filter(parent_pipeline=pipeline):
print(f"Look over {node}")
if node.script:
print(f"Node ID {node.script.id} has script: {node.script}")
print(f"Scripts was: {script_dict}")
script_dict[node.script.id] = exportScriptYAMLObj(
node.script.id)
print(f"Script YAML created...")
if node.script.data_file and node.script.data_file.data_file:
print("Script had a data file")
# THe precise field with the valid filename / path depends on the storage adapter, so handle accordingly
if usingS3:
filename = node.script.data_file.data_file.name
else:
filename = node.script.data_file.data_file.path
data_file_bytes = default_storage.open(filename,
mode='rb').read()
zip_file.writestr(
f"/data/{node.script.data_file.uuid}.zip",
data_file_bytes)
scripts = list(script_dict.values())
print(f"Scripts are: {scripts}")
for edge in Edge.objects.filter(parent_pipeline=pipeline):
edges.append(exportPipelineEdgeToYAMLObj(edge.id))
print("Edges converted to YAML")
for node in PipelineNode.objects.filter(parent_pipeline=pipeline):
nodes.append(exportPipelineNodeToYAMLObj(node.id))
print("Nodes converted to YAML")
pipeline_meta = {
'name': pipeline.name,
'description': LiteralScalarString(pipeline.description),
'input_json_schema': pipeline.input_json_schema,
'root_node': pipeline.root_node.id,
'scale': pipeline.scale,
'x_offset': pipeline.x_offset,
'y_offset': pipeline.y_offset,
}
data = {
'pipeline': pipeline_meta,
'scripts': scripts,
'edges': edges,
'nodes': nodes,
}
print("Dump YAML")
yaml.dump(data, myYaml)
zip_file.writestr("pipeline_export.yaml", myYaml.getvalue())
zip_file.close()
zip_bytes.seek(io.SEEK_SET)
print("Done with zip_bytes... returning")
return zip_bytes
except Exception as e:
print(f"Error exporting Pipeline to archive: {e}")
return None
def __yaml_parser(self, kind: str) -> YAML:
'''Get yaml parser.'''
yaml = YAML(typ=kind)
yaml.explicit_start = True # type: ignore
yaml.preserve_quotes = True # type: ignore
return yaml
def main(argv):
args = parse_args()
## check the yaml of these files because ruamel pythin lib has issues with loading em
yaml_files_check_list = [
'ml-operator/values.yaml', 'emailnotifier/values.yaml'
]
ports_array = {
"simapi":
"3000",
"reportapi":
"3002",
"testapi":
"3003",
"https":
"80",
"http":
"80",
"http-admin":
"4001",
"http-api":
"4002",
"mysql":
"3306",
"mongodb":
"27017",
"inboundapi":
"{{ $config.config.schemeAdapter.env.INBOUND_LISTEN_PORT }}",
"outboundapi":
"{{ $config.config.schemeAdapter.env.OUTBOUND_LISTEN_PORT }}"
}
p = Path() / args.directory
print(f"Processing helm charts in directory: [{args.directory}]")
yaml = YAML()
yaml.allow_duplicate_keys = True
yaml.preserve_quotes = True
yaml.width = 4096
# walk the directory structure and process all the values.yaml files
# replace solsson kafka with kymeric
# replace kafa start up check with netcat test (TODO check to see if this is ok)
# replace mysql with arm version of mysql and adjust tag on the following line (TODO: check that latest docker mysql/mysql-server latest tag is ok )
# TODO: maybe don't do this line by line but rather read in the entire file => can match across lines and avoid the next_line_logic
# for now disable metrics and metrics exporting
# replace the mojaloop images with the locally built ones
if (args.all or args.values):
print(
"\n\n============================================================="
)
print("Processing values.yaml files.. ")
print("=============================================================")
for vf in p.rglob('*/values.yaml'):
backupfile = Path(vf.parent) / f"{vf.name}_bak"
print(f"{vf} : {backupfile}")
copyfile(vf, backupfile)
with FileInput(files=[vf], inplace=True) as f:
next_line_is_mojaloop_tag = False
for line in f:
line = line.rstrip()
# now update the mojaloop images
if (next_line_is_mojaloop_tag):
line = re.sub("tag:.*$", "tag: latest", line)
next_line_is_mojaloop_tag = False
# TODO : check that there is no mojaloop image with > 3 parts to its name i.e. > 3 hypens
if re.match(r"(\s+)repository:\s*mojaloop", line):
line = re.sub(
r"(\s+)repository:\s*mojaloop/(\w+)-(\w+)-(\w+)-(\w+)",
r"\1repository: \2_\3_\4_\5_local", line)
line = re.sub(
r"(\s+)repository:\s*mojaloop/(\w+)-(\w+)-(\w+)",
r"\1repository: \2_\3_\4_local", line)
line = re.sub(
r"(\s+)repository:\s*mojaloop/(\w+)-(\w+)",
r"\1repository: \2_\3_local", line)
line = re.sub(r"(\s+)repository:\s*mojaloop/(\w+)",
r"\1repository: \2_local", line)
next_line_is_mojaloop_tag = True
print(line)
## TODO Need to modify the kafka requirements.yaml to update the zookeeper image
## if I am fully automating this
# walk the directory structure and process all the requirements.yaml files
# kafka => local kafka chart
# mysql/percona => local mysql chart with later arm64 based image
# zookeeper => local zookeeper (this is in the requirements.yaml of the kafka local chart)
if (args.all or args.requirements):
print(
"\n\n============================================================="
)
print("Processing requirements.yaml files ")
print("=============================================================")
for rf in p.rglob('*/requirements.yaml'):
backupfile = Path(rf.parent) / f"{rf.name}_bak"
print(f"{rf} : {backupfile}")
copyfile(rf, backupfile)
with open(rf) as f:
reqs_data = yaml.load(f)
#print(reqs_data)
try:
dlist = reqs_data['dependencies']
for i in range(len(dlist)):
if (dlist[i]['name'] == "percona-xtradb-cluster"):
print(f"old was: {dlist[i]}")
dlist[i]['name'] = "mysql"
dlist[i]['version'] = "1.0.0"
dlist[i]['repository'] = "file://../mysql"
dlist[i]['alias'] = "mysql"
dlist[i]['condition'] = "enabled"
print(f"new is: {dlist[i]}")
if (dlist[i]['name'] == "kafka"):
print(f"old was: {dlist[i]}")
dlist[i]['repository'] = "file://../kafka"
dlist[i]['version'] = "1.0.0"
print(f"new is: {dlist[i]}")
if (dlist[i]['name'] == "zookeeper"):
print(f"old was: {dlist[i]}")
dlist[i]['version'] = "1.0.0"
dlist[i]['repository'] = "file://../zookeeper"
print(f"new is: {dlist[i]}")
if (dlist[i]['name'] == "mongodb"):
print(f"old was: {dlist[i]}")
dlist[i]['version'] = "1.0.0"
dlist[i]['repository'] = "file://../mongodb"
print(f"new is: {dlist[i]}")
except Exception:
continue
#print(yaml.dump(reqs_data))
with open(rf, "w") as f:
yaml.dump(reqs_data, f)
if (args.testonly):
print(
"\n\n==============================================================="
)
print("running toms code tests")
print(
"===============================================================")
for vf in p.rglob('*/values.yaml'):
backupfile = Path(vf.parent) / f"{vf.name}_bak"
# print(f"{vf} : {backupfile}")
copyfile(vf, backupfile)
with open(vf) as f:
skip = False
for fn in yaml_files_check_list:
if vf == Path(fn):
print(
f"This yaml file needs checking skipping load/processing for now => {Path(fn)} "
)
skip = True
if not skip:
print(f" Loading yaml for ==> {vf.parent}/{vf.name}",
end="")
data = yaml.load(f)
print(" :[ok]")
# update kafka settings
count = 0
for x, value in lookup("kafka", data):
#print_debug(x,value)
list(
update_key(
'command',
'until nc -vz -w 1 $kafka_host $kafka_port; do echo waiting for Kafka; sleep 2; done;',
value))
list(update_key('repository', 'kymeric/cp-kafka', value))
list(update_key('image', 'kymeric/cp-kafka', value))
list(update_key('imageTag', 'latest', value))
# turn off prometheus jmx and kafka exporter
for x, value in lookup("prometheus", data):
#print_debug(x,value , 2)
if isinstance(value, dict):
if value.get("jmx"):
value['jmx']['enabled'] = False
if value.get("kafka"):
value['kafka']['enabled'] = False
# update mysql settings
for x, value in lookup("mysql", data):
list(update_key('repository', 'mysql/mysql-server', value))
list(update_key('tag', '8.0.28-1.2.7-server', value))
if value.get("image"):
del value['image']
value['image'] = "mysql/mysql-server"
value['imageTag'] = "8.0.28-1.2.7-server"
value['pullPolicy'] = "ifNotPresent"
# turn the side car off for the moment
for x, value in lookup("sidecar", data):
list(update_key('enabled', False, value))
# turn metrics off
# The simulator has metrics clause with no enabled setting => hence need to test
for x, value in lookup("metrics", data):
try:
if value.get("enabled"):
value['enabled'] = False
except Exception:
continue
with open(vf, "w") as f:
yaml.dump(data, f)
if (args.ingress):
print(
"\n\n======================================================================================"
)
print(" Modify charts to implement networking/v1 ")
print(
" and to use bitnami mysql rather than percona (percona / busybox is broken on containerd) "
)
print(
"==========================================================================================="
)
# modify the template files
for vf in p.rglob('*.tpl'):
backupfile = Path(vf.parent) / f"{vf.name}_bak"
#print(f"{vf} : {backupfile}")
#copyfile(vf, backupfile)
with FileInput(files=[vf], inplace=True) as f:
#with fileinput.input(files=([vf]), inplace=True) as f:
for line in f:
line = line.rstrip()
#replace networking v1beta1
line = re.sub(r"networking.k8s.io/v1beta1",
r"networking.k8s.io/v1", line)
line = re.sub(r"extensions/v1beta1",
r"networking.k8s.io/v1", line)
print(line)
# modify the ingress.yaml files
for vf in p.rglob('*/ingress.yaml'):
backupfile = Path(vf.parent) / f"{vf.name}_bak"
#print(f"{vf} : {backupfile}")
#copyfile(vf, backupfile)
with FileInput(files=[vf], inplace=True) as f:
for line in f:
line = line.rstrip()
if re.search("path:", line):
line_dup = line
line_dup = re.sub(
r"- path:.*$",
r" pathType: ImplementationSpecific", line_dup)
print(line)
print(line_dup)
elif re.search("serviceName:", line):
line_dup = line
line_dup = re.sub(r"serviceName:.*$", r"service:",
line_dup)
print(line_dup)
line = re.sub(r"serviceName:", r" name:", line)
print(line)
elif re.search("servicePort:", line):
line_dup = line
line_dup = re.sub(r"servicePort:.*$", r" port:",
line_dup)
line = re.sub(r"servicePort: ", r" number: ", line)
# need to replace port names with numbers
for pname, pnum in ports_array.items():
line = re.sub(f"number: {pname}$",
f"number: {pnum}", line)
print(line_dup)
print(line)
#servicePort {{ .Values.containers.api.service.ports.api.externalPort }}
elif re.search("spec:", line):
print(line)
print(
" ingressClassName: public"
) # well at least it is "public" for microk8s v1.22 => TODO fully figure the chamges and settings out here and simplify!
else:
print(line)
for vf in p.rglob('*/values.yaml'):
with open(vf) as f:
#print(f"{vf.parent}/{vf.name}")
skip = False
for fn in yaml_files_check_list:
if vf == Path(fn):
print(
f"This yaml file needs checking skipping load/processing for now => {Path(fn)} "
)
skip = True
if not skip:
#print(f" Loading yaml for ==> {vf.parent}/{vf.name}", end="")
data = yaml.load(f)
#print(" :[ok]")
for x, value in lookup("mysql", data):
list(update_key('enabled', 'true', value))
# => use these for now
# TODO: update to later DB and get rid of default passwords
for x, value in lookup("mysql", data):
list(update_key('repository', 'mysql/mysql-server', value))
list(update_key('tag', '5.6', value))
if value.get("image"):
del value['image']
value['image'] = "mysql"
value['imageTag'] = '8.0'
value['pullPolicy'] = "ifNotPresent"
### need to set nameOverride for mysql for ml-testing-toolkit as it appears to be missing
if vf == Path('mojaloop/values.yaml'):
print("Updating the ml-testing-toolkit / mysql config ")
for x, value in lookup("ml-testing-toolkit", data):
value['mysql'] = {"nameOverride": "ttk-mysql"}
with open(vf, "w") as f:
yaml.dump(data, f)
# versions of k8s -> 1.20 use containerd not docker and the percona chart
# or at least the busybox dependency of the percona chart has an issue
# so just replace the percona chart with the mysql charts
# for now using the old one because it deploys => TODO fix this and update
for rf in p.rglob('*/requirements.yaml'):
with open(rf) as f:
reqs_data = yaml.load(f)
#print(reqs_data)
try:
dlist = reqs_data['dependencies']
for i in range(len(dlist)):
if (dlist[i]['name'] == "percona-xtradb-cluster"):
print(f"old was: {dlist[i]}")
dlist[i]['name'] = "mysql"
#dlist[i]['version'] = "8.8.8"
#dlist[i]['repository'] = "https://charts.bitnami.com/bitnami"
dlist[i]['version'] = 8.0
dlist[i][
'repository'] = "https://charts.bitnami.com/bitnami"
dlist[i]['alias'] = "mysql"
dlist[i]['condition'] = "enabled"
print(f"new is: {dlist[i]}")
# if (dlist[i]['name'] == "mongodb"):
# print(f"old was: {dlist[i]}")
# dlist[i]['version'] = "11.1.7"
# dlist[i]['repository'] = "file://../mongodb"
# print(f"new is: {dlist[i]}")
except Exception:
continue
with open(rf, "w") as f:
yaml.dump(reqs_data, f)
def update_files():
"""check the filepath extension and update the respective file"""
global datasource_paths
global user_value
datasource_paths = DATASOURCE_PATHS[product_id]
print("Datasource paths: " + str(datasource_paths))
for data_source in datasource_paths:
print("Datasource is " + data_source)
print("storage dist abs path is : " + str(storage_dist_abs_path))
file_path = Path(workspace + "/" + product_id + "/" + data_source)
print("file_path is : " + str(file_path))
if str(file_path).endswith('.yaml'):
print("filepath file is yaml")
yaml = YAML()
yaml.preserve_quotes = True
doc = Path(file_path)
obj = yaml.load(doc)
print("Current username is : " +
obj['wso2.datasources']['dataSources'][0]['definition']
['configuration']['username'])
print("Current password is : " +
obj['wso2.datasources']['dataSources'][0]['definition']
['configuration']['password'])
print("Current jdbcurl is : " + obj['wso2.datasources']
['dataSources'][0]['definition']['configuration']['jdbcUrl'])
print("Current driver name is : " +
obj['wso2.datasources']['dataSources'][0]['definition']
['configuration']['driverClassName'])
print("Current connection query value is : " +
obj['wso2.datasources']['dataSources'][0]['definition']
['configuration']['connectionTestQuery'])
obj['wso2.datasources']['dataSources'][0]['definition'][
'configuration']['password'] = password_value
obj['wso2.datasources']['dataSources'][0]['definition'][
'configuration']['username'] = user_value
obj['wso2.datasources']['dataSources'][0]['definition'][
'configuration']['jdbcUrl'] = url_value
obj['wso2.datasources']['dataSources'][0]['definition'][
'configuration']['driverClassName'] = drive_class_name_value
if ORACLE_DB_ENGINE == database_config['db_engine'].upper():
obj['wso2.datasources']['dataSources'][0]['definition'][
'configuration'][
'connectionTestQuery'] = validation_query_value
print("Changed username is : " +
obj['wso2.datasources']['dataSources'][0]['definition']
['configuration']['username'])
print("Changed password is : " +
obj['wso2.datasources']['dataSources'][0]['definition']
['configuration']['password'])
print("Changed jdbcurl is : " + obj['wso2.datasources']
['dataSources'][0]['definition']['configuration']['jdbcUrl'])
print("Changed driver name is : " +
obj['wso2.datasources']['dataSources'][0]['definition']
['configuration']['driverClassName'])
print("Changed connection query value is : " +
obj['wso2.datasources']['dataSources'][0]['definition']
['configuration']['connectionTestQuery'])
yaml.dump(obj, doc)
elif str(file_path).endswith('.xml'):
print("filepath is xml")
modify_datasources()
else:
print("extension not matched")
def run(self):
"""
This function will try to load integration/script yml file.
Creates test playbook, and converts each command to automation task in test playbook and generates verify
outputs task from command outputs.
All the tasks eventually will be linked to each other:
playbook_start_task => delete_context(all) => task1 => verify_outputs_task1 => task2 => verify_outputs_task2
=> task_end
At the end the functions dumps the new test playbook to the outdir if set, otherwise file will be created in
local directory
"""
if self.outdir:
if not os.path.isdir(self.outdir):
print_error(f'Directory not exist: {self.outdir}')
return
ryaml = YAML()
ryaml.preserve_quotes = True
try:
with open(self.integration_yml_path, 'r') as yf:
yaml_obj = ryaml.load(yf)
yaml_obj.get('name')
except FileNotFoundError as ex:
if self.verbose:
raise
print_error(str(ex))
return
except AttributeError:
print_error(f'Error - failed to parse: {self.integration_yml_path}.\nProbably invalid yml file')
return
test_playbook = Playbook(
name=self.name,
fromversion='4.5.0'
)
if self.file_type == ContentItemType.INTEGRATION:
for command in yaml_obj.get('script').get('commands'):
create_automation_task_and_verify_outputs_task(
test_playbook=test_playbook,
command=command,
item_type=ContentItemType.INTEGRATION,
no_outputs=self.no_outputs
)
elif self.file_type == ContentItemType.SCRIPT:
create_automation_task_and_verify_outputs_task(
test_playbook=test_playbook,
command=yaml_obj,
item_type=ContentItemType.INTEGRATION,
no_outputs=self.no_outputs
)
test_playbook.add_task(create_end_task(test_playbook.task_counter))
with open(self.test_playbook_yml_path, 'w') as yf:
ryaml.dump(test_playbook.to_dict(), yf)
print_color(f'Test playbook yml was saved at:\n{self.test_playbook_yml_path}', LOG_COLORS.GREEN)
"""
heluxup is able to parse the flux control repository for HelmReleases and checks if updates of the charts are
available. If updates are available heluxup updates the yaml files in the flux control respositroy accordingly.
"""
import os
import click
import semver
import urllib3
from ruamel.yaml import YAML
HTTP = urllib3.PoolManager()
YAML_PARSER = YAML()
YAML_PARSER.preserve_quotes = True
YAML_PARSER.explicit_start = True
YAML_PARSER.width = 8000
REPO_CACHE = {}
class HelmRelease:
"""
HelmRelease represents a HelmRelease object that is used by flux.
"""
def __init__(self, release_name, chart_name, repository, git_version):
self.release_name = release_name
self.chart_name = chart_name
self.repository = repository.rstrip('/')
self.git_version = git_version
if self.repository not in REPO_CACHE:
# All the beat types that will be processed
beats = ['winlogbeat',
'filebeat',
'metricbeat',
'packetbeat']
# Start of the program
ipAddr = 'asdf' # Remove the fake IP, only here for testing
port = ''
while not valid_ip(ipAddr):
ipAddr = input("Enter logstash host IP: ")
port = input("Enter port [5044]: ")
if port == '':
# If nothing entered, use default port for beats input to logstash
port = '5044'
# Setting up the ruamel.yaml to properly read libbeat configs
yaml = YAML()
yaml.default_flow_style = False
yaml.indent(mapping=2, sequence=4, offset=2)
yaml.preserve_quotes = True
# Get directories to work with
rootDir = '.'
for dirName, subdirList, fileList in os.walk(rootDir):
for file in fileList:
for beat in beats:
if file == beat + '.yml': #startswith(beat):
print(f"Updating: {dirName}/{beat}.yml")
update_config(beat, dirName+'/')
def migrate(yml_path,
output_path,
demisto_mock,
commonserver=None,
yml_type=None):
try:
from ruamel.yaml import YAML
from ruamel.yaml.scalarstring import SingleQuotedScalarString
except Exception as ex:
print(
"Failed importing ruamel.yaml. Migrate requires ruamel.yaml to work cleanly.\n"
"Install via: pip3 install ruamel.yaml.\nErr: {}".format(ex))
return 1
print("Starting migration of: {} to dir: {}".format(yml_path, output_path))
arg_path = output_path
output_path = os.path.abspath(output_path)
os.makedirs(output_path, exist_ok=True)
base_name = os.path.basename(output_path)
yml_type = get_yml_type(yml_path, yml_type)
code_file = "{}/{}.py".format(output_path, base_name)
extract_code(yml_path, code_file, demisto_mock, commonserver, yml_type)
extract_image(yml_path, "{}/{}_image.png".format(output_path, base_name),
yml_type)
yaml_out = "{}/{}.yml".format(output_path, base_name)
print("Creating yml file: {} ...".format(yaml_out))
ryaml = YAML()
ryaml.preserve_quotes = True
with open(yml_path, 'r') as yf:
yaml_obj = ryaml.load(yf)
script_obj = yaml_obj
if yml_type == INTEGRATION:
script_obj = yaml_obj['script']
del yaml_obj['image']
if script_obj['type'] != 'python':
print('Script is not of type "python". Found type: {}. Nothing to do.'.
format(script_obj['type']))
return 1
script_obj['script'] = SingleQuotedScalarString('')
with open(yaml_out, 'w') as yf:
ryaml.dump(yaml_obj, yf)
print("Running autopep8 on file: {} ...".format(code_file))
try:
subprocess.call(
["autopep8", "-i", "--max-line-length", "130", code_file])
except FileNotFoundError:
print_color(
"autopep8 skipped! It doesn't seem you have autopep8 installed.\n"
"Make sure to install it with: pip install autopep8.\n"
"Then run: autopep8 -i {}".format(code_file), LOG_COLORS.YELLOW)
print("Detecting python version and setting up pipenv files ...")
docker = get_docker_images(script_obj)[0]
py_ver = get_python_version(docker)
pip_env_dir = get_pipenv_dir(py_ver)
print("Copying pipenv files from: {}".format(pip_env_dir))
shutil.copy("{}/Pipfile".format(pip_env_dir), output_path)
shutil.copy("{}/Pipfile.lock".format(pip_env_dir), output_path)
try:
subprocess.call(["pipenv", "install", "--dev"], cwd=output_path)
print("Installing all py requirements from docker: [{}] into pipenv".
format(docker))
requirements = subprocess.check_output(
[
"docker", "run", "--rm", docker, "pip", "freeze",
"--disable-pip-version-check"
],
universal_newlines=True,
stderr=subprocess.DEVNULL).strip()
fp = tempfile.NamedTemporaryFile(delete=False)
fp.write(requirements.encode('utf-8'))
fp.close()
subprocess.check_call(["pipenv", "install", "-r", fp.name],
cwd=output_path)
os.unlink(fp.name)
except FileNotFoundError:
print_color(
"pipenv install skipped! It doesn't seem you have pipenv installed.\n"
"Make sure to install it with: pip3 install pipenv.\n"
"Then run in the package dir: pipenv install --dev",
LOG_COLORS.YELLOW)
print_color("\nCompleted: setting up package: {}\n".format(arg_path),
LOG_COLORS.GREEN)
print(
"Next steps: \n"
"* Install additional py packages for unit testsing (if needed): pipenv install <package>\n"
"* Create unit tests\n"
"* Check linting and unit tests by running: ./Tests/scripts/pkg_dev_test_tasks.py -d {}\n"
.format(arg_path))
return 0
def yaml():
y = YAML()
y.explicit_start = True
y.explicit_end = False
y.preserve_quotes = True
return y
INDICATOR_FIELDS_DIR, INTEGRATIONS_DIR, JSON_ALL_INDICATOR_TYPES_REGEXES,
LAYOUTS_DIR, PACKAGE_SUPPORTING_DIRECTORIES, PACKAGE_YML_FILE_REGEX,
PACKS_DIR, PACKS_DIR_REGEX, PACKS_README_FILE_NAME, PLAYBOOKS_DIR,
RELEASE_NOTES_DIR, RELEASE_NOTES_REGEX, REPORTS_DIR, SCRIPTS_DIR,
SDK_API_GITHUB_RELEASES, TEST_PLAYBOOKS_DIR, TYPE_PWSH, UNRELEASE_HEADER,
WIDGETS_DIR, FileType)
from ruamel.yaml import YAML
# disable insecure warnings
urllib3.disable_warnings()
# inialize color palette
colorama.init()
ryaml = YAML()
ryaml.preserve_quotes = True # type: ignore
ryaml.allow_duplicate_keys = True
class LOG_COLORS:
NATIVE = colorama.Style.RESET_ALL
RED = colorama.Fore.RED
GREEN = colorama.Fore.GREEN
YELLOW = colorama.Fore.YELLOW
WHITE = colorama.Fore.WHITE
LOG_VERBOSE = False
LAYOUT_CONTAINER_FIELDS = {
'details', 'detailsV2', 'edit', 'close', 'mobile', 'quickView',
def merge_yamls(rule_file, tag_file, output_file):
yaml = YAML(typ='rt')
yaml.width = 4096 # prevent line wrap
yaml.preserve_quotes = True
indexed_rules_tags = {}
with open(tag_file, "r") as file:
rules_tags = yaml.load(file)
for rule in rules_tags:
indexed_rules_tags[rule['rule']] = rule['tags']
stats = {
'lists': 0,
'macros': 0,
'rules': 0,
'rules_unmodified': 0,
'rules_modified': 0,
'rules_notfound': 0,
'rules_notags': 0,
'other': 0
}
other_items = []
rules_not_found = []
rules_no_tags_key = []
required_engine_version = 0
with open(rule_file, "r") as file:
falco_doc = yaml.load(file)
for item in falco_doc:
if item.get("list") != None:
stats['lists'] += 1
continue
if item.get("macro") != None:
stats['macros'] += 1
continue
if item.get("required_engine_version") != None:
required_engine_version = item.get("required_engine_version")
continue
if item.get("rule") == None:
# Something that is not a rule, a macro or a list
stats['other'] += 1
other_items.append(item)
continue
# A rule
stats['rules'] += 1
if item.get("tags") == None:
# Rule doesn't have a 'tags' key
stats['rules_notags'] += 1
rules_no_tags_key.append(item.get("rule"))
continue
if item.get("rule") not in indexed_rules_tags.keys():
# Tags file doesn't have a rule with same name
rules_not_found.append(item.get("rule"))
continue
if len(indexed_rules_tags[item.get("rule")]) == 0:
# Tag file doesn't have new tags for this rule
stats['rules_unmodified'] += 1
continue
# Append non existing tags
for newtag in indexed_rules_tags[item.get("rule")]:
if (not newtag in item['tags']):
item['tags'].append(newtag)
stats['rules_modified'] += 1
# Write output file
with open(output_file, "w") as stream:
stream.write('# Merged tags from ' + os.path.basename(tag_file) +
'\n\n')
yaml.dump(falco_doc, stream)
stream.close()
# Output results
if (len(rules_not_found) > 0):
print("\nRules not found:")
for rule in rules_not_found:
print(rule)
if (len(rules_no_tags_key) > 0):
print("\nRules without 'tags' keyword:")
for rule in rules_no_tags_key:
print(rule)
if (len(other_items) > 0):
print("\nOther elements:")
for item in other_items:
print(item)
print("\nLists: ", stats['lists'])
print("Macros: ", stats['macros'])
print("Rules: ", stats['rules'])
print(" Modified rules: ", stats['rules_modified'])
print(" Unmodified rules: ", stats['rules_unmodified'])
print(" Rules not found: ", len(rules_not_found))
print(" Rules no tags key: ", stats['rules_notags'])
print("required_engine_version: ", required_engine_version)
print("Other: ", len(other_items))
if (len(rules_not_found) > 0):
sys.exit(1)
sys.exit(0)
def main(argv):
args = parse_args()
ingress_cn = set_ingressclassname(args.kubernetes)
script_path = Path(__file__).absolute()
mysql_values_file = script_path.parent.parent / "./etc/mysql_values.yaml"
db_pass = gen_password()
if (args.verbose):
print(f"mysql_values_file is {mysql_values_file}")
print(f"mysql password is {db_pass}")
## check the yaml of these files because ruamel python lib has issues with loading em
yaml_files_check_list = [
'ml-operator/values.yaml', 'emailnotifier/values.yaml'
]
ports_array = {
"simapi":
"3000",
"reportapi":
"3002",
"testapi":
"3003",
"https":
"80",
"http":
"80",
"http-admin":
"4001",
"http-api":
"4002",
"mysql":
"3306",
"mongodb":
"27017",
"inboundapi":
"{{ $config.config.schemeAdapter.env.INBOUND_LISTEN_PORT }}",
"outboundapi":
"{{ $config.config.schemeAdapter.env.OUTBOUND_LISTEN_PORT }}"
}
ingress_cn = set_ingressclassname(args.kubernetes)
print(f"ingressclassname in main is {ingress_cn}")
p = Path() / args.directory
print(f"Processing helm charts in directory: [{args.directory}]")
yaml = YAML()
yaml.allow_duplicate_keys = True
yaml.preserve_quotes = True
yaml.width = 4096
# walk the directory structure and process all the values.yaml files
# replace solsson kafka with kymeric
# replace kafa start up check with netcat test (TODO check to see if this is ok)
# replace mysql with arm version of mysql and adjust tag on the following line (TODO: check that latest docker mysql/mysql-server latest tag is ok )
# TODO: maybe don't do this line by line but rather read in the entire file => can match across lines and avoid the next_line_logic
# for now disable metrics and metrics exporting
# replace the mojaloop images with the locally built ones
print(
" ==> mod_local_miniloop : Modify helm template files (.tpl) to implement networking/v1"
)
# modify the template files
for vf in p.rglob('*.tpl'):
backupfile = Path(vf.parent) / f"{vf.name}_bak"
with FileInput(files=[str(vf)], inplace=True) as f:
for line in f:
line = line.rstrip()
#replace networking v1beta1
line = re.sub(r"networking.k8s.io/v1beta1",
r"networking.k8s.io/v1", line)
line = re.sub(r"extensions/v1beta1", r"networking.k8s.io/v1",
line)
print(line)
# modify the ingress.yaml files
print(
" ==> mod_local_miniloop : Modify helm template ingress.yaml files to implement newer ingress"
)
print(
f" ==> mod_local_miniloop : Modify helm template ingress.yaml implement correct ingressClassName [{ingress_cn}]"
)
for vf in p.rglob('*/ingress.yaml'):
backupfile = Path(vf.parent) / f"{vf.name}_bak"
with FileInput(files=[str(vf)], inplace=True) as f:
for line in f:
line = line.rstrip()
if re.search("path:", line):
line_dup = line
line_dup = re.sub(r"- path:.*$",
r" pathType: ImplementationSpecific",
line_dup)
print(line)
print(line_dup)
elif re.search("serviceName:", line):
line_dup = line
line_dup = re.sub(r"serviceName:.*$", r"service:",
line_dup)
print(line_dup)
line = re.sub(r"serviceName:", r" name:", line)
print(line)
elif re.search("servicePort:", line):
line_dup = line
line_dup = re.sub(r"servicePort:.*$", r" port:", line_dup)
line = re.sub(r"servicePort: ", r" number: ", line)
# need to replace port names with numbers
for pname, pnum in ports_array.items():
line = re.sub(f"number: {pname}$", f"number: {pnum}",
line)
print(line_dup)
print(line)
elif re.search("ingressClassName", line):
# skip any ingressClassname already set => we can re-run program without issue
continue
elif re.search("spec:", line):
print(line)
print(f" ingressClassName: {ingress_cn}")
else:
print(line)
# put the database password file into the mysql helm chart values file
print(f" ==> mod_local_miniloop : generating a new database password")
print(
f" ==> mod_local_miniloop : insert new pw into [{mysql_values_file}]")
with FileInput(files=[str(mysql_values_file)], inplace=True) as f:
for line in f:
line = line.rstrip()
line = re.sub(r"password: .*$", r"password: '******'",
line)
line = re.sub(r"mysql_native_password BY .*$",
r"mysql_native_password BY '" + db_pass + "';", line)
print(line)
print(
" ==> mod_local_miniloop : Modify helm values to implement single mysql database"
)
for vf in p.glob('**/*values.yaml'):
with open(vf) as f:
if (args.verbose):
print(f"===> Processing file < {vf.parent}/{vf.name} > ")
skip = False
for fn in yaml_files_check_list:
if vf == Path(fn):
if (args.verbose):
print(
f"This yaml file needs checking skipping load/processing for now => {Path(fn)} "
)
skip = True
if not skip:
data = yaml.load(f)
for x, value in lookup("mysql", data):
if (value.get("name") == "wait-for-mysql"):
value['repository'] = "mysql"
value['tag'] = '8.0'
if value.get("mysqlDatabase"):
value['enabled'] = False
# update the values files to use a mysql instance that has already been deployed
# and that uses a newly generated database password
for x, value in lookup("config", data):
if isinstance(value, dict):
if (value.get('db_type')):
value['db_host'] = 'mldb'
value['db_password'] = db_pass
### need to set nameOverride for mysql for ml-testing-toolkit as it appears to be missing
# if vf == Path('mojaloop/values.yaml') :
# print("Updating the ml-testing-toolkit / mysql config ")
# for x, value in lookup("ml-testing-toolkit", data):
# value['mysql'] = { "nameOverride" : "ttk-mysql" }
with open(vf, "w") as f:
yaml.dump(data, f)
# now that we are inserting passwords with special characters in the password it is necessary to ensure
# that $db_password is single quoted in the values files.
print(
" ==> mod_local_miniloop : Modify helm values, single quote db_password field to enable secure database password"
)
for vf in p.glob('**/*values.yaml'):
with FileInput(files=[str(vf)], inplace=True) as f:
for line in f:
line = line.rstrip()
line = re.sub(r"\'\$db_password\'", r"$db_password",
line) # makes this re-runnable.
line = re.sub(r'\$db_password', r"'$db_password'", line)
print(line)
# versions of k8s -> 1.20 use containerd not docker and the percona chart
# or at least the busybox dependency of the percona chart has an issue
# So here update the chart dependencies to ensure correct mysql is configured
# using the bitnami helm chart BUT as we are disabling the database in the
# values files and relying on separately deployed database this update is not really
# doing anything. see the mini-loop scripts dir for where and how the database deployment
# is now done.
print(
" ==> mod_local_miniloop : Modify helm requirements.yaml replace deprecated percona chart with current mysql"
)
for rf in p.rglob('**/*requirements.yaml'):
with open(rf) as f:
reqs_data = yaml.load(f)
#print(reqs_data)
try:
dlist = reqs_data['dependencies']
for i in range(len(dlist)):
if (dlist[i]['name'] in ["percona-xtradb-cluster", "mysql"]):
dlist[i]['name'] = "mysql"
dlist[i]['version'] = 8.0
dlist[i][
'repository'] = "https://charts.bitnami.com/bitnami"
dlist[i]['alias'] = "mysql"
dlist[i]['condition'] = "mysql.enabled"
except Exception:
continue
with open(rf, "w") as f:
yaml.dump(reqs_data, f)
print(
f"Sucessfully finished processing helm charts in directory: [{args.directory}]"
)
async def update_build_data(self, advisories: Dict[str, int],
jira_issue_key: Optional[str]):
if not advisories and not jira_issue_key:
return False
repo = self.working_dir / "ocp-build-data-push"
if not repo.exists():
await self.clone_build_data(repo)
if not self.assembly or self.assembly == "stream":
# update advisory numbers in group.yml
with open(repo / "group.yml", "r") as f:
group_config = f.read()
for kind, advisory in advisories.items():
new_group_config = re.sub(fr"^(\s+{kind}:)\s*[0-9]+$",
fr"\1 {advisory}",
group_config,
count=1,
flags=re.MULTILINE)
group_config = new_group_config
# freeze automation
group_config = re.sub(r"^freeze_automation:.*",
"freeze_automation: scheduled",
group_config,
count=1,
flags=re.MULTILINE)
# update group.yml
with open(repo / "group.yml", "w") as f:
f.write(group_config)
else:
# update releases.yml (if we are operating on a non-stream assembly)
yaml = YAML(typ="rt")
yaml.preserve_quotes = True
async with aiofiles.open(repo / "releases.yml", "r") as f:
old = await f.read()
releases_config = yaml.load(old)
group_config = releases_config["releases"][
self.assembly].setdefault("assembly",
{}).setdefault("group", {})
group_config["advisories"] = advisories
group_config["release_jira"] = jira_issue_key
out = StringIO()
yaml.dump(releases_config, out)
async with aiofiles.open(repo / "releases.yml", "w") as f:
await f.write(out.getvalue())
cmd = ["git", "-C", str(repo), "--no-pager", "diff"]
await exectools.cmd_assert_async(cmd)
cmd = ["git", "-C", str(repo), "add", "."]
await exectools.cmd_assert_async(cmd)
cmd = ["git", "-C", str(repo), "diff-index", "--quiet", "HEAD"]
rc = await exectools.cmd_assert_async(cmd, check=False)
if rc == 0:
_LOGGER.warn("Skip saving advisories: No changes.")
return False
cmd = [
"git", "-C",
str(repo), "commit", "-m", f"Prepare release {self.release_name}"
]
await exectools.cmd_assert_async(cmd)
if not self.dry_run:
_LOGGER.info("Pushing changes to upstream...")
cmd = ["git", "-C", str(repo), "push", "origin", self.group_name]
await exectools.cmd_assert_async(cmd)
else:
_LOGGER.warn("Would have run %s", cmd)
_LOGGER.warn("Would have pushed changes to upstream")
return True
def main(docname):
with open(docname, "r") as fi:
lines = fi.readlines()
context = {}
rest_lines = []
for line in lines:
# print(line)
if "{%" in line:
set_expr = re.search("{%(.*)%}", line)
set_expr = set_expr.group(1)
set_expr = set_expr.replace("set", "", 1).strip()
exec(set_expr, globals(), context)
else:
rest_lines.append(line)
yaml = YAML(typ="rt")
yaml.preserve_quotes = True
yaml.default_flow_style = False
yaml.indent(sequence=4, offset=2)
yaml.width = 1000
yaml.Representer = MyRepresenter
yaml.Loader = ruamel.yaml.RoundTripLoader
result_yaml = CommentedMap()
result_yaml["context"] = context
def has_selector(s):
return s.strip().endswith("]")
quoted_lines = []
for line in rest_lines:
if has_selector(line):
selector_start = line.rfind("[")
selector_end = line.rfind("]")
selector_content = line[selector_start + 1 : selector_end]
if line.strip().startswith("-"):
line = (
line[: line.find("-") + 1]
+ f" sel({selector_content}): "
+ line[
line.find("-") + 1 : min(line.rfind("#"), line.rfind("["))
].strip()
+ "\n"
)
quoted_lines.append(line)
rest_lines = quoted_lines
def check_if_quoted(s):
s = s.strip()
return s.startswith('"') or s.startswith("'")
quoted_lines = []
for line in rest_lines:
if "{{" in line:
# make sure that jinja stuff is quoted
if line.find(":") != -1:
idx = line.find(":")
elif line.strip().startswith("-"):
idx = line.find("-")
rest = line[idx + 1 :]
if not check_if_quoted(rest):
if "'" in rest:
rest = rest.replace("'", '"')
line = line[: idx + 1] + f" '{rest.strip()}'\n"
quoted_lines.append(line)
rest_lines = quoted_lines
skips, wo_skip_lines = [], []
for line in rest_lines:
if line.strip().startswith("skip"):
parts = line.split(":")
rhs = parts[1].strip()
if rhs.startswith("true"):
selector_start = line.rfind("[")
selector_end = line.rfind("]")
selector_content = line[selector_start + 1 : selector_end]
skips.append(selector_content)
else:
print("ATTENTION skip: false not handled!")
else:
wo_skip_lines.append(line)
rest_lines = wo_skip_lines
result_yaml.update(
ruamel.yaml.load("".join(rest_lines), ruamel.yaml.RoundTripLoader)
)
if len(skips) != 0:
result_yaml["build"]["skip"] = skips
if result_yaml.get("outputs"):
for o in result_yaml["outputs"]:
name = o["name"]
package = {"name": name}
del o["name"]
if o.get("version"):
package["version"] = o["version"]
del o["version"]
build = {}
if o.get("script"):
build["script"] = o["script"]
del o["script"]
o["package"] = package
o["build"] = build
for d in result_yaml["outputs"]:
print(order_output_dict(d))
result_yaml["outputs"] = [order_output_dict(d) for d in result_yaml["outputs"]]
from io import StringIO
output = StringIO()
yaml.dump(result_yaml, output)
# Hacky way to insert an empty line after the context-key-object
context_output = StringIO()
yaml.dump(context, context_output)
context_output = context_output.getvalue()
context_output_len = len(context_output.split("\n"))
final_result = output.getvalue()
final_result_lines = final_result.split("\n")
final_result_lines.insert(context_output_len, "")
print("\n".join(final_result_lines))
def store_iam_resources_in_git(
iam_resources,
account_id,
git_url=config.get("cache_iam_resources_for_account.store_in_git.repo"),
git_message="[Automated] Update IAM Cache",
):
"""
Experimental function to force-push discovered IAM resources into a Git repository's master branch.
Use at your own risk.
"""
accounts_d = async_to_sync(get_account_id_to_name_mapping)()
tempdir = tempfile.mkdtemp()
try:
repo = clone_repo(git_url, tempdir)
expected_entries = {
"UserDetailList": {
"category": "iam_users",
"resource_name_key": "UserName",
},
"GroupDetailList": {
"category": "iam_groups",
"resource_name_key": "GroupName",
},
"RoleDetailList": {
"category": "iam_roles",
"resource_name_key": "RoleName",
},
"Policies": {
"category": "iam_policies",
"resource_name_key": "PolicyName"
},
}
for key, settings in expected_entries.items():
category = settings["category"]
for resource in iam_resources[key]:
if key == "RoleDetailList":
resource.pop("RoleLastUsed", None)
resource_name = resource[settings["resource_name_key"]]
yaml = YAML()
yaml.preserve_quotes = True # type: ignore
yaml.indent(mapping=2, sequence=4, offset=2)
account_name = accounts_d.get(account_id, account_id)
if not account_name:
account_name = "unknown"
path_in_repo = os.path.join(
repo.working_dir,
f"{account_name}/{category}/{resource_name}.yaml")
os.makedirs(Path(path_in_repo).parent.absolute(),
exist_ok=True)
should_write = True
to_write = sort_dict(resource)
if os.path.exists(path_in_repo):
with open(path_in_repo, "r") as f:
# Unfortunately at the time of writing, ruamel.yaml loads this into ordered dictionaries.
# We want this to be the same type as `to_write`, so we use the builtin yaml library to load it
existing = builtin_yaml.safe_load(f)
if not DeepDiff(to_write, existing, ignore_order=True):
should_write = False
if should_write:
with open(path_in_repo, "w") as f:
yaml.dump(to_write, f)
repo.git.add("*")
if repo.index.diff("HEAD"):
repo.index.commit(git_message)
origin = repo.remote("origin")
origin.pull()
origin.push("master", force=True)
except Exception: # noqa
sentry_sdk.capture_exception()
shutil.rmtree(tempdir)
from github import Github
from iteration_utilities import unique_everseen
from ruamel.yaml import YAML
GH_TOKEN = os.environ["GH_TOKEN"]
GH_ORG_NAME = os.getenv("GH_ORG_NAME", "cloudposse")
GH_SEARCH_PATTERN = os.getenv("GH_SEARCH_PATTERN", "terraform-")
TF_MODULE_PATH = os.getenv("TF_MODULE_PATH", ".")
TF_CONFIG_INSPECT_BINARY_PATH = os.getenv("TF_CONFIG_INSPECT_BINARY_PATH",
"terraform-config-inspect")
TF_REGISTRY_URL = "https://registry.terraform.io"
gh = Github(GH_TOKEN)
yaml = YAML(typ="rt")
yaml.default_flow_style = False
yaml.preserve_quotes = False
def parse_gh():
gh_repos = []
for repo in gh.get_organization(GH_ORG_NAME).get_repos():
if GH_SEARCH_PATTERN in repo.name:
repo_object = {}
repo_object["name"] = repo.name
repo_object["description"] = repo.description
repo_object["url"] = repo.html_url
gh_repos.append(repo_object)
return gh_repos
def tf_config_inspect():
def extract_to_package_format(self) -> int:
"""Extracts the self.yml_path into several files according to the Demisto standard of the package format.
Returns:
int. status code for the operation.
"""
print("Starting migration of: {} to dir: {}".format(self.yml_path, self.dest_path))
arg_path = self.dest_path
output_path = os.path.abspath(self.dest_path)
os.makedirs(output_path, exist_ok=True)
base_name = os.path.basename(output_path)
yml_type = self.get_yml_type()
code_file = "{}/{}.py".format(output_path, base_name)
self.extract_code(code_file)
self.extract_image("{}/{}_image.png".format(output_path, base_name))
self.extract_long_description("{}/{}_description.md".format(output_path, base_name))
yaml_out = "{}/{}.yml".format(output_path, base_name)
print("Creating yml file: {} ...".format(yaml_out))
ryaml = YAML()
ryaml.preserve_quotes = True
with open(self.yml_path, 'r') as yf:
yaml_obj = ryaml.load(yf)
script_obj = yaml_obj
if yml_type == INTEGRATION:
script_obj = yaml_obj['script']
del yaml_obj['image']
if 'detaileddescription' in yaml_obj:
del yaml_obj['detaileddescription']
if script_obj['type'] != 'python':
print('Script is not of type "python". Found type: {}. Nothing to do.'.format(script_obj['type']))
return 1
script_obj['script'] = SingleQuotedScalarString('')
with open(yaml_out, 'w') as yf:
ryaml.dump(yaml_obj, yf)
print("Running autopep8 on file: {} ...".format(code_file))
try:
subprocess.call(["autopep8", "-i", "--max-line-length", "130", code_file])
except FileNotFoundError:
print_color("autopep8 skipped! It doesn't seem you have autopep8 installed.\n"
"Make sure to install it with: pip install autopep8.\n"
"Then run: autopep8 -i {}".format(code_file), LOG_COLORS.YELLOW)
print("Detecting python version and setting up pipenv files ...")
docker = get_docker_images(script_obj)[0]
py_ver = get_python_version(docker, self.config.log_verbose)
pip_env_dir = get_pipenv_dir(py_ver, self.config.envs_dirs_base)
print("Copying pipenv files from: {}".format(pip_env_dir))
shutil.copy("{}/Pipfile".format(pip_env_dir), output_path)
shutil.copy("{}/Pipfile.lock".format(pip_env_dir), output_path)
try:
subprocess.call(["pipenv", "install", "--dev"], cwd=output_path)
print("Installing all py requirements from docker: [{}] into pipenv".format(docker))
requirements = subprocess.check_output(["docker", "run", "--rm", docker,
"pip", "freeze", "--disable-pip-version-check"],
universal_newlines=True, stderr=subprocess.DEVNULL).strip()
fp = tempfile.NamedTemporaryFile(delete=False)
fp.write(requirements.encode('utf-8'))
fp.close()
try:
subprocess.check_call(["pipenv", "install", "-r", fp.name], cwd=output_path)
except Exception:
print_color("Failed installing requirements in pipenv.\n "
"Please try installing manually after extract ends\n", LOG_COLORS.RED)
os.unlink(fp.name)
print("Installing flake8 for linting")
subprocess.call(["pipenv", "install", "--dev", "flake8"], cwd=output_path)
except FileNotFoundError:
print_color("pipenv install skipped! It doesn't seem you have pipenv installed.\n"
"Make sure to install it with: pip3 install pipenv.\n"
"Then run in the package dir: pipenv install --dev", LOG_COLORS.YELLOW)
# check if there is a changelog
yml_changelog = os.path.splitext(self.yml_path)[0] + '_CHANGELOG.md'
changelog = arg_path + '/CHANGELOG.md'
if os.path.exists(yml_changelog):
shutil.copy(yml_changelog, changelog)
else:
with open(changelog, 'wt', encoding='utf-8') as changelog_file:
changelog_file.write("## [Unreleased]\n-\n")
print_color("\nCompleted: setting up package: {}\n".format(arg_path), LOG_COLORS.GREEN)
print("Next steps: \n",
"* Install additional py packages for unit testing (if needed): cd {}; pipenv install <package>\n".format(
arg_path),
"* Create unit tests\n",
"* Check linting and unit tests by running: ./Tests/scripts/pkg_dev_test_tasks.py -d {}\n".format(
arg_path),
"* When ready rm from git the source yml and add the new package:\n",
" git rm {}\n".format(self.yml_path),
" git add {}\n".format(arg_path),
sep=''
)
return 0
def yaml() -> YAML:
"""Return default YAML parser."""
yamlp = YAML(typ='safe', pure=True)
yamlp.preserve_quotes = True
yamlp.default_flow_style = False
return yamlp
from ruamel.yaml import YAML
from chartpress import _check_call
from chartpress import _get_git_remote_url
from chartpress import _get_identifier_from_parts
from chartpress import _get_image_build_args
from chartpress import _get_latest_commit_tagged_or_modifying_paths
from chartpress import _image_needs_pushing
from chartpress import _strip_build_suffix_from_identifier
from chartpress import Builder
from chartpress import GITHUB_TOKEN_KEY
# use safe roundtrip yaml loader
yaml = YAML(typ="rt")
yaml.preserve_quotes = True ## avoid mangling of quotes
yaml.indent(mapping=2, offset=2, sequence=4)
def test__strip_build_suffix_from_identifier():
assert (_strip_build_suffix_from_identifier(
identifier="0.1.2-n005.hasdf1234") == "0.1.2")
assert (_strip_build_suffix_from_identifier(
identifier="0.1.2-alpha.1.n005.hasdf1234") == "0.1.2-alpha.1")
def test__get_identifier_from_parts():
assert (_get_identifier_from_parts(tag="0.1.2",
n_commits="0",
commit="asdf123",
long=True) == "0.1.2-n000.hasdf123")
assert (_get_identifier_from_parts(tag="0.1.2",
from typing import Union
import demisto_sdk.commands.common.content.errors as exc
from ruamel.yaml import YAML
from ruamel.yaml.scanner import ScannerError
from wcmatch.pathlib import EXTGLOB, NEGATE, Path
from .dictionary_based_object import DictionaryBasedObject
RUYAML = YAML(typ='rt')
RUYAML.preserve_quotes = True # type: ignore
RUYAML.width = 50000 # type: ignore
class YAMLObject(DictionaryBasedObject):
def __init__(self, path: Union[Path, str], file_name_prefix: str = ""):
super().__init__(path=path, file_name_prefix=file_name_prefix)
@staticmethod
def _fix_path(path: Union[Path, str]):
"""Find and validate object path is valid.
Rules:
1. Path exists.
2. One of the following options:
a. Path is a file.
b. Path is directory and file with a yml/yaml suffix exists in the given directory.
3. File suffix equal "yml" or "yaml".
Returns:
Path: valid file path.
def pretty_format_yaml(argv=None):
parser = argparse.ArgumentParser()
parser.add_argument(
'--autofix',
action='store_true',
dest='autofix',
help='Automatically fixes encountered not-pretty-formatted files',
)
parser.add_argument(
'--indent',
type=int,
default='2',
help=('The number of indent spaces or a string to be used as delimiter'
' for indentation level e.g. 4 or "\t" (Default: 2)'),
)
parser.add_argument(
'--preserve-quotes',
action='store_true',
dest='preserve_quotes',
help='Keep existing string quoting',
)
parser.add_argument('filenames', nargs='*', help='Filenames to fix')
args = parser.parse_args(argv)
status = 0
yaml = YAML()
yaml.indent = args.indent
yaml.preserve_quotes = args.preserve_quotes
# Prevent ruamel.yaml to wrap yaml lines
yaml.width = maxsize
separator = '---\n'
for yaml_file in set(args.filenames):
with open(yaml_file) as f:
string_content = ''.join(f.readlines())
# Split multi-document file into individual documents
#
# Not using yaml.load_all() because it reformats primitive (non-YAML) content. It removes
# newline characters.
separator_pattern = r'^---\s*\n'
original_docs = re.split(separator_pattern,
string_content,
flags=re.MULTILINE)
# A valid multi-document YAML file might starts with the separator.
# In this case the first document of original docs will be empty and should not be consdered
if string_content.startswith('---'):
original_docs = original_docs[1:]
pretty_docs = []
try:
for doc in original_docs:
content = _process_single_document(doc, yaml)
if content is not None:
pretty_docs.append(content)
# Start multi-doc file with separator
pretty_content = '' if len(pretty_docs) == 1 else separator
pretty_content += separator.join(pretty_docs)
if string_content != pretty_content:
print('File {} is not pretty-formatted'.format(yaml_file))
if args.autofix:
print('Fixing file {}'.format(yaml_file))
with io.open(yaml_file, 'w', encoding='UTF-8') as f:
f.write(text_type(pretty_content))
status = 1
except YAMLError: # pragma: no cover
print(
'Input File {} is not a valid YAML file, consider using check-yaml'
.format(yaml_file, ), )
return 1
return status
def load(obj_str: str, default_flow_style=False) -> dict:
yaml = YAML()
yaml.default_flow_style = default_flow_style
yaml.preserve_quotes = True
return yaml.load(obj_str)
def create_compose_file(self):
ds = {
'version': '3',
'networks': {
'ssonet': {
'ipam': {
'config': [{
'subnet': '172.23.0.0/24'
}]
}
}
},
'volumes': {
'local_postgres_data': {},
'local_postgres_data_backups': {},
'local_zookeeper_data': {},
'local_kafka_data': {}
},
'services': {
'sso.local.redhat.com': {
'container_name': 'sso.local.redhat.com',
'image': 'quay.io/keycloak/keycloak:11.0.0',
'environment': {
'DB_VENDOR': 'h2',
'PROXY_ADDRESS_FORWARDING': "true",
'KEYCLOAK_USER': '******',
'KEYCLOAK_PASSWORD': '******',
},
#'ports': ['8443:8443'],
'expose': [8443],
'networks': {
'ssonet': {
'ipv4_address': '172.23.0.3'
}
}
},
'kcadmin': {
'container_name':
'kcadmin',
'image':
'python:3',
'build': {
'context':
f"{os.path.join(self.checkouts_root, 'keycloak_admin')}",
},
'volumes': [
f"./{os.path.join(self.checkouts_root, 'keycloak_admin')}:/app"
],
'depends_on': ['sso.local.redhat.com'],
#'command': '/bin/bash -c "cd /app && pip install -r requirements.txt && flask run --host=0.0.0.0 --port=80"'
'command':
'/bin/bash -c "cd /app && pip install -r requirements.txt && python -c \'from kchelper import init_realm; init_realm()\' && flask run --host=0.0.0.0 --port=80"',
'networks': {
'ssonet': {
'ipv4_address': '172.23.0.4'
}
}
},
'insights_proxy': {
'container_name':
'insights_proxy',
'image':
'redhatinsights/insights-proxy',
#'ports': ['1337:1337'],
'ports': ['8443:8443'],
'environment': ['PLATFORM=linux', 'CUSTOM_CONF=true'],
'security_opt': ['label=disable'],
'extra_hosts': ['prod.foo.redhat.com:127.0.0.1'],
'environment': {
'SPANDX_PORT': 8443
},
'volumes': [
f'./{os.path.join(self.checkouts_root, "www", "spandx.config.js")}:/config/spandx.config.js'
]
},
'webroot': {
'container_name':
'webroot',
'image':
'nginx',
'volumes': [
f"./{os.path.join(self.checkouts_root, 'www')}:/usr/share/nginx/html",
f"./{os.path.join(self.checkouts_root, 'nginx.conf.d')}:/etc/nginx/conf.d"
],
'command': ['nginx-debug', '-g', 'daemon off;']
},
'chrome': {
'container_name':
'chrome',
'image':
'nginx',
'volumes': [
f"./{os.path.join(self.checkouts_root, 'insights-chrome')}:/usr/share/nginx/html"
],
'command': ['nginx-debug', '-g', 'daemon off;']
},
'chrome_beta': {
'container_name':
'chrome_beta',
'image':
'nginx',
'volumes': [
f"./{os.path.join(self.checkouts_root, 'insights-chrome')}:/usr/share/nginx/html"
],
'command': ['nginx-debug', '-g', 'daemon off;']
},
'entitlements': {
'container_name':
'entitlements',
'image':
'python:3',
'build': {
'context':
f"{os.path.join(self.checkouts_root, 'entitlements')}",
},
'volumes': [
f"./{os.path.join(self.checkouts_root, 'entitlements')}:/app"
],
'command':
'/bin/bash -c "cd /app && pip install -r requirements.txt && python api.py"'
},
'rbac': {
'container_name':
'rbac',
'image':
'python:3',
'build': {
'context':
f"{os.path.join(self.checkouts_root, 'rbac')}",
},
'volumes':
[f"./{os.path.join(self.checkouts_root, 'rbac')}:/app"],
'command':
'/bin/bash -c "cd /app && pip install -r requirements.txt && python api.py"'
}
}
}
ds['services'].update(self.get_tower_analytics_frontend_service())
ds['services'].update(self.get_landing_services())
#import epdb; epdb.st()
# macs can't do static IPs
if platform.system().lower() == 'darwin':
ds.pop('networks', None)
# Add squid for the mac users who can't directly connect to containers
if not self.args.integration:
squid_logs = os.path.join(self.checkouts_root, 'squid', 'logs')
squid_conf = os.path.join(self.checkouts_root, 'squid', 'conf')
if not os.path.exists(squid_logs):
os.makedirs(squid_logs)
ds['services']['squid'] = {
'container_name':
'squid',
'image':
'datadog/squid',
'ports': ['3128:3128'],
'volumes': [
f"./{squid_conf}:/etc/squid",
f"./{squid_logs}:/var/log/squid",
]
}
if True:
ds['services']['kcadmin'].pop('networks', None)
ds['services']['sso.local.redhat.com'].pop('networks', None)
ds['services']['sso.local.redhat.com'].pop('depends_on', None)
pf = copy.deepcopy(ds['services']['insights_proxy'])
pf['container_name'] = 'prod.foo.redhat.com'
ds['services'].pop('insights_proxy', None)
ds['services']['prod.foo.redhat.com'] = pf
# if static, chrome/landing/frontend should be compiled and put into wwwroot
if self.args.static:
if 'all' in self.args.static or 'chrome' in self.args.static:
ds['services'].pop('chrome', None)
ds['services'].pop('chrome_beta', None)
if 'all' in self.args.static or 'landing' in self.args.static:
ds['services'].pop('landing', None)
if 'all' in self.args.static or 'tower-analytics-frontend' in self.args.static:
ds['services'].pop('aafrontend', None)
for fc in self.frontend_services:
if 'all' in self.args.static or fc.www_app_name in self.args.static:
for dp in fc.www_deploy_paths:
src = os.path.join(fc.srcpath, fc.distdir)
dst = f"/usr/share/nginx/html/{dp}"
volume = f"./{src}:{dst}"
ds['services']['webroot']['volumes'].append(volume)
#import epdb; epdb.st()
# build the backend?
if self.args.backend_mock:
aa_be_srcpath = os.path.join(self.checkouts_root,
'aa_backend_mock')
bs = {
'container_name':
'aabackend',
'image':
'python:3',
'build': {
'context': f"./{aa_be_srcpath}"
},
'environment': {
'API_SECURE': '1',
},
'volumes': [f"./{aa_be_srcpath}:/app"],
'command':
'/bin/bash -c "cd /app && pip install -r requirements.txt && python api.py"'
}
ds['services']['aabackend'] = bs
elif self.args.backend_address:
pass
else:
svcs = self.get_backend_compose_services()
ds['services'].update(svcs)
#import epdb; epdb.st()
if self.args.integration:
ds['services']['integration'] = self.get_integration_compose()
yaml = YAML(typ='rt', pure=True)
yaml.preserve_quotes = False
yaml.indent = 4
yaml.block_seq_indent = 4
yaml.explicit_start = True
yaml.width = 1000
yaml.default_flow_style = False
with open('genstack.yml', 'w') as f:
yaml.dump(ds, f)
# fix port quoting for sshd ...
with open('genstack.yml', 'r') as f:
fyaml = f.read()
fyaml = fyaml.replace('2222:22', '\'2222:22\'')
with open('genstack.yml', 'w') as f:
f.write(fyaml)
import sys
from pathlib import Path
from ruamel.yaml import YAML
val = sys.argv[1]
subst = sys.argv[2]
file_name = Path(sys.argv[3])
def update(d, val, sub):
if isinstance(d, dict):
for k in d:
v = d[k]
if v == val:
d[k] = sub
else:
update(v, val, sub)
elif isinstance(d, list):
for item in d:
update(item, val, sub)
yaml = YAML()
yaml.preserve_quotes = True # to preserve superfluous quotes in the input
data = yaml.load(file_name)
update(data, val, subst)
yaml.dump(data, file_name)
import pytest
from demisto_sdk.commands.common.constants import PACKS_DIR, PLAYBOOKS_DIR
from demisto_sdk.commands.common.content.errors import (ContentInitializeError,
ContentSerializeError)
from demisto_sdk.commands.common.content.objects.abstract_objects import \
YAMLObject
from demisto_sdk.commands.common.tools import src_root
from ruamel.yaml import YAML
TEST_DATA = src_root() / 'tests' / 'test_files'
TEST_CONTENT_REPO = TEST_DATA / 'content_slim'
TEST_VALID_YAML = TEST_CONTENT_REPO / PACKS_DIR / 'Sample01' / PLAYBOOKS_DIR / 'playbook-sample_new.yml'
TEST_NOT_VALID_YAML = TEST_DATA / 'malformed.yaml'
RUYAML = YAML(typ='rt')
RUYAML.preserve_quotes = True
RUYAML.width = 50000
class TestValidYAML:
def test_valid_yaml_file_path(self):
obj = YAMLObject(TEST_VALID_YAML)
assert obj.to_dict() == RUYAML.load(TEST_VALID_YAML.open())
def test_get_item(self):
obj = YAMLObject(TEST_VALID_YAML)
assert obj["fromversion"] == RUYAML.load(
TEST_VALID_YAML.open())["fromversion"]
@pytest.mark.parametrize(argnames="default_value",
def experiment(args):
print(f"Running experiment with args: {args}")
args.cuda = not args.no_cuda and torch.cuda.is_available()
args.train_batch_num -= 1
args.val_batch_num -= 1
args.eval_batch_num -= 1
# TODO (JON): What is yaml for right now?
yaml = YAML()
yaml.preserve_quotes = True
yaml.boolean_representation = ['False', 'True']
torch.manual_seed(args.seed)
np.random.seed(args.seed)
if args.cuda: torch.cuda.manual_seed(args.seed)
########## Setup dataset
# TODO (JON): Verify that the loaders are shuffling the validation / test sets.
if args.dataset == 'MNIST':
num_train = args.datasize
if num_train == -1: num_train = 50000
train_loader, val_loader, test_loader = load_mnist(
args.batch_size,
subset=[args.datasize, args.valsize, args.testsize],
num_train=num_train)
in_channel = 1
imsize = 28
fc_shape = 800
num_classes = 10
else:
raise Exception("Must choose MNIST dataset")
# TODO (JON): Right now we are not using the test loader for anything. Should evaluate it occasionally.
##################### Setup model
if args.model == "mlp":
model = Net(args.num_layers,
args.dropout,
imsize,
in_channel,
args.l2,
num_classes=num_classes)
else:
raise Exception("bad model")
hyper = init_hyper_train(args, model) # We need this when doing all_weight
if args.cuda:
model = model.cuda()
model.weight_decay = model.weight_decay.cuda()
# model.Gaussian.dropout = model.Gaussian.dropout.cuda()
############ Setup Optimizer
# TODO (JON): Add argument for other optimizers?
init_optimizer = torch.optim.Adam(model.parameters(),
lr=args.lr) # , momentum=0.9)
hyper_optimizer = torch.optim.RMSprop([get_hyper_train(
args, model)]) # , lr=args.lrh) # try 0.1 as lr
############## Setup Inversion Algorithms
KFAC_damping = 1e-2
kfac_opt = KFACOptimizer(model, damping=KFAC_damping) # sec_optimizer
########### Perform the training
global_step = 0
hp_k, update = 0, 0
for epoch_h in range(0, args.hepochs + 1):
print(f"Hyper epoch: {epoch_h}")
if (epoch_h) % args.hyper_log_interval == 0:
if args.hyper_train == 'opt_data':
if args.dataset == 'MNIST':
save_learned(
get_hyper_train(args,
model).reshape(args.batch_size, imsize,
imsize), True,
args.batch_size, args)
elif args.hyper_train == 'various':
print(
f"saturation: {torch.sigmoid(model.various[0])}, brightness: {torch.sigmoid(model.various[1])}, decay: {torch.exp(model.various[2])}"
)
eval_train_corr, eval_train_loss = evaluate(
args, model, global_step, train_loader, 'train')
# TODO (JON): I don't know if we want normal train loss, or eval?
eval_val_corr, eval_val_loss = evaluate(args, model, epoch_h,
val_loader, 'valid')
eval_test_corr, eval_test_loss = evaluate(args, model, epoch_h,
test_loader, 'test')
if args.break_perfect_val and eval_val_corr >= 0.999 and eval_train_corr >= 0.999:
break
min_loss = 10e8
elementary_epochs = args.epochs
if epoch_h == 0:
elementary_epochs = args.init_epochs
if True: # epoch_h == 0:
optimizer = init_optimizer
# else:
# optimizer = sec_optimizer
for epoch in range(1, elementary_epochs + 1):
global_step, epoch_train_loss = train(args, model, train_loader,
optimizer, train_loss_func,
kfac_opt, epoch, global_step)
if np.isnan(epoch_train_loss):
print("Loss is nan, stop the loop")
break
elif False: # epoch_train_loss >= min_loss:
print(
f"Breaking on epoch {epoch}. train_loss = {epoch_train_loss}, min_loss = {min_loss}"
)
break
min_loss = epoch_train_loss
# if epoch_h == 0:
# continue
hp_k, update = KFAC_optimize(args, model, train_loader, val_loader,
hyper_optimizer, kfac_opt, KFAC_damping,
epoch_h)
def yaml() -> YAML:
"""Return default YAML parser."""
yamlp = YAML(typ='safe', pure=True)
yamlp.preserve_quotes = True
yamlp.default_flow_style = False
return yamlp
def _parse_sequence(self):
"""Parse a sequence file to be able to provide information about it.
Make sure the sequence file is valid before calling this function.
File can be checked with `SequenceReader.check_sequence`.
"""
with open(self._seq_file_path) as f:
yaml = YAML(typ='safe')
yaml.preserve_quotes = True
loaded = yaml.load(f)
# Collect constants
if 'constants' in loaded['sequence']:
self._constants = loaded['sequence']['constants']
else:
self._constants = dict()
# Create node objects
for node_dict in loaded['sequence']['nodes']:
ntype = node_dict['type']
if ntype == "function":
# list of wrappers is converted into an OrderedDict
wrapper_list = node_dict.get('wrappers')
if wrapper_list:
wrapper_dict = OrderedDict()
for wrapper in wrapper_list:
# A wrapper can be written as a simple string (name)
# or as a dict if some arguments are given.
if isinstance(wrapper, str):
wrapper_dict[wrapper] = {}
elif isinstance(wrapper, dict):
wrapper_dict.update(wrapper)
else:
raise SequenceFileError(
('The following wrapper is neiter a str '
' or a dict: {}').format(wrapper))
else:
wrapper_dict = None
# create function node
new_node = FunctionNode(
nid=node_dict.get('id'),
name=node_dict.get('name'),
transitions=node_dict.get('transitions'),
function_name=node_dict.get('function'),
function_kwargs=node_dict.get('arguments'),
timeout=node_dict.get('timeout'),
return_var_name=node_dict.get('return'),
wrappers=wrapper_dict)
elif ntype == "start":
new_node = StartNode(nid=node_dict.get('id'),
name=node_dict.get('name'),
transitions=node_dict.get('transitions'))
elif ntype == "stop":
new_node = StopNode(nid=node_dict.get('id'),
name=node_dict.get('name'))
elif ntype == "variable":
new_node = VariableNode(
nid=node_dict.get('id'),
name=node_dict.get('name'),
transitions=node_dict.get('transitions'),
variables=node_dict.get('variables'))
elif ntype == "parallel_split":
new_node = ParallelSplitNode(
nid=node_dict.get('id'),
name=node_dict.get('name'),
transitions=node_dict.get('transitions'))
elif ntype == "parallel_sync":
new_node = ParallelSyncNode(
nid=node_dict.get('id'),
name=node_dict.get('name'),
transitions=node_dict.get('transitions'))
else:
raise SequenceFileError("Node n°{} has an unknown type "
"{}.".format(node_dict['id'],
node_dict['type']))
# Add the new node to set of nodes in the SequenceReader
self._nodes.add(new_node)
