def read_config(base_dir, fns, fn):
"""读取目录下的配置文件"""
configs = dict()
kind = {
PATH_SECRETS: "Secret",
PATH_CONFIGS: "ConfigMap"
}.get(base_dir, None)
if os.path.exists(LOCAL_FUNC_CONFIG_PATH) and kind is not None:
docs = yaml.full_load_all(open(LOCAL_FUNC_CONFIG_PATH))
for doc in docs:
if doc.get("kind", None) == kind:
paths = [
doc.get("metadata", {}).get("namespace", "none"),
doc.get("metadata", {}).get("name", "none")
]
for k, v in doc.get("data", {}).items():
add_params(configs, paths, k, v)
if os.path.exists(GLOBAL_FUNC_CONFIG_PATH) and kind is not None:
docs = yaml.full_load_all(open(GLOBAL_FUNC_CONFIG_PATH))
for doc in docs:
if doc.get("kind", None) == kind:
paths = [
doc.get("metadata", {}).get("namespace", "none"),
doc.get("metadata", {}).get("name", "none")
]
for k, v in doc.get("data", {}).items():
add_params(configs, paths, k, v)
# set alias, make it easy for user to get the parameters
if "fission-secret-configmap" in configs:
configs[GLOBAL_CONFIG_KEY] = configs["fission-secret-configmap"].get(
"fission-function-global-configmap", {})
local_key = "func-{}".format(fn)
if fns in configs and local_key in configs.get(fns, {}):
configs[LOCAL_CONFIG_KEY] = configs.get(fns, {}).get(local_key, {})
return configs
def render(self, values: HelmValues) -> List[Any]:
with NamedTemporaryFile() as tmp_file:
values_json = json.loads(values.json(exclude_none=True))
pprint(values_json)
content = yaml.dump(values_json)
tmp_file.write(content.encode())
tmp_file.flush()
command = [
"helm",
"template",
self.name,
os.path.join(git_repo_root(), "helm", "dagster"),
"--dependency-update",
"--debug",
*['--values', tmp_file.name],
*["--show-only", self.output],
]
templates = subprocess.check_output(command)
print("\n--- Helm Templates ---") # pylint: disable=print-call
print(templates.decode()) # pylint: disable=print-call
k8s_objects = [
self.api_client._ApiClient__deserialize_model( # pylint: disable=W0212
k8s_object, self.model
)
for k8s_object in yaml.full_load_all(templates)
if k8s_object
]
return k8s_objects
def test_rc_file_readable_by_scopesim_parser(self):
default_file = os.path.join(rc.__pkg_dir__, "defaults.yaml")
with open(default_file, "r") as f:
default_dict = [dic for dic in yaml.full_load_all(f)]
assert isinstance(default_dict[0], dict)
assert len(default_dict[0]) > 0
def load_yamls(*paths: str) -> dict:
result = {}
for p in paths:
with open(p, 'r', encoding='utf-8') as fp:
for doc in yaml.full_load_all(fp):
result.update(doc)
return result
def _read_services_descriptions(files):
descriptions = []
for f in files:
docs = list(yaml.full_load_all(f))
descriptions.append(docs)
services = list(chain.from_iterable(descriptions))
return ServiceDescriptions._slugify_names(services)
def render_chart(name="RELEASE-NAME",
values=None,
show_only=None,
validate_schema=True):
"""
Function that renders a helm chart into dictionaries. For helm chart testing only
"""
values = values or {}
with NamedTemporaryFile() as tmp_file:
content = yaml.dump(values)
tmp_file.write(content.encode())
tmp_file.flush()
command = [
"helm", "template", name, sys.path[0], '--values', tmp_file.name
]
if show_only:
for i in show_only:
command.extend(["--show-only", i])
templates = subprocess.check_output(command)
k8s_objects = yaml.full_load_all(templates)
k8s_objects = [k8s_object for k8s_object in k8s_objects
if k8s_object] # type: ignore
if validate_schema:
for k8s_object in k8s_objects:
validate_k8s_object(k8s_object)
return k8s_objects
def main():
"""
"""
parser = argparse.ArgumentParser(
description=
'Fix up source adoc files for nicer formatting in the static site')
parser.add_argument('topic_map', type=str, help='Path to _topic_map.yml')
args = parser.parse_args()
# FIX: The Config class should have methods to parse and save config options.
config = Config()
if validate_file_path(args.topic_map):
config.topic_map_path = pathlib.Path(args.topic_map)
config.input_dir = config.topic_map_path.parent
# Open and read the topic map. If the file cannot be parsed,
# exit the program and return 1.
with config.topic_map_path.open(mode='r') as stream:
try:
config.topic_map_yaml = list(yaml.full_load_all(stream))
except yaml.YAMLError as e:
print(f'Error: Invalid YAML in _topic_map.yml: {e}')
sys.exit(1)
fix_doc_tree(config)
print('OK')
def loadYamlReplaceVarLocal(yaml_file, fields, multi_resource=False):
fileToBeReplaced = path.join(path.dirname(__file__), yaml_file)
if not path.exists(fileToBeReplaced):
print("The file {} does not exist" "".format(fileToBeReplaced))
sys.exit(1)
try:
with open(fileToBeReplaced, 'r') as f:
filedata = f.read()
for searchwrd, replwrd in fields.items():
filedata = filedata.replace(searchwrd, replwrd)
if multi_resource:
yaml_data = list(yaml.full_load_all(filedata))
else:
yaml_data = yaml.full_load(filedata)
# print(yaml_data)
except request.URLError as e:
print(e.reason)
sys.exit(1)
return yaml_data
# dataDict = {"{{region_name}}":"us-west-2","{{cluster_name}}":"_my_cluster","{{vpc_id}}": "testeste12345"}
# loadYamlLocal('../app_resources/jupyter-config.yaml', True)
def logs(log):
logfile = os.path.join(context.resources.logger.log_path, log)
if not os.path.exists(logfile):
flask.abort(404)
with open(logfile, 'r') as f:
objs = [obj for obj in yaml.full_load_all(f)]
if len(objs) == 1:
obj = objs[0]
else:
obj = {'entries': objs}
object_type = obj.get('object_type', None)
if object_type == fetch.rid.FetchedISAs.__name__:
obj = {
'summary': summarize.isas(fetch.rid.FetchedISAs(obj)),
'details': obj,
}
elif object_type == fetch.scd.FetchedEntities.__name__:
obj = {
'summary': summarize.entities(fetch.scd.FetchedEntities(obj)),
'details': obj,
}
elif object_type == fetch.rid.FetchedFlights.__name__:
obj = {
'summary': summarize.flights(fetch.rid.FetchedFlights(obj)),
'details': obj,
}
return flask.render_template('log.html',
log=_redact_and_augment_log(obj),
title=logfile)
def test_all_files_referenced_in_yamls_exist(self):
missing_files = []
for pkg_name in PKG_DICT:
no_missing = 0
yaml_files = glob(PKG_DICT[pkg_name] + "/*.yaml")
for yaml_file in yaml_files:
with open(yaml_file) as f:
try:
yaml_dicts = [dic for dic in yaml.full_load_all(f)]
except:
yaml_dicts = []
fnames = []
for yaml_dict in yaml_dicts:
fnames += recursive_filename_search(yaml_dict)
for fname in fnames:
if fname is not None:
if not isinstance(fname, (list, tuple)):
fname = [fname]
for fn in fname:
if fn.lower() != "none" and fn[0] != "!":
full_fname = pth.join(PKG_DICT[pkg_name], fn)
if not pth.exists(full_fname):
BADGES[pkg_name]["structure"][
fn] = "missing"
no_missing += 1
missing_files += [full_fname]
if no_missing == 0:
BADGES[f"!{pkg_name}.structure.no_missing_files"] = True
assert not missing_files, f"{missing_files}"
def load_yaml_replace_var_local(yaml_file, fields, multi_resource=False, write_output=False):
file_to_replace=path.join(path.dirname(__file__), yaml_file)
if not path.exists(file_to_replace):
print("The file {} does not exist"
"".format(file_to_replace))
sys.exit(1)
try:
with open(file_to_replace, 'r') as f:
filedata = f.read()
for searchwrd, replwrd in fields.items():
filedata = filedata.replace(searchwrd, replwrd)
if multi_resource:
yaml_data = list(yaml.full_load_all(filedata))
else:
yaml_data = yaml.full_load(filedata)
if write_output:
with open(file_to_replace, "w") as f:
yaml.dump(yaml_data, f, default_flow_style=False, allow_unicode = True, sort_keys=False)
# print(yaml_data)
except request.URLError as e:
print(e.reason)
sys.exit(1)
return yaml_data
def read_yaml_all(path):
with open(path, mode='r', encoding='utf-8') as f:
raw_data = f.read()
yaml_data = yaml.full_load_all(raw_data)
print(yaml_data)
print(type(yaml_data))
return list(yaml_data)
def getDatas(self, key):
all_data = yaml.full_load_all(self.f)
d = None
for data in all_data:
if data.get(key):
d = data.get(key)
return d
def check_pipeline_run_was_successful(pipeline_run_path: str) -> None:
with open(pipeline_run_path, 'r') as f:
for pipeline_run in yaml.full_load_all(f):
run_status = pipeline_run['status']['state']
if run_status != 'SUCCESS':
raise AssertionError(
f'pipeline run {pipeline_run_path} was unsuccessful, with status: {run_status}'
)
def load_file(r_path):
with open(r_path, 'r') as resource:
if r_path.endswith('json'):
return loads(resource).get("Resources")
elif r_path.endswith('yaml') or r_path.endswith('yml'):
return yaml.full_load_all(resource).get("Resources")
elif r_path.endswith('template'):
return load(resource).get("Resources")
def load_yaml(yaml_content):
try:
return list(yaml.full_load_all(fix_yaml(yaml_content)))
except yaml.parser.ParserError:
raise BrokenYamlError("""
It seems that you're yaml file is broken.
Run in through some online validators to find the reason behind it.
""")
def _load_template(self, path, update):
with open(path, "r", encoding="utf-8") as fp:
loaded_template = yaml.full_load_all(fp.read())
for index, raw_template in enumerate(loaded_template):
self._get_validated_obj(raw_template, self._template_schema)
kind = raw_template["kind"]
name = raw_template["name"]
meta = raw_template["meta"]
raw_template["spec"] = raw_template.get("spec", {})
random_name = False
if name == "random":
name = base64.b64encode(
(path + str(index)).encode()).decode()
random_name = True
category = raw_template["category"]
tags = raw_template["tags"]
if name in self._templates and update == False:
raise ValueError(f"name already exists. name is {name}")
if kind == "":
raise ValueError("kind is empty.")
if name == "":
raise ValueError("name is empty.")
if category == "":
raise ValueError("category is empty.")
if not isinstance(meta, dict):
raise ValueError(f"meta is not a dict. meta is {meta}")
# spec validate
if "template_schema" in raw_template and raw_template[
"template_schema"] != {}:
raw_template["spec"] = self._get_validated_obj(
raw_template["spec"], raw_template["template_schema"])
self._templates[name] = {
"kind": kind,
"name": name,
"category": category,
"tags": tags,
"origin": raw_template,
"path": path,
"meta": meta,
"random_name": random_name
}
def LoadYamlDictionary(self, yaml_output):
"""Load YAML parameter dictionary generated by GenerateYamlDictionary()."""
with open(yaml_output, 'r') as f:
for param in yaml.full_load_all(f):
pid = param.pop('id')
device_net = param.pop('device_net')
macro = param.pop('macro')
can = param.pop('can')
bank = param.pop('bank')
typ = param.pop('type')
self._AddTableEntry(pid, device_net, macro, can, bank, typ)
def logs(log):
logfile = os.path.join(context.resources.logger.log_path, log)
if not os.path.exists:
flask.abort(404)
with open(logfile, 'r') as f:
objs = [obj for obj in yaml.full_load_all(f)]
if len(objs) == 1:
obj = objs[0]
else:
obj = {'entries': objs}
return flask.render_template('log.html',
log=_redact_log(obj),
title=logfile)
def _create_resource(self, yaml_file_path):
try:
cluster = self._app.selected_cluster
with open(yaml_file_path, "r") as f:
resources = yaml.full_load_all(f)
for resource in resources:
cluster.do_post(
resource["apiVersion"],
resource["kind"],
namespace=resource["metadata"].get("namespace"),
body=resource,
)
except Exception as e:
self._app.show_error(e)
def loadYamlRemotely(url, multi_resource=False):
try:
fileToBeParsed = request.urlopen(url)
if multi_resource:
yaml_data = list(yaml.full_load_all(fileToBeParsed))
else:
yaml_data = yaml.full_load(fileToBeParsed)
# print(yaml_data)
except:
print("Cannot read yaml config file {}, check formatting."
"".format(fileToBeParsed))
sys.exit(1)
return yaml_data
def load(yaml_file: str) -> List[dict]:
"""
Read the yaml file into python
:return: list of all yaml configs
:rtype: list
:raise FileNotFoundError: yaml file could not be found on disk
:raise yaml.error.YAMLError: yaml file is malformed
"""
if not os.path.isfile(yaml_file):
raise FileNotFoundError("Yaml file is not found at path")
with open(yaml_file, 'r') as file:
yaml_configs = list(yaml.full_load_all(file))
return yaml_configs
def from_file(path_: Union[str, Path]) -> List['RegexQA']:
"""Converts multi-doc yaml file into list of RegexQA instances"""
log.info(f'creating RegexQA from file: {str(path_)}')
if isinstance(path_, str):
path = Path(path_)
else:
path = cast(Path, path_)
result: List['RegexQA'] = []
with open(path) as file:
for doc in yaml.full_load_all(file):
regex = doc['regex']
responses = doc['responses']
result.append(RegexQA(regex, responses))
return result
def parse_endpoint(endpoint, job=None):
epItem = {
"id": endpoint["id"],
"name": endpoint["name"],
"username": endpoint["username"],
"status": endpoint["status"],
"hostNetwork": endpoint["hostNetwork"],
"podName": endpoint["podName"],
"domain": config["domain"],
}
if "podPort" in endpoint:
epItem["podPort"] = endpoint["podPort"]
if endpoint["status"] == "running":
port = int(
endpoint["endpointDescription"]["spec"]["ports"][0]["nodePort"])
epItem["port"] = port
if "nodeName" in endpoint:
epItem["nodeName"], epItem["domain"] = getNodename()
if epItem["name"] == "ssh":
try:
if job:
desc = list(
yaml.full_load_all(
base64.b64decode(job["jobDescription"])))
if epItem["id"].find("worker") != -1:
desc = desc[int(
re.match(".*worker(\d+)-ssh",
epItem["id"]).groups()[0]) + 1]
elif epItem["id"].find("ps0") != -1:
desc = desc[0]
else:
desc = desc[0]
for i in desc["spec"]["containers"][0]["env"]:
if i["name"] == "DLTS_JOB_TOKEN":
epItem["password"] = i["value"]
except Exception as e:
logging.error(e)
elif epItem["name"] == "inference-url":
epItem["modelname"] = endpoint["modelname"]
epItem["port"] = base64.b64encode(
str(epItem["port"]).encode("utf-8"))
elif epItem["name"] in ["ipython", "tensorboard", "vscode"]:
epItem["port"] = base64.b64encode(
str(epItem["port"]).encode("utf-8"))
if epItem["name"] in ["ipython", "tensorboard", "vscode", "inference-url"]:
if "extranet_port" in config and config["extranet_port"]:
epItem["domain"] = epItem["domain"] + ":" + str(
config["extranet_port"])
return epItem
def test_flux_scales_with_pixel_scale(self, filter_name, bg_level):
yaml_text = YAML_TEXT % (WAVE_MIN, WAVE_MAX, PIXEL_SCALE, PIXEL_SCALE,
filter_name)
yamls = [yml for yml in yaml.full_load_all(yaml_text)]
cmd = sim.UserCommands(yamls=yamls)
opt = sim.OpticalTrain(cmd)
opt.cmds["!TEL.area"] = 1 * u.m**2
src = empty_sky()
opt.observe(src)
img = opt.image_planes[0].data
# Ks band photon flux is 1026 ph/s/m2/arcsec2
assert np.median(img) == pytest.approx(bg_level, rel=0.01)
def load_from_yaml(self, yaml_path: str):
with open(yaml_path, "r") as file:
settings: dict = yaml.full_load_all(file)
if settings["max_episode_steps"] != "":
self.max_episode_steps = settings["max_episode_steps"]
if settings["save_game_name"] != "":
self.save_game_name = settings["save_game_name"]
if settings["krpc"]["address"] != "":
self.krpc_address = settings["krpc"]["address"]
if settings["krpc"]["rpc_port"] != "":
self.krpc_rpc_port = settings["krpc"]["rpc_port"]
if settings["krpc"]["stream_port"] != "":
self.krpc_stream_port = settings["krpc"]["stream_port"]
def _get_headers_from_yaml(self, yaml_str):
try:
if not yaml_str.strip().startswith('---'):
raise StopIteration()
return next(yaml.full_load_all(yaml_str))
except yaml.YAMLError as e:
logger.info(
'YAML header is incorrectly formatted or missing. The '
f'following information may be useful:\n{e}\nIf you continue '
'to have difficulties, try pasting your YAML header into an '
'online parser such as http://yaml-online-parser.appspot.com/.'
)
except StopIteration as e:
logger.info('YAML header is missing!')
return {}
def render(
self,
values: Union[DagsterHelmValues, DagsterUserDeploymentsHelmValues],
chart_version: Optional[str] = None,
) -> List[Any]:
with NamedTemporaryFile() as tmp_file:
helm_dir_path = os.path.join(git_repo_root(), self.helm_dir_path)
values_json = json.loads(
values.json(exclude_none=True, by_alias=True))
pprint(values_json)
content = yaml.dump(values_json)
tmp_file.write(content.encode())
tmp_file.flush()
command = [
"helm",
"template",
self.name,
helm_dir_path,
"--debug",
*["--values", tmp_file.name],
]
if self.output:
## Uncomment to render all templates before filtering to surface Helm templating
## errors with better error messages
# subprocess.check_output(command)
command += ["--show-only", self.output]
with self._with_chart_yaml(helm_dir_path, chart_version):
templates = subprocess.check_output(command)
print("\n--- Helm Templates ---") # pylint: disable=print-call
print(templates.decode()) # pylint: disable=print-call
k8s_objects = [
k8s_object for k8s_object in yaml.full_load_all(templates)
if k8s_object
]
if self.model:
k8s_objects = [
self.api_client._ApiClient__deserialize_model( # pylint: disable=W0212
k8s_object, self.model) for k8s_object in k8s_objects
]
return k8s_objects
def test_full_load_all_yaml(self):
txt = dedent('''
---
file1: !include tests/data/include.d/1.yaml
---
file2: !include tests/data/include.d/2.yaml
''').strip()
iterable = yaml.full_load_all(txt)
for i, data in enumerate(iterable):
if i == 0:
self.assertEqual(data, {'file1': YAML1})
elif i == 1:
self.assertEqual(data, {'file2': YAML2})
else:
raise RuntimeError()
def read(filename):
try:
with open(filename, 'rt', encoding='utf8') as settings_file:
return next(yaml.full_load_all(settings_file))
except FileNotFoundError:
#print('brak pliku konfiguracyjnego')
sg.popup('brak pliku konfiguracyjnego')
sys.exit(-1)
except yaml.scanner.ScannerError:
#print('błędy w pliku konfiguracyjnym')
sg.popup('błędy pliku konfiguracyjnego')
sys.exit(-2)
except:
#print("nieoczekiwany błąd przy wczytywaniu konfiguracji:", sys.exc_info()[0])
sg.popup("nieoczekiwany błąd przy wczytywaniu konfiguracji")
sys.exit(-3)
