def setup_struct(struct: typing.MutableMapping):
if struct['options'][predef.PredefParseKVMode]:
fields = get_struct_kv_fields(struct)
struct['fields'] = fields
comment = struct.get("comment", "")
if comment == "":
comment = struct["options"].get(predef.PredefClassComment, "")
if comment != "":
struct["comment"] = comment
def override(target_dict: typing.MutableMapping,
override_dict: typing.Mapping):
"""Apply the updates in override_dict to the dict target_dict. This is like
dict.update, but recursive. i.e. if the existing element is a dict, then
override elements of the sub-dict rather than wholesale replacing.
One special case is added. If a key within override dict starts with '!' then
it is interpretted as follows:
- if the associated value is "REMOVE", the key is removed from the parent
dict
- use !! for keys that actually start with ! and shouldn't be removed.
e.g.
override(
{
'outer': { 'inner': { 'key': 'oldValue', 'existingKey': True } }
},
{
'outer': { 'inner': { 'key': 'newValue' } },
'newKey': { 'newDict': True },
}
)
yields:
{
'outer': {
'inner': {
'key': 'newValue',
'existingKey': True
}
},
'newKey': { newDict: True }
}
"""
for key, value in override_dict.items():
#
# Handle special ! syntax:
# "!keyname" : "REMOVE", --> remove the key 'keyname' from target_dict
#
if key[0:1] == "!" and key[1:2] != "!":
key = key[1:]
if value == "REMOVE":
target_dict.pop(key, None)
continue
current_value = target_dict.get(key)
if not isinstance(current_value, Mapping):
# Thing or Mapping overrides Thing or None
target_dict[key] = value
elif isinstance(value, Mapping):
# Mapping overrides mapping, recurse
target_dict[key] = override(current_value, value)
else:
# Thing overrides Mapping
target_dict[key] = value
return target_dict
def _lookup(
self,
key: str,
context: typing.MutableMapping,
) -> typing.Union[T, None, bool]:
result = self.mapping.get(key)
if result is not None:
result = getattr(result, context.get('profile') or 'default')
return result if result != '__ignored__' else False
return None
def recursive_worker(d: MutableMappingType,
u: MappingType) -> MutableMappingType:
for k, v in u.items():
dv = d.get(k, {})
if not isinstance(dv, MutableMapping):
d[k] = v
elif isinstance(v, Mapping):
d[k] = recursive_worker(dv, v)
else:
d[k] = v
return d
def setup_meta_kv_mode(meta: typing.MutableMapping):
meta[predef.PredefParseKVMode] = False
text = meta.get(predef.PredefKeyValueColumns, "")
if text != "":
columns = text.split(",")
assert len(columns) >= 3
meta[predef.PredefParseKVMode] = True
meta[predef.PredefKeyColumn] = int(columns[0])
meta[predef.PredefValueTypeColumn] = int(columns[1])
meta[predef.PredefValueColumn] = int(columns[2])
if len(columns) > 3:
meta[predef.PredefCommentColumn] = int(columns[3])
def __init__(
self,
message: str,
context: typing.MutableMapping,
exc_info: typing.Optional[str] = None,
) -> None:
"""
:param exc_info: Exception traceback (if applicable).
"""
super().__init__()
self.message = message
self.context = context
self.code = context.get('code') or message
self.exc_info = exc_info
def get_encoding_from_headers(headers: typing.MutableMapping) -> str:
"""Returns encodings from given HTTP Header Dict.
:param headers: dictionary to extract encoding from.
:rtype: str
"""
content_type = headers.get("Content-Type")
if not content_type:
return None
content_type, params = cgi.parse_header(content_type)
if "charset" in params:
return params["charset"].strip("'\"")
if "text" in content_type:
return "ISO-8859-1"
def handle_invalid_value(
self,
message: str,
exc_info: bool,
context: typing.MutableMapping,
) -> None:
key = context.get('key', '')
msg = FilterMessage(
message=message,
context=context,
exc_info=format_exc() if exc_info else None,
)
try:
self.messages[key].append(msg)
except KeyError:
self.messages[key] = [msg]
def merge(directory: pathlib.Path, data: typing.MutableMapping,
yaml_data: typing.MutableMapping) -> typing.MutableMapping:
result = copy.deepcopy(data)
yaml_data = copy.deepcopy(yaml_data)
yaml_items = collections.OrderedDict()
def override(key):
if key in yaml_data:
result[key] = yaml_data[key]
items_by_path = {}
for item in result.get('items', []):
items_by_path[item['path']] = item
if ('title' in yaml_data and not yaml_data['title'] == data['title']
and 'slug' not in yaml_data):
result['slug'] = slugify(yaml_data['title'])
for key in ('title', 'authors', 'categories', 'description', 'slug'):
override(key)
for idx, item in enumerate(yaml_data.get('items', [])):
item_path = directory / item['path']
current_item = items_by_path.get(item_path)
if current_item is None:
LOG.warn(f'Unknown item: {item_path!s} in YAML data')
continue
overrides = {}
for k in ('title', 'categories', 'description', 'chapters', 'sequence',
'explicit'):
if k in item:
overrides[k] = item[k]
if overrides:
current_item.update(overrides)
# Retain index for sorting purposes.
yaml_items[item_path] = idx
# Decide sorting order.
if len(items_by_path) == len(yaml_items):
LOG.debug(
'Manifest contains all the items, items without sequence will be '
'sorted by their order in the manifest')
sequences = {p: i for i, p in enumerate(yaml_items, start=1)}
else:
LOG.debug('Items without sequence will be sorted by path')
sequences = {
p: i
for i, p in enumerate(sorted(items_by_path), start=1)
}
def sort_key(item):
return item.get('sequence', sequences[item['path']])
result.update({
'items': sorted(result.get('items', []), key=sort_key),
})
return result
def populate_feedstock_attributes(
name: str,
sub_graph: typing.MutableMapping,
meta_yaml: typing.Union[str, Response] = "",
conda_forge_yaml: typing.Union[str, Response] = "",
mark_not_archived=False,
feedstock_dir=None,
) -> typing.MutableMapping:
"""Parse the various configuration information into something usable
Notes
-----
If the return is bad hand the response itself in so that it can be parsed
for meaning.
"""
sub_graph.update({"feedstock_name": name, "bad": False})
if mark_not_archived:
sub_graph.update({"archived": False})
# handle all the raw strings
if isinstance(meta_yaml, Response):
sub_graph["bad"] = f"make_graph: {meta_yaml.status_code}"
return sub_graph
sub_graph["raw_meta_yaml"] = meta_yaml
# Get the conda-forge.yml
if isinstance(conda_forge_yaml, str):
sub_graph["conda-forge.yml"] = {
k: v
for k, v in yaml.safe_load(conda_forge_yaml).items() if k in {
"provider",
"min_r_ver",
"min_py_ver",
"max_py_ver",
"max_r_ver",
"compiler_stack",
"bot",
}
}
if (feedstock_dir is not None and
len(glob.glob(os.path.join(feedstock_dir, ".ci_support",
"*.yaml"))) > 0):
recipe_dir = os.path.join(feedstock_dir, "recipe")
ci_support_files = glob.glob(
os.path.join(feedstock_dir, ".ci_support", "*.yaml"), )
varient_yamls = []
plat_arch = []
for cbc_path in ci_support_files:
cbc_name = os.path.basename(cbc_path)
cbc_name_parts = cbc_name.replace(".yaml", "").split("_")
plat = cbc_name_parts[0]
if len(cbc_name_parts) == 1:
arch = "64"
else:
if cbc_name_parts[1] in ["64", "aarch64", "ppc64le", "arm64"]:
arch = cbc_name_parts[1]
else:
arch = "64"
plat_arch.append((plat, arch))
varient_yamls.append(
parse_meta_yaml(
meta_yaml,
platform=plat,
arch=arch,
recipe_dir=recipe_dir,
cbc_path=cbc_path,
), )
# collapse them down
final_cfgs = {}
for plat_arch, varyml in zip(plat_arch, varient_yamls):
if plat_arch not in final_cfgs:
final_cfgs[plat_arch] = []
final_cfgs[plat_arch].append(varyml)
for k in final_cfgs:
ymls = final_cfgs[k]
final_cfgs[k] = _convert_to_dict(ChainDB(*ymls))
plat_arch = []
varient_yamls = []
for k, v in final_cfgs.items():
plat_arch.append(k)
varient_yamls.append(v)
else:
plat_arch = [("win", "64"), ("osx", "64"), ("linux", "64")]
for k in set(sub_graph["conda-forge.yml"].get("provider", {})):
if "_" in k:
plat_arch.append(k.split("_"))
varient_yamls = [
parse_meta_yaml(meta_yaml, platform=plat, arch=arch)
for plat, arch in plat_arch
]
# this makes certain that we have consistent ordering
sorted_varient_yamls = [
x for _, x in sorted(zip(plat_arch, varient_yamls))
]
yaml_dict = ChainDB(*sorted_varient_yamls)
if not yaml_dict:
logger.error(f"Something odd happened when parsing recipe {name}")
sub_graph["bad"] = "make_graph: Could not parse"
return sub_graph
sub_graph["meta_yaml"] = _convert_to_dict(yaml_dict)
meta_yaml = sub_graph["meta_yaml"]
for k, v in zip(plat_arch, varient_yamls):
plat_arch_name = "_".join(k)
sub_graph[f"{plat_arch_name}_meta_yaml"] = v
_, sub_graph[
f"{plat_arch_name}_requirements"], _ = extract_requirements(v)
(
sub_graph["total_requirements"],
sub_graph["requirements"],
sub_graph["strong_exports"],
) = extract_requirements(meta_yaml)
# handle multi outputs
if "outputs" in yaml_dict:
sub_graph["outputs_names"] = sorted(
list({d.get("name", "")
for d in yaml_dict["outputs"]}), )
# TODO: Write schema for dict
# TODO: remove this
req = get_requirements(yaml_dict)
sub_graph["req"] = req
keys = [("package", "name"), ("package", "version")]
missing_keys = [k[1] for k in keys if k[1] not in yaml_dict.get(k[0], {})]
source = yaml_dict.get("source", [])
if isinstance(source, collections.abc.Mapping):
source = [source]
source_keys: Set[str] = set()
for s in source:
if not sub_graph.get("url"):
sub_graph["url"] = s.get("url")
source_keys |= s.keys()
for k in keys:
if k[1] not in missing_keys:
sub_graph[k[1]] = yaml_dict[k[0]][k[1]]
kl = list(sorted(source_keys & hashlib.algorithms_available, reverse=True))
if kl:
sub_graph["hash_type"] = kl[0]
return sub_graph
def populate_feedstock_attributes(
name: str,
sub_graph: typing.MutableMapping,
meta_yaml: typing.Union[str, Response] = "",
conda_forge_yaml: typing.Union[str, Response] = "",
mark_not_archived=False,
feedstock_dir=None,
) -> typing.MutableMapping:
"""Parse the various configuration information into something usable
Notes
-----
If the return is bad hand the response itself in so that it can be parsed
for meaning.
"""
sub_graph.update({
"feedstock_name": name,
"bad": False,
"branch": "master"
})
if mark_not_archived:
sub_graph.update({"archived": False})
# handle all the raw strings
if isinstance(meta_yaml, Response):
sub_graph["bad"] = f"make_graph: {meta_yaml.status_code}"
return sub_graph
# strip out old keys - this removes old platforms when one gets disabled
for key in list(sub_graph.keys()):
if key.endswith("meta_yaml") or key.endswith(
"requirements") or key == "req":
del sub_graph[key]
sub_graph["raw_meta_yaml"] = meta_yaml
# Get the conda-forge.yml
if isinstance(conda_forge_yaml, str):
sub_graph["conda-forge.yml"] = {
k: v
for k, v in yaml.safe_load(conda_forge_yaml).items()
if k in CONDA_FORGE_YML_KEYS_TO_KEEP
}
if feedstock_dir is not None:
LOGGER.debug(
"# of ci support files: %s",
len(glob.glob(os.path.join(feedstock_dir, ".ci_support",
"*.yaml"))),
)
try:
if (feedstock_dir is not None and len(
glob.glob(os.path.join(feedstock_dir, ".ci_support",
"*.yaml"))) > 0):
recipe_dir = os.path.join(feedstock_dir, "recipe")
ci_support_files = glob.glob(
os.path.join(feedstock_dir, ".ci_support", "*.yaml"), )
varient_yamls = []
plat_arch = []
for cbc_path in ci_support_files:
cbc_name = os.path.basename(cbc_path)
cbc_name_parts = cbc_name.replace(".yaml", "").split("_")
plat = cbc_name_parts[0]
if len(cbc_name_parts) == 1:
arch = "64"
else:
if cbc_name_parts[1] in [
"64", "aarch64", "ppc64le", "arm64"
]:
arch = cbc_name_parts[1]
else:
arch = "64"
plat_arch.append((plat, arch))
varient_yamls.append(
parse_meta_yaml(
meta_yaml,
platform=plat,
arch=arch,
recipe_dir=recipe_dir,
cbc_path=cbc_path,
), )
# sometimes the requirements come out to None and this ruins the
# aggregated meta_yaml
if "requirements" in varient_yamls[-1]:
for section in ["build", "host", "run"]:
# We make sure to set a section only if it is actually in
# the recipe. Adding a section when it is not there might
# confuse migrators trying to move CB2 recipes to CB3.
if section in varient_yamls[-1]["requirements"]:
val = varient_yamls[-1]["requirements"].get(
section, [])
varient_yamls[-1]["requirements"][
section] = val or []
# collapse them down
final_cfgs = {}
for plat_arch, varyml in zip(plat_arch, varient_yamls):
if plat_arch not in final_cfgs:
final_cfgs[plat_arch] = []
final_cfgs[plat_arch].append(varyml)
for k in final_cfgs:
ymls = final_cfgs[k]
final_cfgs[k] = _convert_to_dict(ChainDB(*ymls))
plat_arch = []
varient_yamls = []
for k, v in final_cfgs.items():
plat_arch.append(k)
varient_yamls.append(v)
else:
plat_arch = [("win", "64"), ("osx", "64"), ("linux", "64")]
for k in set(sub_graph["conda-forge.yml"].get("provider", {})):
if "_" in k:
plat_arch.append(k.split("_"))
varient_yamls = [
parse_meta_yaml(meta_yaml, platform=plat, arch=arch)
for plat, arch in plat_arch
]
except Exception as e:
import traceback
trb = traceback.format_exc()
sub_graph["bad"] = f"make_graph: render error {e}\n{trb}"
raise
LOGGER.debug("platforms: %s", plat_arch)
# this makes certain that we have consistent ordering
sorted_varient_yamls = [
x for _, x in sorted(zip(plat_arch, varient_yamls))
]
yaml_dict = ChainDB(*sorted_varient_yamls)
if not yaml_dict:
LOGGER.error(f"Something odd happened when parsing recipe {name}")
sub_graph["bad"] = "make_graph: Could not parse"
return sub_graph
sub_graph["meta_yaml"] = _convert_to_dict(yaml_dict)
meta_yaml = sub_graph["meta_yaml"]
for k, v in zip(plat_arch, varient_yamls):
plat_arch_name = "_".join(k)
sub_graph[f"{plat_arch_name}_meta_yaml"] = v
_, sub_graph[
f"{plat_arch_name}_requirements"], _ = _extract_requirements(v)
(
sub_graph["total_requirements"],
sub_graph["requirements"],
sub_graph["strong_exports"],
) = _extract_requirements(meta_yaml)
# handle multi outputs
outputs_names = set()
if "outputs" in yaml_dict:
outputs_names.update(
set(list({d.get("name", "")
for d in yaml_dict["outputs"]}), ), )
# handle implicit meta packages
if "run" in sub_graph.get("meta_yaml", {}).get("requirements", {}):
outputs_names.add(meta_yaml["package"]["name"])
# add in single package name
else:
outputs_names = {meta_yaml["package"]["name"]}
sub_graph["outputs_names"] = outputs_names
# TODO: Write schema for dict
# TODO: remove this
req = _get_requirements(yaml_dict)
sub_graph["req"] = req
# set name and version
keys = [("package", "name"), ("package", "version")]
missing_keys = [k[1] for k in keys if k[1] not in yaml_dict.get(k[0], {})]
for k in keys:
if k[1] not in missing_keys:
sub_graph[k[1]] = yaml_dict[k[0]][k[1]]
# set the url and hash
sub_graph.pop("url", None)
sub_graph.pop("hash_type", None)
source = yaml_dict.get("source", [])
if isinstance(source, collections.abc.Mapping):
source = [source]
source_keys: Set[str] = set()
for s in source:
if not sub_graph.get("url"):
sub_graph["url"] = s.get("url")
source_keys |= s.keys()
kl = list(sorted(source_keys & hashlib.algorithms_available, reverse=True))
if kl:
sub_graph["hash_type"] = kl[0]
return sub_graph
def filter_empty_fields(data: typing.MutableMapping) -> typing.MutableMapping:
required = ('errors', ) if data.get('errors') else ('data', )
return {
key: value
for key, value in data.items() if key in required or value
}
def pick(_dict: MM,
keys: Iterable,
pop=True,
default=None) -> Generator[Any, None, None]:
for key in keys:
yield _dict.pop(key, default) if pop else _dict.get(key, default)
