def from_dict(d: Dict, _filepath: str = None):
"""Construct Metadata from a dict.
Args:
d (dict): A dictionary of values
_filepath (str): path to the file where the data is coming from
"""
# be flexible to a common error where
if isinstance(d, list) and len(d) > 0:
d = merge_dicts(d)
lc_keys = lowercase_keys(d)
# common error: have metadata twice
if len(lc_keys) == 1 and "metadata" in lc_keys:
d = d[lc_keys["metadata"]]
lc_keys = lowercase_keys(d)
dd = d.copy()
for field_name in Metadata.COMMON_FIELDS:
key = field_name.lower()
if key in lc_keys:
dd[field_name] = dd.pop(lc_keys[key])
# try with _ instead of space in the field name
if " " in field_name:
key = field_name.lower().replace(" ", "_")
if key in lc_keys:
dd[field_name] = dd.pop(lc_keys[key])
return Metadata(
_filepath=_filepath,
**dd,
)
def from_file(filepath: str = None, parent_filepath: str = None):
"""Load a Collection from a file.
Args:
filepath (str): File from which to load the collection
parent_filepath (str): Parent filename (if file is imported from another file)
"""
fullpath = full_filepath(filepath, parent_filepath)
raw, md_path = load_any_file(filepath, parent_filepath)
d = raw
if isinstance(raw, dict):
lc_keys = lowercase_keys(raw)
if "collection" in lc_keys:
d = raw[lc_keys["collection"]]
elif "collections" in lc_keys:
# called Collection.from_file() on a collection list, fallback to CollectionList
d = raw[lc_keys["collections"]]
if isinstance(d, list):
from modelindex.models.CollectionList import CollectionList
return CollectionList(d, fullpath)
return Collection.from_dict(d, fullpath, md_path)
else:
raise ValueError(f"Expected a collection dict, but got "
f"something else in file '{fullpath}'")
def from_dict(d: Dict, _filepath: str = None):
"""Create a Result from a dict.
Args:
d (dict): dictionary containing result data
_filepath (str): The file path to where the data was loaded from
"""
lc_keys = lowercase_keys(d)
task = None
dataset = None
metrics = None
if "task" in lc_keys:
task = d[lc_keys["task"]]
if "dataset" in lc_keys:
dataset = d[lc_keys["dataset"]]
if "metrics" in lc_keys:
metrics = d[lc_keys["metrics"]]
return Result(
task=task,
dataset=dataset,
metrics=metrics,
_filepath=_filepath,
)
def __init__(self,
task: [str, Dict],
dataset: [str, Dict],
metrics: [Dict, List],
_filepath: str = None,
):
"""
Args:
task (str, Dict): The name of the ML task, or dict with "name"
dataset (str, Dict): The name of the dataset, or dict with "name"
metrics (dict, List): A dictionary of metrics, or a list of dicts with "name" and "value"
_filepath (str): Path to the file where the data was loaded from
"""
if isinstance(task, dict):
lc_keys = lowercase_keys(task)
if "name" in lc_keys:
task = task[lc_keys["name"]]
if isinstance(dataset, dict):
lc_keys = lowercase_keys(dataset)
if "name" in lc_keys:
dataset = dataset[lc_keys["name"]]
if isinstance(metrics, list):
metrics_out = {}
for m in metrics:
lc_keys = lowercase_keys(m)
if "name" in lc_keys and "value" in lc_keys:
key = m[lc_keys["name"]]
value = m[lc_keys["value"]]
metrics_out[key] = value
if metrics_out:
metrics = metrics_out
d = {
"Task": task,
"Dataset": dataset,
"Metrics": metrics,
}
super().__init__(
data=d,
filepath=_filepath
)
def from_dict(cls,
d: Dict,
_filepath: str = None,
_path_to_readme: str = None):
"""Create a Model from a dictionary.
Args:
d (dict): dictionary containing models data
_filepath (str): The file path to where the data was loaded from
_path_to_readme (str): Path to the README file if metadata was extracted from a README
"""
lc_keys = lowercase_keys(d)
copy_fields = [
"name",
"paper",
"code",
"weights",
"config",
"readme",
"metadata",
"results",
"in_collection",
"image",
]
dd = d.copy()
for field_name in copy_fields:
key = field_name.lower()
if key in lc_keys:
dd[field_name] = dd.pop(lc_keys[key])
# try with _ instead of space in the field name
if " " in field_name:
key = field_name.lower().replace(" ", "_")
if key in lc_keys:
dd[field_name] = dd.pop(lc_keys[key])
if _path_to_readme:
dd["readme"] = _path_to_readme
return cls(
_filepath=_filepath,
_path_to_readme=_path_to_readme,
**dd,
)
def from_dict(cls,
d: Dict,
_filepath: str = None,
_path_to_readme: str = None):
"""Create a Library from a dictionary.
Args:
d (dict): dictionary containing library data
_filepath (str): The file path to where the data was loaded from
_path_to_readme (str): Path to the README file if metadata was extracted from a README
"""
lc_keys = lowercase_keys(d)
copy_fields = [
"name",
"repository",
"headline",
"website",
"docs",
"readme",
"image",
]
dd = d.copy()
for field_name in copy_fields:
key = field_name.lower()
if key in lc_keys:
dd[field_name] = dd.pop(lc_keys[key])
# try with _ instead of space in the field name
if " " in field_name:
key = field_name.lower().replace(" ", "_")
if key in lc_keys:
dd[field_name] = dd.pop(lc_keys[key])
if _path_to_readme:
dd["readme"] = _path_to_readme
return cls(
_filepath=_filepath,
_path_to_readme=_path_to_readme,
**dd,
)
def from_file(filepath: str = None, parent_filepath: str = None):
"""Load a Library from a file.
Args:
filepath (str): File from which to load the library
parent_filepath (str): Parent filename (if file is imported from another file)
"""
fullpath = full_filepath(filepath, parent_filepath)
raw, md_path = load_any_file(filepath, parent_filepath)
d = raw
if isinstance(raw, dict):
lc_keys = lowercase_keys(raw)
if "library" in lc_keys:
d = raw[lc_keys["library"]]
return Library.from_dict(d, fullpath, md_path)
else:
raise ValueError(f"Expected a library dict, but got "
f"something else in file '{fullpath}'")
def from_file(filepath: str = None, parent_filepath: str = None):
"""Load a ModelList from a file.
Args:
filepath (str): File from which to load the list of models.
parent_filepath (str): Parent filename (if file is imported from another file)
"""
fullpath = full_filepath(filepath, parent_filepath)
raw, md_path = load_any_file(filepath, parent_filepath)
d = raw
if isinstance(d, list):
return ModelList(d, fullpath)
elif isinstance(d, dict):
lc_keys = lowercase_keys(raw)
if "models" in lc_keys:
return ModelList(d[lc_keys["models"]], fullpath)
raise ValueError(f"Expected a list of models, but got something else"
f"in file {fullpath}")
def __init__(self,
data: dict = None,
filepath: str = None,
_path_to_readme: str = None,
is_root: bool = False,
):
"""
Args:
data (dict): The root model index as a dictionary
filepath (str): The path from which it was loaded
_path_to_readme (str): The path to the readme file (if loaded from there)
is_root (bool): If this is the root ModelIndex instance for the whole project
"""
check_errors = OrderedSet()
if data is None:
data = {}
d = {
"Models": ModelList(_filepath=filepath),
"Collections": CollectionList(_filepath=filepath),
"Library": None,
}
lc_keys = lowercase_keys(data)
if "models" in lc_keys:
models = data[lc_keys["models"]]
# Syntax: Models: <path to file(s)>
if models is not None and isinstance(models, str):
models_list = []
for model_file in expand_wildcard_path(models, filepath):
try:
models_list.append(ModelList.from_file(model_file, filepath))
except (IOError, ValueError) as e:
check_errors.add(str(e))
models = merge_lists_data(models_list)
# Syntax: Models: list[ model dict ]
elif models is not None and not isinstance(models, ModelList):
models = ModelList(models, filepath, _path_to_readme)
d["Models"] = models
if "collections" in lc_keys:
collections = data[lc_keys["collections"]]
# Syntax: Collections: <path to file(s)>
if collections is not None and isinstance(collections, str):
collections_list = []
for model_file in expand_wildcard_path(collections, filepath):
try:
collections_list.append(CollectionList.from_file(model_file, filepath))
except (IOError, ValueError) as e:
check_errors.add(str(e))
collections = merge_lists_data(collections_list)
# Syntax: Collections: list[ model dict ]
elif collections is not None and not isinstance(collections, CollectionList):
collections = CollectionList(collections, filepath, _path_to_readme)
d["Collections"] = collections
if "library" in lc_keys:
lib = data[lc_keys["library"]]
if isinstance(lib, dict):
d["Library"] = Library.from_dict(lib, filepath)
elif isinstance(lib, str):
d["Library"] = Library.from_file(lib, filepath)
else:
check_errors.add("Mis-formatted `Library` entry: expected a dict or a filepath but got something else.")
if "import" in lc_keys:
imp = data[lc_keys["import"]]
if not isinstance(imp, list):
imp = list(imp)
for import_file in imp:
try:
for relpath in expand_wildcard_path(import_file, filepath):
raw, md_name = load_any_file(relpath, filepath)
fullpath = full_filepath(relpath, filepath)
mi = ModelIndex.from_dict(raw, fullpath, md_name)
if mi.models:
for model in mi.models:
d["Models"].add(model)
if mi.collections:
for col in mi.collections:
d["Collections"].add(col)
if mi.library:
d["Library"] = mi.library
except (IOError, ValueError) as e:
check_errors.add(str(e))
super().__init__(
data=d,
filepath=filepath,
check_errors=check_errors,
)
self.lc_keys = lowercase_keys(data)
self.is_root = is_root
if is_root:
self.build_models_with_collections()
