async def add_index(
self,
index_data: Union[str, Mapping[str, Any], IndexConfig, BringIndexTing],
allow_existing: bool = False,
) -> BringIndexTing:
if is_instance_or_subclass(index_data, BringIndexTing):
index: BringIndexTing = index_data # type: ignore
else:
index = await self.create_index(
index_data=index_data, allow_existing=allow_existing # type: ignore
)
if index.id in self.index_ids and not allow_existing:
raise FrklException(
f"Can't add index '{index.id}'.",
reason="Index with this id already added.",
)
if (
index.id in self.index_ids
and self._indexes.get(index.id, None) is not None
and index != self._indexes.get(index.id)
):
raise FrklException(
f"Can't add index '{index.id}'.",
reason="Different index with same id already exists.",
)
self._indexes[index.id] = index
return index
async def retrieve(self, *value_names: str,
**requirements) -> Dict[str, Any]:
result: Dict[str, Any] = {}
config: IndexConfig = requirements["config"]
if not is_instance_or_subclass(config, IndexConfig):
raise FrklException(
f"Can't process index {self.name}",
reason=f"Invalid index config type: {type(config)}",
)
self._id = config.id
if not self._initialized:
await self.init(config)
self._initialized = True
if "config" in value_names:
result["config"] = config.to_dict()
if "id" in value_names:
result["id"] = self._id
if "index_type" in value_names:
result["index_type"] = config.index_type
if "index_file" in value_names:
result["index_file"] = config.index_file
if "uri" in value_names:
result["uri"] = await self.get_uri()
if "index_type_config" in value_names:
result["index_type_config"] = config.index_type_config
if "defaults" in value_names:
_defaults = calculate_defaults(
typistry=self._tingistry_obj.typistry, data=config.defaults)
result["defaults"] = _defaults
if "info" in value_names:
result["info"] = config.info
if "labels" in value_names:
result["labels"] = config.labels
if "tags" in value_names:
result["tags"] = config.tags
if "metadata_timestamp" in value_names:
result["metadata_timestamp"] = await self.get_metadata_timestamp()
if "pkgs" in value_names:
# await self._ensure_pkgs(config)
result["pkgs"] = await self._get_pkgs()
return result
async def _create_task(self) -> Optional[FreckletTask]:
vars = await self.get_vars()
# input_vars = copy.deepcopy(vars)
# preprocess_task = await self.create_preprocessing_task(**input_vars)
input_vars = copy.deepcopy(vars)
_tasks_list = await self.create_processing_tasks(**input_vars)
if is_instance_or_subclass(_tasks_list, Frecklet):
tasks_list: Iterable[Union[Task, Frecklet]] = [_tasks_list
] # type: ignore
elif is_instance_or_subclass(_tasks_list, Task):
tasks_list = [_tasks_list] # type: ignore
else:
tasks_list = _tasks_list # type: ignore
final_list: List[Task] = []
for item in tasks_list:
if is_instance_or_subclass(item, Frecklet):
final_list.append(await item._create_task()) # type: ignore
else:
final_list.append(item) # type: ignore
msg = await self.get_msg()
desc = BringTaskDesc(name=self.name, msg=msg)
frecklet_task = FreckletTask(task_desc=desc,
result_type=self.get_result_type())
for t in final_list:
frecklet_task.add_task(t)
input_vars = copy.deepcopy(vars)
postprocess_task = await self.create_postprocess_task(**input_vars)
if postprocess_task:
# pp_desc = BringTaskDesc(name="postprocessing", msg=f"postprocessing {self.name}")
# postprocess_task.task_desc = pp_desc
frecklet_task.set_postprocess_task(postprocess_task)
return frecklet_task
async def get_pkg(self,
name: str,
raise_exception: bool = True) -> Optional[PkgTing]:
pkgs = await self.get_pkgs()
pkg = pkgs.get(name, None)
if pkg is None and raise_exception:
pkg_names = await self.pkg_names
raise FrklException(
msg=
f"Can't retrieve package '{name}' from index '{self.name}'.",
reason="No package with that name available.",
solution=
f"Make sure the package name is correct, available packages: {', '.join(pkg_names)}.",
)
elif is_instance_or_subclass(pkg, Exception) and raise_exception:
raise pkg # type: ignore
elif is_instance_or_subclass(pkg, Exception):
return None
return pkg
async def get_parent_pkg(self, source_details: Mapping[str, Any],
bring_index: BringIndexTing) -> PkgTing:
pkg_name = source_details["name"]
pkg_index = source_details.get("index", None)
if pkg_index is None:
pkg_index = bring_index
elif "." not in pkg_index:
ctx = await self._bring.get_index(pkg_index)
if ctx is None:
ctx_names = self._bring.index_ids
raise FrklException(
msg=f"Can't retrieve child pkg '{pkg_name}'.",
reason=
f"Requested index '{pkg_index}' not among available indexes: {', '.join(ctx_names)}",
)
pkg_index = ctx
else:
raise NotImplementedError()
# ting_name = f"{pkg_index.full_name}.pkgs.{pkg_name}"
ting = await pkg_index.get_pkg(pkg_name)
ting_name = ting.full_name
if ting is None:
pkg_list = []
for tn in self._bring._tingistry_obj.ting_names:
# if '.pkgs.' in tn:
pkg_list.append(tn)
pkg_list_string = "\n - ".join(pkg_list)
raise FrklException(
msg="Can't resolve bring pkg.",
reason=
f"Requested child pkg '{ting_name}' not among available pkgs:\n\n{pkg_list_string}",
)
if not is_instance_or_subclass(ting, PkgTing):
raise FrklException(
msg="Can't resolve bring pkg.",
reason=
f"Parent pkg '{ting_name}' does not sub-class the PkgTing class.",
)
return ting # type: ignore
def create(cls, pkg_spec: Any) -> "PkgSpec":
if is_instance_or_subclass(pkg_spec, PkgSpec):
return pkg_spec
if not pkg_spec:
pkg_spec = {}
elif isinstance(pkg_spec, str):
pkg_spec = {"items": [{PATH_KEY: pkg_spec}]}
elif not isinstance(pkg_spec, collections.abc.Mapping):
raise TypeError(
f"Invalid type '{type(pkg_spec)}' for pkg spec: {pkg_spec}")
pkg_spec_obj = PkgSpec(**pkg_spec)
return pkg_spec_obj
async def get_pkg(
self,
pkg_name: str,
bring_index: BringIndexTing,
pkg_index: Optional[str] = None,
) -> PkgTing:
if pkg_index is None:
pkg_index = bring_index.full_name
elif "." not in pkg_index:
ctx = await self._bring.get_index(pkg_index)
if ctx is None:
ctx_names = self._bring.index_ids
raise FrklException(
msg=f"Can't retrieve child pkg '{pkg_name}'.",
reason=f"Requested index '{pkg_index}' not among available indexes: {', '.join(ctx_names)}",
)
pkg_index = ctx.full_name
ting_name = f"{pkg_index}.pkgs.{pkg_name}"
ting = self._bring._tingistry_obj.get_ting(ting_name)
if ting is None:
pkg_list = []
for tn in self._bring._tingistry_obj.ting_names:
# if '.pkgs.' in tn:
pkg_list.append(tn)
pkg_list_string = "\n - ".join(pkg_list)
raise FrklException(
msg="Can't resolve bring pkg.",
reason=f"Requested child pkg '{ting_name}' not among available pkgs:\n\n{pkg_list_string}",
)
if not is_instance_or_subclass(ting, PkgTing):
raise FrklException(
msg="Can't resolve bring pkg.",
reason=f"Parent pkg '{ting_name}' does not sub-class the PkgTing class.",
)
return ting # type: ignore