def get_dataclass_data(obj: Any, allow_objects: Optional[bool] = None) -> Dict[str, Any]: from omegaconf.omegaconf import MISSING, OmegaConf, _maybe_wrap flags = { "allow_objects": allow_objects } if allow_objects is not None else {} d = {} is_type = isinstance(obj, type) obj_type = get_type_of(obj) dummy_parent = OmegaConf.create({}, flags=flags) dummy_parent._metadata.object_type = obj_type resolved_hints = get_type_hints(obj_type) for field in dataclasses.fields(obj): name = field.name is_optional, type_ = _resolve_optional(resolved_hints[field.name]) type_ = _resolve_forward(type_, obj.__module__) has_default = field.default != dataclasses.MISSING has_default_factory = field.default_factory != dataclasses.MISSING # type: ignore if not is_type: value = getattr(obj, name) else: if has_default: value = field.default elif has_default_factory: value = field.default_factory() # type: ignore else: value = MISSING if is_union_annotation( type_) and not is_supported_union_annotation(type_): e = ConfigValueError( f"Unions of containers are not supported:\n{name}: {type_str(type_)}" ) format_and_raise(node=None, key=None, value=value, cause=e, msg=str(e)) try: d[name] = _maybe_wrap( ref_type=type_, is_optional=is_optional, key=name, value=value, parent=dummy_parent, ) except (ValidationError, GrammarParseError) as ex: format_and_raise(node=dummy_parent, key=name, value=value, cause=ex, msg=str(ex)) d[name]._set_parent(None) dict_subclass_data = extract_dict_subclass_data(obj=obj, parent=dummy_parent) if dict_subclass_data is not None: d.update(dict_subclass_data) return d
def get_attr_data(obj: Any, allow_objects: Optional[bool] = None) -> Dict[str, Any]: from omegaconf.omegaconf import OmegaConf, _maybe_wrap flags = {"allow_objects": allow_objects} if allow_objects is not None else {} dummy_parent = OmegaConf.create(flags=flags) from omegaconf import MISSING d = {} is_type = isinstance(obj, type) obj_type = obj if is_type else type(obj) for name, attrib in attr.fields_dict(obj_type).items(): is_optional, type_ = _resolve_optional(attrib.type) type_ = _resolve_forward(type_, obj.__module__) if not is_type: value = getattr(obj, name) else: value = attrib.default if value == attr.NOTHING: value = MISSING if _is_union(type_): e = ConfigValueError( f"Union types are not supported:\n{name}: {type_str(type_)}" ) format_and_raise(node=None, key=None, value=value, cause=e, msg=str(e)) d[name] = _maybe_wrap( ref_type=type_, is_optional=is_optional, key=name, value=value, parent=dummy_parent, ) d[name]._set_parent(None) return d
def _ensure_container(target: Any, flags: Optional[Dict[str, bool]] = None) -> Any: from omegaconf import OmegaConf if is_primitive_container(target): assert isinstance(target, (list, dict)) target = OmegaConf.create(target, flags=flags) elif is_structured_config(target): target = OmegaConf.structured(target, flags=flags) assert OmegaConf.is_config(target) return target
def get_dataclass_data(obj: Any, allow_objects: Optional[bool] = None) -> Dict[str, Any]: from omegaconf.omegaconf import MISSING, OmegaConf, _maybe_wrap flags = { "allow_objects": allow_objects } if allow_objects is not None else {} dummy_parent = OmegaConf.create({}, flags=flags) d = {} resolved_hints = get_type_hints(get_type_of(obj)) for field in dataclasses.fields(obj): name = field.name is_optional, type_ = _resolve_optional(resolved_hints[field.name]) type_ = _resolve_forward(type_, obj.__module__) if hasattr(obj, name): value = getattr(obj, name) if value == dataclasses.MISSING: value = MISSING else: if field.default_factory == dataclasses.MISSING: # type: ignore value = MISSING else: value = field.default_factory() # type: ignore if _is_union(type_): e = ConfigValueError( f"Union types are not supported:\n{name}: {type_str(type_)}") format_and_raise(node=None, key=None, value=value, cause=e, msg=str(e)) try: d[name] = _maybe_wrap( ref_type=type_, is_optional=is_optional, key=name, value=value, parent=dummy_parent, ) except ValidationError as ex: format_and_raise(node=None, key=name, value=value, cause=ex, msg=str(ex)) d[name]._set_parent(None) return d
def _ensure_container(target: Any, flags: Optional[Dict[str, bool]] = None) -> Any: from omegaconf import OmegaConf if is_primitive_container(target): assert isinstance(target, (list, dict)) target = OmegaConf.create(target, flags=flags) elif is_structured_config(target): target = OmegaConf.structured(target, flags=flags) elif not OmegaConf.is_config(target): raise ValueError( "Invalid input. Supports one of " + "[dict,list,DictConfig,ListConfig,dataclass,dataclass instance,attr class,attr class instance]" ) return target
def test_cnyes_api_parse(req_json, snapshot): es.connect() scraper = cnyes.CnyesApiScraper( OmegaConf.create( { "proxy": {"enabled": False}, "run": { "throttle": 1, "max_startpoints": 1, "startpoints_csv": "startpoints_csv", }, "store": {"es": {"host": "es01:9200"}}, } ) ) [pages, new_urls] = scraper.parse(None, req_json) assert {"pages": [e.to_dict() for e in pages][:3], "new_urls": new_urls} == snapshot