def get_dataclass_data(obj: Any,
allow_objects: Optional[bool] = None) -> Dict[str, Any]:
from omegaconf.omegaconf import MISSING, OmegaConf, _maybe_wrap
flags = {
"allow_objects": allow_objects
} if allow_objects is not None else {}
d = {}
is_type = isinstance(obj, type)
obj_type = get_type_of(obj)
dummy_parent = OmegaConf.create({}, flags=flags)
dummy_parent._metadata.object_type = obj_type
resolved_hints = get_type_hints(obj_type)
for field in dataclasses.fields(obj):
name = field.name
is_optional, type_ = _resolve_optional(resolved_hints[field.name])
type_ = _resolve_forward(type_, obj.__module__)
has_default = field.default != dataclasses.MISSING
has_default_factory = field.default_factory != dataclasses.MISSING # type: ignore
if not is_type:
value = getattr(obj, name)
else:
if has_default:
value = field.default
elif has_default_factory:
value = field.default_factory() # type: ignore
else:
value = MISSING
if is_union_annotation(
type_) and not is_supported_union_annotation(type_):
e = ConfigValueError(
f"Unions of containers are not supported:\n{name}: {type_str(type_)}"
)
format_and_raise(node=None,
key=None,
value=value,
cause=e,
msg=str(e))
try:
d[name] = _maybe_wrap(
ref_type=type_,
is_optional=is_optional,
key=name,
value=value,
parent=dummy_parent,
)
except (ValidationError, GrammarParseError) as ex:
format_and_raise(node=dummy_parent,
key=name,
value=value,
cause=ex,
msg=str(ex))
d[name]._set_parent(None)
dict_subclass_data = extract_dict_subclass_data(obj=obj,
parent=dummy_parent)
if dict_subclass_data is not None:
d.update(dict_subclass_data)
return d
def get_attr_data(obj: Any, allow_objects: Optional[bool] = None) -> Dict[str, Any]:
from omegaconf.omegaconf import OmegaConf, _maybe_wrap
flags = {"allow_objects": allow_objects} if allow_objects is not None else {}
dummy_parent = OmegaConf.create(flags=flags)
from omegaconf import MISSING
d = {}
is_type = isinstance(obj, type)
obj_type = obj if is_type else type(obj)
for name, attrib in attr.fields_dict(obj_type).items():
is_optional, type_ = _resolve_optional(attrib.type)
type_ = _resolve_forward(type_, obj.__module__)
if not is_type:
value = getattr(obj, name)
else:
value = attrib.default
if value == attr.NOTHING:
value = MISSING
if _is_union(type_):
e = ConfigValueError(
f"Union types are not supported:\n{name}: {type_str(type_)}"
)
format_and_raise(node=None, key=None, value=value, cause=e, msg=str(e))
d[name] = _maybe_wrap(
ref_type=type_,
is_optional=is_optional,
key=name,
value=value,
parent=dummy_parent,
)
d[name]._set_parent(None)
return d
def _ensure_container(target: Any, flags: Optional[Dict[str, bool]] = None) -> Any:
from omegaconf import OmegaConf
if is_primitive_container(target):
assert isinstance(target, (list, dict))
target = OmegaConf.create(target, flags=flags)
elif is_structured_config(target):
target = OmegaConf.structured(target, flags=flags)
assert OmegaConf.is_config(target)
return target
def get_dataclass_data(obj: Any,
allow_objects: Optional[bool] = None) -> Dict[str, Any]:
from omegaconf.omegaconf import MISSING, OmegaConf, _maybe_wrap
flags = {
"allow_objects": allow_objects
} if allow_objects is not None else {}
dummy_parent = OmegaConf.create({}, flags=flags)
d = {}
resolved_hints = get_type_hints(get_type_of(obj))
for field in dataclasses.fields(obj):
name = field.name
is_optional, type_ = _resolve_optional(resolved_hints[field.name])
type_ = _resolve_forward(type_, obj.__module__)
if hasattr(obj, name):
value = getattr(obj, name)
if value == dataclasses.MISSING:
value = MISSING
else:
if field.default_factory == dataclasses.MISSING: # type: ignore
value = MISSING
else:
value = field.default_factory() # type: ignore
if _is_union(type_):
e = ConfigValueError(
f"Union types are not supported:\n{name}: {type_str(type_)}")
format_and_raise(node=None,
key=None,
value=value,
cause=e,
msg=str(e))
try:
d[name] = _maybe_wrap(
ref_type=type_,
is_optional=is_optional,
key=name,
value=value,
parent=dummy_parent,
)
except ValidationError as ex:
format_and_raise(node=None,
key=name,
value=value,
cause=ex,
msg=str(ex))
d[name]._set_parent(None)
return d
def _ensure_container(target: Any, flags: Optional[Dict[str, bool]] = None) -> Any:
from omegaconf import OmegaConf
if is_primitive_container(target):
assert isinstance(target, (list, dict))
target = OmegaConf.create(target, flags=flags)
elif is_structured_config(target):
target = OmegaConf.structured(target, flags=flags)
elif not OmegaConf.is_config(target):
raise ValueError(
"Invalid input. Supports one of "
+ "[dict,list,DictConfig,ListConfig,dataclass,dataclass instance,attr class,attr class instance]"
)
return target
def test_cnyes_api_parse(req_json, snapshot):
es.connect()
scraper = cnyes.CnyesApiScraper(
OmegaConf.create(
{
"proxy": {"enabled": False},
"run": {
"throttle": 1,
"max_startpoints": 1,
"startpoints_csv": "startpoints_csv",
},
"store": {"es": {"host": "es01:9200"}},
}
)
)
[pages, new_urls] = scraper.parse(None, req_json)
assert {"pages": [e.to_dict() for e in pages][:3], "new_urls": new_urls} == snapshot