def test_get_set_value_init(self):
item = self.item_class(name="asdf", value=1234)
adapter = ItemAdapter(item)
self.assertEqual(adapter.get("name"), "asdf")
self.assertEqual(adapter.get("value"), 1234)
self.assertEqual(adapter["name"], "asdf")
self.assertEqual(adapter["value"], 1234)
def test_get_field_meta(self):
"""Metadata is always empty for the default implementation."""
adapter = ItemAdapter(self.item_class())
self.assertEqual(adapter.get_field_meta("_undefined_"),
MappingProxyType({}))
self.assertEqual(adapter.get_field_meta("name"), MappingProxyType({}))
self.assertEqual(adapter.get_field_meta("value"), MappingProxyType({}))
def test_non_item(self):
with self.assertRaises(TypeError):
ItemAdapter(ScrapySubclassedItem)
with self.assertRaises(TypeError):
ItemAdapter(dict)
with self.assertRaises(TypeError):
ItemAdapter(1234)
def test_as_dict_nested(self):
item = self.item_class_nested(
nested=self.item_class(name="asdf", value=1234),
adapter=ItemAdapter(dict(foo="bar", nested_list=[1, 2, 3, 4, 5])),
dict_={
"foo": "bar",
"answer": 42,
"nested_dict": {
"a": "b"
}
},
list_=[1, 2, 3],
set_={1, 2, 3},
tuple_=(1, 2, 3),
int_=123,
)
adapter = ItemAdapter(item)
self.assertEqual(
adapter.asdict(),
dict(
nested=dict(name="asdf", value=1234),
adapter=dict(foo="bar", nested_list=[1, 2, 3, 4, 5]),
dict_={
"foo": "bar",
"answer": 42,
"nested_dict": {
"a": "b"
}
},
list_=[1, 2, 3],
set_={1, 2, 3},
tuple_=(1, 2, 3),
int_=123,
),
)
def test_get_field_meta(self):
adapter = ItemAdapter(self.item_class())
self.assertEqual(adapter.get_field_meta("_undefined_"),
MappingProxyType({}))
self.assertEqual(adapter.get_field_meta("name"),
MappingProxyType({"serializer": str}))
self.assertEqual(adapter.get_field_meta("value"),
MappingProxyType({"serializer": int}))
def test_get_field_meta_from_class(self):
self.assertEqual(
ItemAdapter.get_field_meta_from_class(self.item_class,
"_undefined_"),
MappingProxyType({}),
)
self.assertEqual(
ItemAdapter.get_field_meta_from_class(self.item_class, "name"),
MappingProxyType({"serializer": str}),
)
self.assertEqual(
ItemAdapter.get_field_meta_from_class(self.item_class, "value"),
MappingProxyType({"serializer": int}),
)
def test_repr_pydantic(self):
item = PydanticModel(name="asdf", value=1234)
adapter = ItemAdapter(item)
self.assertEqual(
repr(adapter),
"<ItemAdapter for PydanticModel(name='asdf', value=1234)>",
)
def test_repr_attrs(self):
item = AttrsItem(name="asdf", value=1234)
adapter = ItemAdapter(item)
self.assertEqual(
repr(adapter),
"<ItemAdapter for AttrsItem(name='asdf', value=1234)>",
)
def test_repr_dataclass(self):
item = DataClassItem(name="asdf", value=1234)
adapter = ItemAdapter(item)
self.assertEqual(
repr(adapter),
"<ItemAdapter for DataClassItem(name='asdf', value=1234)>",
)
def is_item(obj: Any) -> bool:
"""Return True if the given object belongs to one of the supported types, False otherwise.
Alias for ItemAdapter.is_item
"""
from itemadapter.adapter import ItemAdapter
return ItemAdapter.is_item(obj)
def test_repr_scrapy_item(self):
item = ScrapySubclassedItem(name="asdf", value=1234)
adapter = ItemAdapter(item)
# Scrapy fields are stored in a dict, which is not guarantied to be sorted in py35
self.assertTrue(
repr(adapter) == "<ItemAdapter for ScrapySubclassedItem(name='asdf', value=1234)>"
or repr(adapter) == "<ItemAdapter for ScrapySubclassedItem(value=1234, name='asdf')>",
)
def test_repr_dict(self):
item = dict(name="asdf", value=1234)
adapter = ItemAdapter(item)
# dicts are not guarantied to be sorted in py35
self.assertTrue(
repr(adapter) == "<ItemAdapter for dict(name='asdf', value=1234)>"
or repr(adapter) == "<ItemAdapter for dict(value=1234, name='asdf')>",
)
def test_repr_dataclass_init_false(self):
item = DataClassWithoutInit()
adapter = ItemAdapter(item)
self.assertEqual(repr(adapter),
"<ItemAdapter for DataClassWithoutInit()>")
adapter["name"] = "set after init"
self.assertEqual(
repr(adapter),
"<ItemAdapter for DataClassWithoutInit(name='set after init')>")
def test_repr(self):
item = self.item_class()
adapter = ItemAdapter(item)
self.assertEqual(repr(adapter), "<ItemAdapter for FakeItemClass()>")
adapter["name"] = "asdf"
adapter["value"] = 1234
self.assertEqual(
repr(adapter),
"<ItemAdapter for FakeItemClass(name='asdf', value=1234)>")
def test_repr_attrs_init_false(self):
item = AttrsItemWithoutInit()
adapter = ItemAdapter(item)
self.assertEqual(repr(adapter),
"<ItemAdapter for AttrsItemWithoutInit()>")
adapter["name"] = "set after init"
self.assertEqual(
repr(adapter),
"<ItemAdapter for AttrsItemWithoutInit(name='set after init')>")
def test_metadata_common(self):
adapter = ItemAdapter(self.item_class())
self.assertIsInstance(adapter.get_field_meta("name"), MappingProxyType)
self.assertIsInstance(adapter.get_field_meta("value"),
MappingProxyType)
with self.assertRaises(KeyError):
adapter.get_field_meta("undefined_field")
def test_get_set_value(self):
item = self.item_class()
adapter = ItemAdapter(item)
self.assertEqual(adapter.get("name"), None)
self.assertEqual(adapter.get("value"), None)
adapter["name"] = "asdf"
adapter["value"] = 1234
self.assertEqual(adapter.get("name"), "asdf")
self.assertEqual(adapter.get("value"), 1234)
self.assertEqual(adapter["name"], "asdf")
self.assertEqual(adapter["value"], 1234)
def parse(self, response):
script = response.xpath("//script[contains(text(), 'window.As24ClassifiedList')]/text()").extract_first()
xml = lxml.etree.tostring(js2xml.parse(script), encoding='unicode')
selector = scrapy.Selector(text=xml)
models = selector.css('property[name="availableModelModelLines"] > array > object')
# create result dict for updating maker with models in pipeline
result = {
'makerId': selector.css('property[name="makeId"] > string::text').extract_first(),
'models': []
}
for raw_model in models:
# only queryable models are extracted and persisted in the database
if (raw_model.css('property[name="isModel"] > boolean::text').extract_first() == 'true'):
l = ItemLoader(item=lmpd.ModelItem(), selector=raw_model)
l.add_css('id', 'property[name="id"] > string')
l.add_css('name', 'property[name="name"] > string')
result['models'].append(ItemAdapter(l.load_item()).asdict())
return result
def get_field_meta_from_class(item_class: type,
field_name: str) -> MappingProxyType:
"""Return a read-only mapping with metadata for the given field name, within the given item class.
If there is no metadata for the field, or the item class does not support field metadata,
an empty object is returned.
Field metadata is taken from different sources, depending on the item type:
* scrapy.item.Item: corresponding scrapy.item.Field object
* dataclass items: "metadata" attribute for the corresponding field
* attrs items: "metadata" attribute for the corresponding field
* pydantic models: corresponding pydantic.field.FieldInfo/ModelField object
The returned value is an instance of types.MappingProxyType, i.e. a dynamic read-only view
of the original mapping, which gets automatically updated if the original mapping changes.
"""
from itemadapter.adapter import ItemAdapter
return ItemAdapter.get_field_meta_from_class(item_class, field_name)
def test_delitem_len_iter(self):
item = self.item_class(name="asdf", value=1234)
adapter = ItemAdapter(item)
self.assertEqual(len(adapter), 2)
self.assertEqual(sorted(list(iter(adapter))), ["name", "value"])
del adapter["name"]
self.assertEqual(len(adapter), 1)
self.assertEqual(sorted(list(iter(adapter))), ["value"])
del adapter["value"]
self.assertEqual(len(adapter), 0)
self.assertEqual(sorted(list(iter(adapter))), [])
with self.assertRaises(KeyError):
del adapter["name"]
with self.assertRaises(KeyError):
del adapter["value"]
with self.assertRaises(KeyError):
del adapter["_undefined_"]
def test_as_dict(self):
item = self.item_class(name="asdf", value=1234)
adapter = ItemAdapter(item)
self.assertEqual(dict(name="asdf", value=1234), dict(adapter))
def test_empty_metadata(self):
adapter = ItemAdapter(self.item_class(name="foo", value=5))
for field_name in ("name", "value", "undefined_field"):
self.assertEqual(adapter.get_field_meta(field_name), MappingProxyType({}))
def test_field_names(self):
item = self.item_class(name="asdf", value=1234)
adapter = ItemAdapter(item)
self.assertIsInstance(adapter.field_names(), KeysView)
self.assertEqual(sorted(adapter.field_names()), ["NAME", "VALUE"])
def test_repr_dict(self):
item = dict(name="asdf", value=1234)
adapter = ItemAdapter(item)
self.assertEqual(repr(adapter),
"<ItemAdapter for dict(name='asdf', value=1234)>")
def test_repr_scrapy_item(self):
item = ScrapySubclassedItem(name="asdf", value=1234)
adapter = ItemAdapter(item)
self.assertEqual(
repr(adapter),
"<ItemAdapter for ScrapySubclassedItem(name='asdf', value=1234)>")
def test_field_names_updated(self):
item = self.item_class(name="asdf")
field_names = ItemAdapter(item).field_names()
self.assertEqual(sorted(field_names), ["name"])
item["value"] = 1234
self.assertEqual(sorted(field_names), ["name", "value"])
def test_get_value_keyerror_item_dict(self):
"""Instantiate without default values."""
adapter = ItemAdapter(self.item_class())
with self.assertRaises(KeyError):
adapter["name"]
def test_set_value_keyerror(self):
item = self.item_class()
adapter = ItemAdapter(item)
with self.assertRaises(KeyError):
adapter["_undefined_"] = "some value"
def test_get_value_keyerror(self):
item = self.item_class()
adapter = ItemAdapter(item)
with self.assertRaises(KeyError):
adapter["undefined_field"]
def test_repr(self):
adapter = ItemAdapter(dict(foo="bar"))
self.assertEqual(repr(adapter), "<ItemAdapter for dict(foo='bar')>")