def __check_parent(self, new_mb_rc):
parent_mb_cls = get_member_class(self.context.__parent__)
for attr_name, attr in type(new_mb_rc).get_attributes().iteritems():
if attr.kind == ResourceAttributeKinds.MEMBER \
and get_member_class(attr.value_type) is parent_mb_cls \
and getattr(new_mb_rc, attr_name) is None:
setattr(new_mb_rc, attr_name, self.context.__parent__)
def after(self):
# Register resources eagerly so the various adapters and utilities are
# available for other directives.
discriminator = ('resource', self.interface)
reg = get_current_registry()
config = Configurator(reg, package=self.context.package)
config.add_resource(self.interface, self.member, self.entity,
collection=self.collection,
collection_root_name=self.collection_root_name,
collection_title=self.collection_title,
repository=self.repository,
expose=self.expose,
_info=self.context.info)
for key, value in iteritems_(self.representers):
cnt_type, rc_kind = key
opts, mp_opts = value
if rc_kind == RESOURCE_KINDS.MEMBER:
rc = get_member_class(self.interface)
elif rc_kind == RESOURCE_KINDS.COLLECTION:
rc = get_collection_class(self.interface)
else: # None
rc = self.interface
discriminator = ('resource_representer', rc, cnt_type, rc_kind)
self.action(discriminator=discriminator, # pylint: disable=E1101
callable=config.add_resource_representer,
args=(rc, cnt_type),
kw=dict(options=opts,
attribute_options=mp_opts,
_info=self.context.info),
)
def visit(self, key, attribute_options):
# We only collect load options for attributes which are explicitly not
# ignored or member attributes unless they are explicitly ignored.
ignore_opt = attribute_options.get(IGNORE_OPTION)
if not ignore_opt is True:
rc = get_member_class(self._context)
entity_attr_names = []
store_key = True
for idx, attr_name in enumerate(key):
attr = get_resource_class_attribute(rc, attr_name)
if attr is None:
# Referencing non-existing attribute; ignore.
store_key = False
break
elif idx == 0 \
and is_collection_attribute(attr) \
and not ignore_opt is False:
# Referencing collection attribute which was not
# explicitly enabled.
store_key = False
break
entity_attr_name = attr.entity_attr
if is_terminal_attribute(attr):
if '.' in entity_attr_name:
entity_attr_name = \
entity_attr_name[:entity_attr_name.rfind('.')]
else:
store_key = False
break
entity_attr_names.append(entity_attr_name)
rc = attr.attr_type
if store_key:
self.__attr_map[key] = entity_attr_names
def __process_row(self, row_data, mapped_class, attribute_key):
is_repeating_row = len(attribute_key) == 0 \
and not self.__coll_data is None \
and self.__coll_data.has(self.__row_data_key)
if is_repeating_row:
self.__process_row(row_data,
self.__coll_data.collection_class,
self.__coll_data.attribute_key)
new_data_el = None
else:
mb_cls = get_member_class(mapped_class)
new_data_el = \
self._mapping.create_data_element(mapped_class=mb_cls)
for attr in self._mapping.terminal_attribute_iterator(
mapped_class, attribute_key):
self.__process_attr(attr, attribute_key, new_data_el,
row_data)
for attr in self._mapping.nonterminal_attribute_iterator(
mapped_class, attribute_key):
self.__process_attr(attr, attribute_key, new_data_el,
row_data)
if not self.__coll_data is None \
and mapped_class == self.__coll_data.collection_class:
if not self.__coll_data.has(self.__row_data_key):
coll_data_el = \
self._mapping.create_data_element(mapped_class=
mapped_class)
self.__coll_data.set(self.__row_data_key, coll_data_el)
else:
coll_data_el = self.__coll_data.get(self.__row_data_key)
coll_data_el.add_member(new_data_el)
new_data_el = coll_data_el
return new_data_el
def after(self):
# Register resources eagerly so the various adapters and utilities are
# available for other directives.
discriminator = ('resource', self.interface)
reg = get_current_registry()
config = Configurator(reg, package=self.context.package)
config.add_resource(self.interface,
self.member,
self.entity,
collection=self.collection,
collection_root_name=self.collection_root_name,
collection_title=self.collection_title,
repository=self.repository,
expose=self.expose,
_info=self.context.info)
for key, value in self.representers.iteritems():
cnt_type, rc_kind = key
opts, mp_opts = value
if rc_kind == RESOURCE_KINDS.member:
rc = get_member_class(self.interface)
elif rc_kind == RESOURCE_KINDS.collection:
rc = get_collection_class(self.interface)
else: # None
rc = self.interface
discriminator = ('resource_representer', rc, cnt_type, rc_kind)
self.action(
discriminator=discriminator, # pylint: disable=E1101
callable=config.add_resource_representer,
args=(rc, cnt_type),
kw=dict(options=opts,
attribute_options=mp_opts,
_info=self.context.info),
)
def __get_q_tag(self, attr):
# FIXME: We should cache the namespace for each attribute.
if not attr.namespace is None:
q_tag = '{%s}%s' % (attr.namespace, attr.repr_name)
else:
if attr.kind == RESOURCE_ATTRIBUTE_KINDS.TERMINAL:
xml_ns = \
self.mapping.configuration.get_option(XML_NAMESPACE_OPTION)
else:
if attr.kind == RESOURCE_ATTRIBUTE_KINDS.MEMBER:
attr_type = get_member_class(attr.value_type)
elif attr.kind == RESOURCE_ATTRIBUTE_KINDS.COLLECTION:
attr_type = get_collection_class(attr.value_type)
mp = self.mapping.mapping_registry.find_mapping(attr_type)
if not mp is None:
xml_ns = mp.configuration.get_option(XML_NAMESPACE_OPTION)
else: # pragma: no cover
# Not mapped.
# FIXME This case is neither tested nor documented.
xml_ns = None
if not xml_ns is None:
q_tag = '{%s}%s' % (xml_ns, attr.repr_name)
else:
q_tag = attr.repr_name
return q_tag
def __process_row(self, row_data, mapped_class, attribute_key):
is_repeating_row = len(attribute_key) == 0 \
and not self.__coll_data is None \
and self.__coll_data.has(self.__row_data_key)
if is_repeating_row:
self.__process_row(row_data, self.__coll_data.collection_class,
self.__coll_data.attribute_key)
new_data_el = None
else:
mb_cls = get_member_class(mapped_class)
new_data_el = \
self._mapping.create_data_element(mapped_class=mb_cls)
for attr in self._mapping.terminal_attribute_iterator(
mapped_class, attribute_key):
self.__process_attr(attr, attribute_key, new_data_el, row_data)
for attr in self._mapping.nonterminal_attribute_iterator(
mapped_class, attribute_key):
self.__process_attr(attr, attribute_key, new_data_el, row_data)
if not self.__coll_data is None \
and mapped_class == self.__coll_data.collection_class:
if not self.__coll_data.has(self.__row_data_key):
coll_data_el = \
self._mapping.create_data_element(mapped_class=
mapped_class)
self.__coll_data.set(self.__row_data_key, coll_data_el)
else:
coll_data_el = self.__coll_data.get(self.__row_data_key)
coll_data_el.add_member(new_data_el)
new_data_el = coll_data_el
return new_data_el
def test_dependency_graph(self):
grph = build_resource_dependency_graph(self._interfaces)
self.assert_equal(len(grph.nodes()), 4)
entity_mb_cls = get_member_class(IMyEntity)
entity_parent_mb_cls = get_member_class(IMyEntityParent)
entity_child_mb_cls = get_member_class(IMyEntityChild)
entity_grandchild_mb_cls = get_member_class(IMyEntityGrandchild)
# Entity Parent resource deps should be empty.
self.assert_equal(grph.neighbors(entity_parent_mb_cls), [])
# Entity Child has Grandchild.
self.assert_equal(grph.neighbors(entity_child_mb_cls),
[entity_grandchild_mb_cls])
# Entity Grandchild has Child, but backrefs are excluded by default.
self.assert_equal(grph.neighbors(entity_grandchild_mb_cls), [])
# Entity has Parent and Child.
self.assert_equal(set(grph.neighbors(entity_mb_cls)),
set([entity_parent_mb_cls, entity_child_mb_cls]))
def __get__(self, dummy, entity_class):
mb_cls = get_member_class(entity_class)
attr_map = OrderedDict()
for rc_attr in itervalues_(mb_cls.__everest_attributes__):
attr_map[rc_attr.entity_attr] = rc_attr
# This replaces the injector.
entity_class.__everest_attributes__ = attr_map
return attr_map
def _create_link_data_element(self, attribute, member_node):
if attribute.kind == RESOURCE_ATTRIBUTE_KINDS.MEMBER:
kind = RESOURCE_KINDS.MEMBER
rc_cls = get_member_class(attribute.value_type)
else:
kind = RESOURCE_KINDS.COLLECTION
rc_cls = get_collection_class(attribute.value_type)
return self._mapping.create_linked_data_element(
member_node, kind, relation=rc_cls.relation, title=rc_cls.title)
def test_dependency_graph(self):
grph = build_resource_dependency_graph(self._interfaces)
self.assert_equal(len(grph.nodes()), 4)
entity_mb_cls = get_member_class(IMyEntity)
entity_parent_mb_cls = get_member_class(IMyEntityParent)
entity_child_mb_cls = get_member_class(IMyEntityChild)
entity_grandchild_mb_cls = get_member_class(IMyEntityGrandchild)
# Entity Parent resource deps should be empty.
self.assert_equal(grph.neighbors(entity_parent_mb_cls), [])
# Entity Child has Grandchild.
self.assert_equal(grph.neighbors(entity_child_mb_cls),
[entity_grandchild_mb_cls])
# Entity Grandchild has Child, but backrefs are excluded by default.
self.assert_equal(grph.neighbors(entity_grandchild_mb_cls),
[])
# Entity has Parent and Child.
self.assert_equal(set(grph.neighbors(entity_mb_cls)),
set([entity_parent_mb_cls, entity_child_mb_cls]))
def _get_node_nested(self, node, attr):
nested_data = node.get(attr.repr_name)
if isinstance(nested_data, dict): # Can also be None or a URL.
json_class = nested_data.pop('__jsonclass__', None)
if not json_class is None:
rel = get_member_class(attr.value_type).relation
if json_class != rel:
raise ValueError('Expected data for %s, got %s.'
% (rel, json_class))
return nested_data
def __collect(self, resource):
ent_cls = get_entity_class(resource)
coll_cls = get_collection_class(resource)
cache = EntityCacheMap()
agg = StagingAggregate(ent_cls, cache)
coll = coll_cls.create_from_aggregate(agg)
coll.add(resource)
return dict([(get_member_class(ent_cls),
coll.get_root_collection(ent_cls))
for ent_cls in cache.keys()])
def __collect(self, resource):
ent_cls = get_entity_class(resource)
coll_cls = get_collection_class(resource)
cache = EntityCacheMap()
agg = StagingAggregate(ent_cls, cache)
coll = coll_cls.create_from_aggregate(agg)
coll.add(resource)
return dict([(get_member_class(ent_cls),
coll.get_root_collection(ent_cls))
for ent_cls in cache.keys()])
def _get_node_nested(self, node, attr):
nested_data = node.get(attr.repr_name)
if isinstance(nested_data, dict): # Can also be None or a URL.
json_class = nested_data.pop('__jsonclass__', None)
if not json_class is None:
rel = get_member_class(attr.value_type).relation
if json_class != rel:
raise ValueError('Expected data for %s, got %s.' %
(rel, json_class))
return nested_data
def _traverse_member(self,
attr_key,
attr,
member_node,
parent_data,
visitor,
index=None):
member_data = OrderedDict()
is_link_node = \
self._is_link_node(member_node, attr) \
or (not index is None and not attr is None and
attr.options.get(WRITE_MEMBERS_AS_LINK_OPTION) is True)
# Ignore links for traversal.
if not is_link_node:
if not attr is None:
node_type = get_member_class(attr.value_type)
else:
node_type = self._get_node_type(member_node)
for mb_attr in self._mapping.attribute_iterator(
node_type, attr_key):
ignore_opt = self.__check_attribute_ignore \
and mb_attr.should_ignore(attr_key)
if ignore_opt:
continue
if mb_attr.kind == RESOURCE_ATTRIBUTE_KINDS.TERMINAL:
# Terminal attribute - extract.
value = self._get_node_terminal(member_node, mb_attr)
if value is None and self.__ignore_none_values:
# We ignore None attribute values when reading
# representations.
continue
member_data[mb_attr] = value
else:
# Nested attribute - traverse.
nested_node = self._get_node_nested(member_node, mb_attr)
if nested_node is None:
# Stop condition - the given data element does not
# contain a nested attribute of the given mapped
# name.
continue
nested_attr_key = attr_key + (mb_attr, )
if ignore_opt is False:
# The offset in the attribute key ensures that
# the defaults for ignoring attributes of the
# nested attribute can be retrieved correctly.
nested_attr_key.offset = len(nested_attr_key)
self._dispatch(nested_attr_key, mb_attr, nested_node,
member_data, visitor)
visitor.visit_member(attr_key,
attr,
member_node,
member_data,
is_link_node,
parent_data,
index=index)
def update_from_data(self, data_element):
"""
Updates this collection from the given data element.
This iterates over the members of this collection and checks if
a member with the same ID exists in the given update data. If yes,
the existing member is updated with the update member; if no,
the member is removed. All data elements in the update data that
have no ID are added as new members. Data elements with an ID that
can not be found in this collection trigger an error.
:param data_element: data element (hierarchical) to create a resource
from
:type data_element: object implementing
`:class:everest.resources.interfaces.IExplicitDataElement`
:raises ValueError: when a data element with an ID that is not present
in this collection is encountered.
"""
attrs = data_element.mapping.get_attribute_map()
id_attr = attrs['id']
update_ids = set()
new_mb_els = []
self_id_map = dict([(self_mb.id, self_mb) for self_mb in iter(self)])
for member_el in data_element.get_members():
# if ILinkedDataElement in provided_by(member_el):
# # Found a link - do not do anything.
# mb_id = member_el.get_id()
# else:
mb_id = member_el.get_terminal(id_attr)
if mb_id is None:
# New data element without an ID - queue for adding.
new_mb_els.append(member_el)
continue
else:
self_mb = self_id_map.get(mb_id)
if not self_mb is None:
# Found an existing member - update.
self_mb.update_from_data(member_el)
else:
# New data element with a new ID. This is suspicious.
raise ValueError('New member data should not provide '
'an ID attribute.')
update_ids.add(mb_id)
# Before adding any new members, check for delete operations.
for self_mb in iter(self):
if not self_mb.id in update_ids:
# Found an existing member ID that was not supplied with
# the update data- remove.
self.remove(self_mb)
# Now, add new members.
mb_cls = get_member_class(self.__class__)
for new_member_el in new_mb_els:
new_member = mb_cls.create_from_data(new_member_el)
self.add(new_member)
def _create_link_data_element(self, attribute, member_node):
if attribute.kind == RESOURCE_ATTRIBUTE_KINDS.MEMBER:
kind = RESOURCE_KINDS.MEMBER
rc_cls = get_member_class(attribute.value_type)
else:
kind = RESOURCE_KINDS.COLLECTION
rc_cls = get_collection_class(attribute.value_type)
return self._mapping.create_linked_data_element(
member_node, kind,
relation=rc_cls.relation,
title=rc_cls.title)
def _set_order(self, order_spec):
if not order_spec is None:
# Translate to entity order expression before passing on to the
# aggregate.
visitor = ResourceToEntityOrderSpecificationVisitor(
get_member_class(self))
order_spec.accept(visitor)
self.__aggregate.order = visitor.expression
else:
self.__aggregate.order = None
self._order_spec = order_spec
def _set_order(self, order_spec):
if not order_spec is None:
# Translate to entity order expression before passing on to the
# aggregate.
visitor = ResourceToEntityOrderSpecificationVisitor(
get_member_class(self))
order_spec.accept(visitor)
self.__aggregate.order = visitor.expression
else:
self.__aggregate.order = None
self._order_spec = order_spec
def _get_node_type(self, node):
relation = node.get('__jsonclass__')
if not relation is None:
tpe = get_resource_class_for_relation(relation)
else:
# In the absence of class hinting, the best we can do is to
# look up the member class for the mapped class. For polymorphic
# types, this will only work if a representer was initialized
# for every derived class separately.
tpe = get_member_class(self._mapping.mapped_class)
return tpe
def _get_node_type(self, node):
relation = node.get('__jsonclass__')
if not relation is None:
tpe = get_resource_class_for_relation(relation)
else:
# In the absence of class hinting, the best we can do is to
# look up the member class for the mapped class. For polymorphic
# types, this will only work if a representer was initialized
# for every derived class separately.
tpe = get_member_class(self._mapping.mapped_class)
return tpe
def __get__(self, resource, resource_class):
if not resource is None:
ent = get_nested_attribute(resource.get_entity(), self.entity_attr)
if not ent is None:
fac = get_member_class(self.attr_type).as_related_member
member = fac(ent, self.make_relationship(resource))
else:
member = None
else:
# class level access
member = self
return member
def __process_link(self, link, attr):
if not self.__is_link(link):
raise ValueError('Value for nested attribute "%s" '
'is not a link.' % attr.repr_name)
if attr.kind == RESOURCE_ATTRIBUTE_KINDS.MEMBER:
kind = RESOURCE_KINDS.MEMBER
rc_cls = get_member_class(attr.value_type)
else:
kind = RESOURCE_KINDS.COLLECTION
rc_cls = get_collection_class(attr.value_type)
return self._mapping.create_linked_data_element(
link, kind, relation=rc_cls.relation, title=rc_cls.title)
def __get__(self, resource, resource_class):
if not resource is None:
ent = get_nested_attribute(resource.get_entity(), self.entity_attr)
if not ent is None:
fac = get_member_class(self.attr_type).as_related_member
member = fac(ent, self.make_relationship(resource))
else:
member = None
else:
# class level access
member = self
return member
def build_resource_graph(resource, dependency_graph=None):
"""
Traverses the graph of resources that is reachable from the given
resource.
If a resource dependency graph is given, links to other resources are
only followed if the dependency graph has an edge connecting the two
corresponding resource classes; otherwise, a default graph is built
which ignores all direct cyclic resource references.
:resource: a :class:`everest.resources.MemberResource` instance.
:returns: a :class:`ResourceGraph` instance representing the graph of
resources reachable from the given resource.
"""
def visit(rc, grph, dep_grph):
mb_cls = type(rc)
attr_map = get_resource_class_attributes(mb_cls)
for attr_name, attr in iteritems_(attr_map):
if is_resource_class_terminal_attribute(mb_cls, attr_name):
continue
# Only follow the resource attribute if the dependency graph
# has an edge here.
child_mb_cls = get_member_class(attr.attr_type)
if not dep_grph.has_edge((mb_cls, child_mb_cls)):
continue
child_rc = getattr(rc, attr_name)
if is_resource_class_collection_attribute(mb_cls, attr_name):
for child_mb in child_rc:
if not grph.has_node(
child_mb): # Ignore cyclic references.
grph.add_node(child_mb)
grph.add_edge((rc, child_mb))
visit(child_mb, grph, dep_grph)
else: # Member.
if not grph.has_node(child_rc): # Ignore cyclic references.
grph.add_node(child_rc)
grph.add_edge((rc, child_rc))
visit(child_rc, grph, dep_grph)
if dependency_graph is None:
dependency_graph = build_resource_dependency_graph(
[get_member_class(resource)])
graph = ResourceGraph()
if provides_member_resource(resource):
rcs = [resource]
else:
rcs = resource
for rc in rcs:
graph.add_node(rc)
visit(rc, graph, dependency_graph)
return graph
def test_filter_specification_visitor(self):
coll = get_root_collection(IMyEntity)
mb_cls = get_member_class(coll)
my_entity = create_entity()
member = coll.create_member(my_entity)
spec_fac = FilterSpecificationFactory()
specs = [
# Terminal access.
spec_fac.create_equal_to('text', self.TEST_TEXT),
# Terminal access with different name in entity.
spec_fac.create_equal_to('text_rc', self.TEST_TEXT),
# Nested member access with different name in entity.
spec_fac.create_equal_to('parent.text_rc', self.TEST_TEXT),
# Nested collection access with different name in entity.
spec_fac.create_equal_to('children.text_rc', self.TEST_TEXT),
# Access with dotted entity name in rc attr declaration.
spec_fac.create_equal_to('parent_text', self.TEST_TEXT),
# Access to member.
spec_fac.create_equal_to('parent',
member.parent.get_entity()),
]
expecteds = [('text', MyEntity.text.__eq__(self.TEST_TEXT)),
('text_ent', MyEntity.text_ent.__eq__(self.TEST_TEXT)),
('parent.text_ent',
MyEntity.parent.has(
MyEntityParent.text_ent.__eq__(
self.TEST_TEXT))),
('children.text_ent',
MyEntity.children.any(
MyEntityChild.text_ent.__eq__(
self.TEST_TEXT))),
('parent.text_ent',
MyEntity.parent.has(
MyEntityParent.text_ent.__eq__(
self.TEST_TEXT))),
('parent',
MyEntity.parent.__eq__(member.parent.get_entity())),
]
for spec, expected in zip(specs, expecteds):
new_attr_name, expr = expected
visitor = ResourceToEntityFilterSpecificationVisitor(mb_cls)
spec.accept(visitor)
new_spec = visitor.expression
self.assert_equal(new_spec.attr_name, new_attr_name)
visitor = SqlFilterSpecificationVisitor(MyEntity)
new_spec.accept(visitor)
self.assert_equal(str(visitor.expression), str(expr))
invalid_spec = spec_fac.create_equal_to('foo', self.TEST_TEXT)
vst = ResourceToEntityFilterSpecificationVisitor(mb_cls)
self.assert_raises(AttributeError, invalid_spec.accept, vst)
def build_resource_graph(resource, dependency_graph=None):
"""
Traverses the graph of resources that is reachable from the given
resource.
If a resource dependency graph is given, links to other resources are
only followed if the dependency graph has an edge connecting the two
corresponding resource classes; otherwise, a default graph is built
which ignores all direct cyclic resource references.
:resource: a :class:`everest.resources.MemberResource` instance.
:returns: a :class:`ResourceGraph` instance representing the graph of
resources reachable from the given resource.
"""
def visit(rc, grph, dep_grph):
mb_cls = type(rc)
attr_map = mb_cls.get_attributes()
for attr_name, attr in attr_map.iteritems():
if mb_cls.is_terminal(attr_name):
continue
# Only follow the resource attribute if the dependency graph
# has an edge here.
child_mb_cls = get_member_class(attr.value_type)
if not dep_grph.has_edge((mb_cls, child_mb_cls)):
continue
child_rc = getattr(rc, attr_name)
if mb_cls.is_collection(attr_name):
for child_mb in child_rc:
if not grph.has_node(child_mb): # Ignore cyclic references.
grph.add_node(child_mb)
grph.add_edge((rc, child_mb))
visit(child_mb, grph, dep_grph)
else: # Member.
if not grph.has_node(child_rc): # Ignore cyclic references.
grph.add_node(child_rc)
grph.add_edge((rc, child_rc))
visit(child_rc, grph, dep_grph)
if dependency_graph is None:
dependency_graph = build_resource_dependency_graph(
[get_member_class(resource)])
graph = ResourceGraph()
if provides_member_resource(resource):
rcs = [resource]
else:
rcs = resource
for rc in rcs:
graph.add_node(rc)
visit(rc, graph, dependency_graph)
return graph
def __process_link(self, link, attr):
if not self.__is_link(link):
raise ValueError('Value for nested attribute "%s" '
'is not a link.' % attr.repr_name)
if attr.kind == RESOURCE_ATTRIBUTE_KINDS.MEMBER:
kind = RESOURCE_KINDS.MEMBER
rc_cls = get_member_class(attr.value_type)
else:
kind = RESOURCE_KINDS.COLLECTION
rc_cls = get_collection_class(attr.value_type)
return self._mapping.create_linked_data_element(link,
kind,
relation=
rc_cls.relation,
title=rc_cls.title)
def __dump_entities(self, entity_class):
cache = self._get_cache(entity_class)
coll_cls = get_collection_class(entity_class)
mb_cls = get_member_class(entity_class)
fn = get_write_collection_path(coll_cls,
self._config['content_type'],
directory=self._config['directory'])
# Wrap the entities in a temporary collection.
coll = create_staging_collection(coll_cls)
for ent in cache.iterator():
coll.add(mb_cls.create_from_entity(ent))
# Open stream for writing and dump the collection.
stream = file(fn, 'w')
with stream:
dump_resource(coll, stream,
content_type=self._config['content_type'])
def __convert_to_entity_attr(self, rc_attr_name):
entity_attr_tokens = []
rc_class = self.__rc_class
for rc_attr_token in rc_attr_name.split('.'):
rc_attr = get_resource_class_attributes(rc_class)[rc_attr_token]
ent_attr_name = rc_attr.entity_name
if ent_attr_name is None:
raise ValueError('Resource attribute "%s" does not have a '
'corresponding entity attribute.'
% rc_attr.name)
if rc_attr.kind != ResourceAttributeKinds.TERMINAL:
# Look up the member class for the specified member or
# collection resource interface.
rc_class = get_member_class(rc_attr.value_type)
entity_attr_tokens.append(ent_attr_name)
return '.'.join(entity_attr_tokens)
def __setup_backref(self):
if not self.backref is None:
# We require the backref to be a resource attribute of the target.
attr_mb_class = get_member_class(self.attr_type)
backref_rc_descr = getattr(attr_mb_class, self.backref, None)
# if backref_rc_descr is None:
# raise ValueError('The "backref" attribute must be a '
# 'resource attribute declared on the '
# 'target of the descriptor.')
if not backref_rc_descr is None:
self.__resource_backref = self.backref
self.__entity_backref = backref_rc_descr.entity_attr
else:
# FIXME: Falling back on the entity attribute here is fishy.
self.__entity_backref = self.backref # pragma: no cover
self.__need_backref_setup = False
def _traverse_member(self, attr_key, attr, member_node, parent_data,
visitor, index=None):
member_data = OrderedDict()
is_link_node = \
self._is_link_node(member_node, attr) \
or (not index is None and not attr is None and
attr.options.get(WRITE_MEMBERS_AS_LINK_OPTION) is True)
# Ignore links for traversal.
if not is_link_node:
if not attr is None:
node_type = get_member_class(attr.value_type)
else:
node_type = self._get_node_type(member_node)
for mb_attr in self._mapping.attribute_iterator(node_type,
attr_key):
ignore_opt = self.__check_attribute_ignore \
and mb_attr.should_ignore(attr_key)
if ignore_opt:
continue
if mb_attr.kind == RESOURCE_ATTRIBUTE_KINDS.TERMINAL:
# Terminal attribute - extract.
value = self._get_node_terminal(member_node, mb_attr)
if value is None and self.__ignore_none_values:
# We ignore None attribute values when reading
# representations.
continue
member_data[mb_attr] = value
else:
# Nested attribute - traverse.
nested_node = self._get_node_nested(member_node,
mb_attr)
if nested_node is None:
# Stop condition - the given data element does not
# contain a nested attribute of the given mapped
# name.
continue
nested_attr_key = attr_key + (mb_attr,)
if ignore_opt is False:
# The offset in the attribute key ensures that
# the defaults for ignoring attributes of the
# nested attribute can be retrieved correctly.
nested_attr_key.offset = len(nested_attr_key)
self._dispatch(nested_attr_key, mb_attr, nested_node,
member_data, visitor)
visitor.visit_member(attr_key, attr, member_node, member_data,
is_link_node, parent_data, index=index)
def test_filter_specification_visitor(self):
coll = get_root_collection(IMyEntity)
mb_cls = get_member_class(coll)
my_entity = create_entity()
member = coll.create_member(my_entity)
spec_fac = FilterSpecificationFactory()
specs = [
# Terminal access.
spec_fac.create_equal_to('text', self.TEST_TEXT),
# Terminal access with different name in entity.
spec_fac.create_equal_to('text_rc', self.TEST_TEXT),
# Nested member access with different name in entity.
spec_fac.create_equal_to('parent.text_rc', self.TEST_TEXT),
# Nested collection access with different name in entity.
spec_fac.create_equal_to('children.text_rc', self.TEST_TEXT),
# Access with dotted entity name in rc attr declaration.
spec_fac.create_equal_to('parent_text', self.TEST_TEXT),
# Access to member.
spec_fac.create_equal_to('parent', member.parent.get_entity()),
]
expecteds = [
('text', MyEntity.text.__eq__(self.TEST_TEXT)),
('text_ent', MyEntity.text_ent.__eq__(self.TEST_TEXT)),
('parent.text_ent',
MyEntity.parent.has(MyEntityParent.text_ent.__eq__(
self.TEST_TEXT))),
('children.text_ent',
MyEntity.children.any(
MyEntityChild.text_ent.__eq__(self.TEST_TEXT))),
('parent.text_ent',
MyEntity.parent.has(MyEntityParent.text_ent.__eq__(
self.TEST_TEXT))),
('parent', MyEntity.parent.__eq__(member.parent.get_entity())),
]
for spec, expected in zip(specs, expecteds):
new_attr_name, expr = expected
visitor = ResourceToEntityFilterSpecificationVisitor(mb_cls)
spec.accept(visitor)
new_spec = visitor.expression
self.assert_equal(new_spec.attr_name, new_attr_name)
visitor = SqlFilterSpecificationVisitor(MyEntity)
new_spec.accept(visitor)
self.assert_equal(str(visitor.expression), str(expr))
invalid_spec = spec_fac.create_equal_to('foo', self.TEST_TEXT)
vst = ResourceToEntityFilterSpecificationVisitor(mb_cls)
self.assert_raises(AttributeError, invalid_spec.accept, vst)
def __dump_entities(self, entity_class):
cache = self._get_cache(entity_class)
coll_cls = get_collection_class(entity_class)
mb_cls = get_member_class(entity_class)
fn = get_write_collection_path(coll_cls,
self._config['content_type'],
directory=self._config['directory'])
# Wrap the entities in a temporary collection.
coll = create_staging_collection(coll_cls)
for ent in cache.iterator():
coll.add(mb_cls.create_from_entity(ent))
# Open stream for writing and dump the collection.
stream = file(fn, 'w')
with stream:
dump_resource(coll,
stream,
content_type=self._config['content_type'])
def visit(mb_cls, grph, path, incl_backrefs):
for attr_name in get_resource_class_attribute_names(mb_cls):
if is_resource_class_terminal_attribute(mb_cls, attr_name):
continue
child_descr = getattr(mb_cls, attr_name)
child_mb_cls = get_member_class(child_descr.attr_type)
# We do not follow cyclic references back to a resource class
# that is last in the path.
if len(path) > 0 and child_mb_cls is path[-1] \
and not incl_backrefs:
continue
if not grph.has_node(child_mb_cls):
grph.add_node(child_mb_cls)
path.append(mb_cls)
visit(child_mb_cls, grph, path, incl_backrefs)
path.pop()
if not grph.has_edge((mb_cls, child_mb_cls)):
grph.add_edge((mb_cls, child_mb_cls))
def visit(mb_cls, grph, path, incl_backrefs):
for attr_name in mb_cls.get_attribute_names():
if mb_cls.is_terminal(attr_name):
continue
child_descr = getattr(mb_cls, attr_name)
child_mb_cls = get_member_class(child_descr.attr_type)
# We do not follow cyclic references back to a resource class
# that is last in the path.
if len(path) > 0 and child_mb_cls is path[-1] \
and not incl_backrefs:
continue
if not grph.has_node(child_mb_cls):
grph.add_node(child_mb_cls)
path.append(mb_cls)
visit(child_mb_cls, grph, path, incl_backrefs)
path.pop()
if not grph.has_edge((mb_cls, child_mb_cls)):
grph.add_edge((mb_cls, child_mb_cls))
def entity_backref(self):
if self.__entity_backref is None:
if self.resource_backref is None:
self.__entity_backref = None
else:
attr_mb_class = get_member_class(self.attr_type)
backref_rc_descr = getattr(attr_mb_class, self.resource_backref, None)
# # We require the backref to be a resource attribute of the
# # target.
# if backref_rc_descr is None:
# raise ValueError('The "backref" attribute must be a '
# 'resource attribute declared on the '
# 'target of the descriptor.')
if not backref_rc_descr is None:
self.__entity_backref = backref_rc_descr.entity_attr
else:
self.__entity_backref = self.resource_backref
return self.__entity_backref
def find_connected_resources(resource, dependency_graph=None):
"""
Collects all resources connected to the given resource and returns a
dictionary mapping member resource classes to new collections containing
the members found.
"""
# Build a resource_graph.
resource_graph = \
build_resource_graph(resource,
dependency_graph=dependency_graph)
entity_map = OrderedDict()
for mb in topological_sorting(resource_graph):
mb_cls = get_member_class(mb)
ents = entity_map.get(mb_cls)
if ents is None:
ents = []
entity_map[mb_cls] = ents
ents.append(mb.get_entity())
return entity_map
def __convert_to_entity_attr(self, rc_attr_name):
entity_attr_tokens = []
rc_class = self.__rc_class
for rc_attr_token in rc_attr_name.split('.'):
rc_attr = get_resource_class_attribute(rc_class, rc_attr_token)
if rc_attr is None:
raise ValueError('%s resource does not have an attribute '
'"%s".' % (rc_class.__name__, rc_attr_name))
ent_attr_name = rc_attr.entity_attr
if ent_attr_name is None:
raise ValueError('Resource attribute "%s" does not have a '
'corresponding entity attribute.'
% rc_attr.resource_attr)
if rc_attr.kind != RESOURCE_ATTRIBUTE_KINDS.TERMINAL:
# Look up the member class for the specified member or
# collection resource interface.
rc_class = get_member_class(rc_attr.attr_type)
entity_attr_tokens.append(ent_attr_name)
return '.'.join(entity_attr_tokens)
def entity_backref(self):
if self.__entity_backref is None:
if self.resource_backref is None:
self.__entity_backref = None
else:
attr_mb_class = get_member_class(self.attr_type)
backref_rc_descr = getattr(attr_mb_class,
self.resource_backref, None)
# # We require the backref to be a resource attribute of the
# # target.
# if backref_rc_descr is None:
# raise ValueError('The "backref" attribute must be a '
# 'resource attribute declared on the '
# 'target of the descriptor.')
if not backref_rc_descr is None:
self.__entity_backref = backref_rc_descr.entity_attr
else:
self.__entity_backref = self.resource_backref
return self.__entity_backref
def find_connected_resources(resource, dependency_graph=None):
"""
Collects all resources connected to the given resource and returns a
dictionary mapping member resource classes to new collections containing
the members found.
"""
# Build a resource_graph.
resource_graph = \
build_resource_graph(resource,
dependency_graph=dependency_graph)
entity_map = OrderedDict()
for mb in topological_sorting(resource_graph):
mb_cls = get_member_class(mb)
ents = entity_map.get(mb_cls)
if ents is None:
ents = []
entity_map[mb_cls] = ents
ents.append(mb.get_entity())
return entity_map
def __convert_to_entity_attr(self, rc_attr_name):
entity_attr_tokens = []
rc_class = self.__rc_class
for rc_attr_token in rc_attr_name.split('.'):
rc_attr = get_resource_class_attribute(rc_class, rc_attr_token)
if rc_attr is None:
raise AttributeError('%s resource does not have an attribute '
'"%s".' %
(rc_class.__name__, rc_attr_name))
ent_attr_name = rc_attr.entity_attr
if ent_attr_name is None:
raise ValueError('Resource attribute "%s" does not have a '
'corresponding entity attribute.' %
rc_attr.resource_attr)
if rc_attr.kind != RESOURCE_ATTRIBUTE_KINDS.TERMINAL:
# Look up the member class for the specified member or
# collection resource interface.
rc_class = get_member_class(rc_attr.attr_type)
entity_attr_tokens.append(ent_attr_name)
return '.'.join(entity_attr_tokens)
def build_resource_dependency_graph(resource_classes,
include_backrefs=False):
"""
Builds a graph of dependencies among the given resource classes.
The dependency graph is a directed graph with member resource classes as
nodes. An edge between two nodes represents a member or collection
attribute.
:param resource_classes: resource classes to determine interdependencies
of.
:type resource_classes: sequence of registered resources.
:param bool include_backrefs: flag indicating if dependencies
introduced by back-references (e.g., a child resource referencing its
parent) should be included in the dependency graph.
"""
def visit(mb_cls, grph, path, incl_backrefs):
for attr_name in mb_cls.get_attribute_names():
if mb_cls.is_terminal(attr_name):
continue
child_descr = getattr(mb_cls, attr_name)
child_mb_cls = get_member_class(child_descr.attr_type)
# We do not follow cyclic references back to a resource class
# that is last in the path.
if len(path) > 0 and child_mb_cls is path[-1] \
and not incl_backrefs:
continue
if not grph.has_node(child_mb_cls):
grph.add_node(child_mb_cls)
path.append(mb_cls)
visit(child_mb_cls, grph, path, incl_backrefs)
path.pop()
if not grph.has_edge((mb_cls, child_mb_cls)):
grph.add_edge((mb_cls, child_mb_cls))
dep_grph = digraph()
for resource_class in resource_classes:
mb_cls = get_member_class(resource_class)
if not dep_grph.has_node(mb_cls):
dep_grph.add_node(mb_cls)
visit(mb_cls, dep_grph, [], include_backrefs)
return dep_grph
def build_resource_dependency_graph(resource_classes, include_backrefs=False):
"""
Builds a graph of dependencies among the given resource classes.
The dependency graph is a directed graph with member resource classes as
nodes. An edge between two nodes represents a member or collection
attribute.
:param resource_classes: resource classes to determine interdependencies
of.
:type resource_classes: sequence of registered resources.
:param bool include_backrefs: flag indicating if dependencies
introduced by back-references (e.g., a child resource referencing its
parent) should be included in the dependency graph.
"""
def visit(mb_cls, grph, path, incl_backrefs):
for attr_name in get_resource_class_attribute_names(mb_cls):
if is_resource_class_terminal_attribute(mb_cls, attr_name):
continue
child_descr = getattr(mb_cls, attr_name)
child_mb_cls = get_member_class(child_descr.attr_type)
# We do not follow cyclic references back to a resource class
# that is last in the path.
if len(path) > 0 and child_mb_cls is path[-1] \
and not incl_backrefs:
continue
if not grph.has_node(child_mb_cls):
grph.add_node(child_mb_cls)
path.append(mb_cls)
visit(child_mb_cls, grph, path, incl_backrefs)
path.pop()
if not grph.has_edge((mb_cls, child_mb_cls)):
grph.add_edge((mb_cls, child_mb_cls))
dep_grph = digraph()
for resource_class in resource_classes:
mb_cls = get_member_class(resource_class)
if not dep_grph.has_node(mb_cls):
dep_grph.add_node(mb_cls)
visit(mb_cls, dep_grph, [], include_backrefs)
return dep_grph
def find_connected_resources(resource, dependency_graph=None):
"""
Collects all resources connected to the given resource and returns a
dictionary mapping member resource classes to new collections containing
the members found.
"""
# Build a resource_graph.
resource_graph = \
build_resource_graph(resource,
dependency_graph=dependency_graph)
# Build an ordered dictionary of collections.
collections = OrderedDict()
for mb in topological_sorting(resource_graph):
mb_cls = get_member_class(mb)
coll = collections.get(mb_cls)
if coll is None:
# Create new collection.
coll = create_staging_collection(mb)
collections[mb_cls] = coll
coll.add(mb)
return collections
def find_connected_resources(resource, dependency_graph=None):
"""
Collects all resources connected to the given resource and returns a
dictionary mapping member resource classes to new collections containing
the members found.
"""
# Build a resource_graph.
resource_graph = \
build_resource_graph(resource,
dependency_graph=dependency_graph)
# Build an ordered dictionary of collections.
collections = OrderedDict()
for mb in topological_sorting(resource_graph):
mb_cls = get_member_class(mb)
coll = collections.get(mb_cls)
if coll is None:
# Create new collection.
coll = create_staging_collection(mb)
collections[mb_cls] = coll
coll.add(mb)
return collections
def __get_q_tag(self, attr):
if not attr.namespace is None:
q_tag = '{%s}%s' % (attr.namespace, attr.repr_name)
else:
if attr.kind == ResourceAttributeKinds.TERMINAL:
xml_ns = \
self.mapping.configuration.get_option(XML_NAMESPACE_OPTION)
else:
if attr.kind == ResourceAttributeKinds.MEMBER:
attr_type = get_member_class(attr.value_type)
elif attr.kind == ResourceAttributeKinds.COLLECTION:
attr_type = get_collection_class(attr.value_type)
mp = self.mapping.mapping_registry.find_mapping(attr_type)
if not mp is None:
xml_ns = mp.configuration.get_option(XML_NAMESPACE_OPTION)
else:
# Not mapped.
xml_ns = None
if not xml_ns is None:
q_tag = '{%s}%s' % (xml_ns, attr.repr_name)
else:
q_tag = attr.repr_name
return q_tag
def visit(rc, grph, dep_grph):
mb_cls = type(rc)
attr_map = mb_cls.get_attributes()
for attr_name, attr in attr_map.iteritems():
if mb_cls.is_terminal(attr_name):
continue
# Only follow the resource attribute if the dependency graph
# has an edge here.
child_mb_cls = get_member_class(attr.value_type)
if not dep_grph.has_edge((mb_cls, child_mb_cls)):
continue
child_rc = getattr(rc, attr_name)
if mb_cls.is_collection(attr_name):
for child_mb in child_rc:
if not grph.has_node(child_mb): # Ignore cyclic references.
grph.add_node(child_mb)
grph.add_edge((rc, child_mb))
visit(child_mb, grph, dep_grph)
else: # Member.
if not grph.has_node(child_rc): # Ignore cyclic references.
grph.add_node(child_rc)
grph.add_edge((rc, child_rc))
visit(child_rc, grph, dep_grph)
def visit(rc, grph, dep_grph):
mb_cls = type(rc)
attr_map = get_resource_class_attributes(mb_cls)
for attr_name, attr in iteritems_(attr_map):
if is_resource_class_terminal_attribute(mb_cls, attr_name):
continue
# Only follow the resource attribute if the dependency graph
# has an edge here.
child_mb_cls = get_member_class(attr.attr_type)
if not dep_grph.has_edge((mb_cls, child_mb_cls)):
continue
child_rc = getattr(rc, attr_name)
if is_resource_class_collection_attribute(mb_cls, attr_name):
for child_mb in child_rc:
if not grph.has_node(
child_mb): # Ignore cyclic references.
grph.add_node(child_mb)
grph.add_edge((rc, child_mb))
visit(child_mb, grph, dep_grph)
else: # Member.
if not grph.has_node(child_rc): # Ignore cyclic references.
grph.add_node(child_rc)
grph.add_edge((rc, child_rc))
visit(child_rc, grph, dep_grph)
def _create_member_data_element(self, attribute, member_node):
if not attribute is None:
mb_cls = get_member_class(attribute.value_type)
else:
mb_cls = get_member_class(self._mapping.mapped_class)
return self._mapping.create_data_element(mapped_class=mb_cls)
def get_collection_name(rc_class):
coll_cls = get_member_class(rc_class)
collection_name = coll_cls.relation.split('/')[-1]
return "%s-collection" % collection_name
def create_from_resource_class(cls, resource_class):
return cls(get_member_class(resource_class), cls.parser_factory)
def _create_member_data_element(self, attribute, member_node):
if not attribute is None:
mb_cls = get_member_class(attribute.value_type)
else:
mb_cls = get_member_class(self._mapping.mapped_class)
return self._mapping.create_data_element(mapped_class=mb_cls)
def get_collection_name(rc_class):
coll_cls = get_member_class(rc_class)
collection_name = coll_cls.relation.split('/')[-1]
return "%s-collection" % collection_name
def __process_attr(self, attribute, attribute_key, data_el, row_data):
try:
attribute_value = row_data.pop(attribute.repr_name)
except KeyError:
attribute_value = None
has_attribute = False
else:
has_attribute = True
# In the first row, we check for extra field names by removing all
# fields we found from the set of all field names.
if self.__is_first_row:
self.__first_row_field_names.discard(attribute.repr_name)
if attribute.should_ignore(attribute_key):
if not attribute_value in (None, ''):
raise ValueError('Value for attribute "%s" found '
'which is configured to be ignored.' %
attribute.repr_name)
else:
if attribute.kind == RESOURCE_ATTRIBUTE_KINDS.TERMINAL:
if not attribute_value is None:
data_el.set_terminal_converted(attribute, attribute_value)
else:
# FIXME: It is peculiar to treat the empty string as None
# here. However, this seems to be the way the csv
# module does it.
if not attribute_value in (None, ''):
link_data_el = \
self.__process_link(attribute_value, attribute)
data_el.set_nested(attribute, link_data_el)
elif not has_attribute and len(row_data) > 0:
nested_attr_key = attribute_key + (attribute, )
# We recursively look for nested resource attributes in
# other fields.
if attribute.kind == RESOURCE_ATTRIBUTE_KINDS.MEMBER:
# For polymorphic classes, this lookup will only work
# if a representer (and a mapping) was initialized
# for each derived class.
nested_rc_cls = get_member_class(attribute.value_type)
else: # collection attribute.
nested_rc_cls = \
get_collection_class(attribute.value_type)
if self.__coll_data is None:
self.__coll_data = \
self.__make_collection_data(nested_rc_cls,
nested_attr_key)
if self.__is_first_row:
self.__row_data_key = \
self.__coll_data.make_key(
self.__first_row_data)
elif self.__coll_data.collection_class \
!= nested_rc_cls:
raise ValueError('All but one nested collection '
'resource attributes have to '
'be provided as links.')
nested_data_el = \
self.__process_row(row_data, nested_rc_cls,
nested_attr_key)
if attribute.kind == RESOURCE_ATTRIBUTE_KINDS.MEMBER:
if len(nested_data_el.data) > 0:
data_el.set_nested(attribute, nested_data_el)
elif len(nested_data_el) > 0:
data_el.set_nested(attribute, nested_data_el)
def __get_member_mapping_and_representer(self):
rc_type = get_member_class(IMyEntity)
return self.__get_mapping_and_representer(rc_type)
