def _generate_lookup(self, data):
self.resolver = PathResolver(parser=self, data=data, parent_template=self.parent)
resolved = self.resolver.resolve()
lookup_table = {}
rows_objects_lookup = {}
for key, (row_num, obj) in resolved.iteritems():
keep_row = self._rows_filter(obj, row_num)
if keep_row:
lookup_table[key] = obj
# +1 for heading row +1 for 0-based to 1-based
rows_objects_lookup[key] = row_num + 2
else:
self._skipped_row(obj, row_num)
self.objects_lookup = lookup_table
self.rows_objects_lookup = rows_objects_lookup
class TemplateParser(BaseParser):
container_model = Template
item_model = TemplateNode
ITEM_ATTRIBUTES = ['name', 'code', 'parent', 'path', 'templates',
'direction', 'description']\
+ translated_fields(TemplateNode)
CONTAINER_ATTRIBUTES = ['name', 'description', 'divisions', 'period_start']
def __init__(self, container_object_dict, rows_filters=None, extends=None, blueprint=None, fill_in_parents=None, interpolate=None):
super(TemplateParser, self).__init__(container_object_dict)
self.parent = extends
self.skipped_rows = []
self.rows_filters = rows_filters or []
self.template_is_different = True
self.parent_cache = None
#TODO: this assumes there's always a base template to inherit from, might need to support parents filling w/o parent template
if extends:
self.fill_in_parents = True if fill_in_parents is None else fill_in_parents
self.interpolate = True if interpolate is None else interpolate
self.template_is_different = False
if not blueprint:
blueprint = extends if extends.is_blueprint else extends.blueprint
#TODO: see if there's a probable case of not finding the blueprint even on the parent template
self.blueprint = blueprint
else:
self.fill_in_parents = False
self.interpolate = False
if self.interpolate:
self.interpolated_lookup = {}
def save(self, dry=False):
saved = super(TemplateParser, self).save(dry=dry)
# `self.dry` was set to `False` so use `dry` instead
if not dry and self.parent_cache is not None:
self._diff_template()
return saved
def _save_items(self):
try:
blueprint = getattr(self, 'blueprint')
except AttributeError:
blueprint = False
if not self.dry:
if blueprint:
# extending the blueprint first
#TODO: need to also extend the parent?
blueprint_nodes = blueprint.nodes.all()
for node in blueprint_nodes:
key = node.path
if key not in self.saved_cache:
self._save_item(obj=node, key=key, is_node=True)
else:
#TODO: this is a stub exception so we don't support this case
# but should probably be removed/replaced later if we support overriding of blueprint nodes
raise Exception('Found a key in saved cache while saving blueprint: %s' % key)
#TODO: diff the nodes in the imported sheet's template against the blueprint's nodes
# fill the parent nodes cache
#TODO: check that parent is NOT blueprint
if self.parent:
self.parent_cache = {}
for node in self.parent.nodes.all():
key = node.path
# make sure we leave out the blueprint's nodes
if key not in self.saved_cache:
self.parent_cache[key] = node
# this also runs in dry run
super(TemplateParser, self)._save_items()
def _save_item(self, obj, key, is_node=False):
inverses = []
item = None
parent = None
# check if we already saved this object and have it in cache
if key in self.saved_cache:
return self.saved_cache[key]
#TODO: this might render the self.fill_in_parents obsolete so check if still needed
#TODO: this might render the self._lookup_node() obsolete so check if still needed
if self.parent_cache is not None:
if key in self.parent_cache:
parent_node = self.parent_cache.pop(key)
different, diff = self._diff_node(obj, parent_node)
if different:
print 'different node'
print obj
# mark this template as different
self.template_is_different = True
#TODO: depending on the difference we can set a backward relation between `obj` and `parent_node`
else:
# Hoorah! we have a match in the parent template's nodes and we'll use it
# we'll skip the rest of the flow with a saved node instead of a raw dict
is_node = True
obj = parent_node
else:
print 'new node'
print obj
# mark this template as different
self.template_is_different = True
# if it's a saved node it's already well connected
if not is_node:
# if inheriting another template then look up this node there
if self.parent:
path = obj['path']
#TODO: this is probably redundant now
item = self._lookup_node(path=path, key=key)
if item:
# in case we found the item, cache the saved node
return self._save_item(item, key, is_node=True)
elif item is False:
# there's an ambiguity here that was caught
#TODO: handle case of ambiguity here
pass
else:
# here we could note there's a delta between the templates
pass
if 'inverse' in obj:
inverses = self._save_inverses(obj, key)
if 'parent' in obj:
parent = self._save_parent(obj, key)
if parent:
obj['parent'] = parent
else:
# clean up parent
del obj['parent']
# if inheriting another template AND got no parent AND didn't find
# the item in the parent template then it has to be an error
# unless we allow new parentless dangling nodes
if self.dry and self.parent and not item and not parent:
self.throw(
ParentNodeNotFoundError(
row=self.rows_objects_lookup.get(key, None),
columns=['code'],
values=[obj['code']]
)
)
self._set_direction(obj, parent)
item = self._create_item(obj, key)
if len(inverses) and not self.dry:
for inverse in inverses:
item.inverse.add(inverse)
else:
if obj.parent:
self._save_item(obj.parent, obj.parent.path, is_node=True)
item = obj
self._add_to_container(item, key)
# cache the saved object
self.saved_cache[key] = item
return item
def _save_inverses(self, obj, key):
inverses = []
inverse_codes = obj.get('inverse', None)
scopes = obj.get('inversescope', None)
if inverse_codes:
inverse_codes = inverse_codes.split(self.ITEM_SEPARATOR)
else:
inverse_codes = []
if len(inverse_codes):
scopes_length = 0
if scopes:
scopes = scopes.split(self.ITEM_SEPARATOR)
scopes_length = len(scopes)
if not scopes_length:
# clean up
del obj['inversescope']
for i, inv_code in enumerate(inverse_codes):
if i < scopes_length:
inverse_key, inverse_obj = self._lookup_object(code=inv_code, scope=scopes[i])
else:
inverse_key, inverse_obj = self._lookup_object(code=inv_code)
if not inverse_key or not inverse_obj:
if self.dry:
self.throw(
DataSyntaxError(
row=self.rows_objects_lookup.get(key, None),
columns=('inverse', 'inversescope'),
values=(obj['inverse'], obj.get('inversescope', None))
)
)
# create a dummy node as the parent
inverse = self.item_model(code=obj['inverse'], name='dummy')
else:
raise ParsingError('Could not locate an inverse, \
probably you have a syntax error: inverse = %s' % obj['inverse'])
else:
if inverse_key in self.saved_cache:
inverse = self.saved_cache[inverse_key]
else:
inverse = self._save_item(inverse_obj, inverse_key)
inverses.append(inverse)
if self.dry:
if inverse.direction and inverse.direction == obj['direction']:
self.throw(
NodeDirectionError(
rows=(
self.rows_objects_lookup.get(key, None),
self.rows_objects_lookup.get(inverse_key, None)
)
)
)
else:
# clean inverse
if 'inversescope' in obj:
del obj['inversescope']
del obj['inverse']
return inverses
def _save_parent(self, obj, key):
if obj['parent']:
if 'parentscope' in obj:
scope = obj['parentscope']
if not scope:
# clean parentscope
del obj['parentscope']
else:
scope = ''
# generate a lookup key for the parent
route = [unicode(obj['parent'])]
if scope:
route += scope.split(self.PATH_DELIMITER)
parent_path = self.PATH_DELIMITER.join(route)
# first look it up in the input from the source file
parent_key, parent = self._lookup_object(key=parent_path)
if not parent or not parent_key:
if self.fill_in_parents:
# look up the node in the parent template
#TODO: we can probably now just check in self.parent_cache instead, check it later
parent = self._lookup_node(path=parent_path, route=route, key=key)
if parent:
# save the node as if it was another object in the lookup
return self._save_item(parent, parent_path, is_node=True)
elif parent is False:
# there's an ambiguity here that was caught
if self.dry:
# create a dummy node as the parent
return self._create_dummy_parent(obj)
else:
_raise_parent_not_found(obj['code'], obj['parent'], scope)
elif self.interpolate and scope:
parent = self._interpolate(route=route, key=key)
if parent:
return parent
else:
return self._create_dummy_parent(obj)
else:
if self.dry:
self.throw(
ParentScopeError(
row=self.rows_objects_lookup.get(key, None)
)
)
return self._create_dummy_parent(obj)
else:
_raise_parent_not_found(obj['code'], obj['parent'], scope)
else:
if self.dry:
self.throw(
ParentNodeNotFoundError(
row=self.rows_objects_lookup.get(key, None)
)
)
# create a dummy node as the parent
return self._create_dummy_parent(obj)
else:
_raise_parent_not_found(obj['code'], obj['parent'], scope)
else:
if parent_key in self.saved_cache:
parent = self.saved_cache[parent_key]
else:
parent = self._save_item(parent, parent_key)
return parent
else:
if 'parentscope' in obj:
# clean parentscope
del obj['parentscope']
return None
def _create_dummy_parent(self, obj):
code = obj['parent']
attrs = {
'code': code,
'name': 'dummy',
}
direction = obj.get('direction', None)
scope = obj.get('parentscope', None)
path = code
attrs['direction'] = direction or 'REVENUE'
if scope:
path = self.PATH_DELIMITER.join((code, scope))
attrs['path'] = path
obj['parent'] = self.item_model(**attrs)
return obj['parent']
def _interpolate(self, route, key=None):
route_copy = list(route)
ancestors_routes = [route_copy]
# first we'll try finding the other end
while len(route):
route.pop(0)
#TODO: check whether we can now just check if it's in self.parent_cache instead
ancestor = self._lookup_node(route=route, key=key)
if ancestor:
# connected!
self._save_item(ancestor, self.PATH_DELIMITER.join(route), is_node=True)
break
ancestors_routes.append(list(route))
else:
if self.dry:
self.throw(
PathInterpolationError(
row=self.rows_objects_lookup[key]
)
)
return None
else:
raise ParsingError(_('Interpolation failed, no ancestor found for path: %s') %
self.PATH_DELIMITER.join(route_copy))
# we managed to find an ancestor and saved it
# now to connect the dots together,
# i.e. create a node per route in `ancestors_routes`
while len(ancestors_routes):
ancestor_route = ancestors_routes.pop()
ancestor = self._insert_item(ancestor_route, ancestor)
return ancestor
def _insert_item(self, route, parent, row_num=None):
"""
Creates a new object representing an implicit parent node,
adds it to the objects_lookup dictionary and saves it as an item.
Takes a list `route` for determining the object's key and the item's
code and parentscope.
Also takes a node instance `parent` to set the item's parent and direction.
Returns the product of `self._save_item()`.
"""
key = self.PATH_DELIMITER.join(route)
code = route[0]
parent_scope = ''
if len(route) > 2:
parent_scope = self.PATH_DELIMITER.join(route[2:])
obj = {
'parent': parent,
'code': code,
'parentscope': parent_scope,
'direction': parent.direction,
'name': parent.name,
'path': key
}
_key, _obj = self._lookup_object(key=key)
if not _key:
# self.objects_lookup[key] = obj
if row_num is not None:
self.rows_objects_lookup[key] = row_num
else:
raise ParsingError(_('You are trying to insert an item that already exists at: %s') % key)
return self._save_item(obj, key)
def _set_direction(self, obj, parent):
if parent and ('direction' not in obj or not obj['direction']):
obj['direction'] = parent.direction
def _create_item(self, obj, key):
# work with a clone so we always keep the source input
return super(TemplateParser, self)._create_item(obj.copy(), key)
def _add_to_container(self, item, key):
if not self.dry:
TemplateNodeRelation.objects.get_or_create(
template=self.container_object,
node=item
)
def _create_container(self, container_dict=None, exclude=None):
data = (container_dict or self.container_object_dict).copy()
if 'name' not in data:
self._generate_container_name(container_dict=data)
divisions = data.pop('divisions') if 'divisions' in data else []
super(TemplateParser, self)._create_container(container_dict=data, exclude=exclude)
for division in divisions:
try:
division = Division.objects.get(slug=division)
if not self.dry:
self.container_object.divisions.add(division)
except Division.DoesNotExist:
self.throw(
MetaParsingError(reason=_('Division with slug %s does not exist') % division))
def _generate_lookup(self, data):
self.resolver = PathResolver(parser=self, data=data, parent_template=self.parent)
resolved = self.resolver.resolve()
lookup_table = {}
rows_objects_lookup = {}
for key, (row_num, obj) in resolved.iteritems():
keep_row = self._rows_filter(obj, row_num)
if keep_row:
lookup_table[key] = obj
# +1 for heading row +1 for 0-based to 1-based
rows_objects_lookup[key] = row_num + 2
else:
self._skipped_row(obj, row_num)
self.objects_lookup = lookup_table
self.rows_objects_lookup = rows_objects_lookup
def _rows_filter(self, obj, row_num=None):
for filter in self.rows_filters:
if not filter(obj, row_num):
return False
return True
def _skipped_row(self, obj, row_num):
# +1 for heading row +1 for 0-based to 1-based
self.skipped_rows.append((row_num + 2, obj))
def _lookup_object(self, code=None, parent='', scope='', key=None):
if code:
key = ''
if code in self.objects_lookup:
key = code
elif parent or scope:
if not parent:
parent = scope.split(self.PATH_DELIMITER)[0]
key = self.PATH_DELIMITER.join((code, parent))
if key not in self.objects_lookup:
key = self.PATH_DELIMITER.join((code, scope))
if key:
if key in self.objects_lookup:
return key, self.objects_lookup[key]
return None, None
def _lookup_node(self, route=None, path=None, key=None):
if path is None:
if route and len(route):
path = self.PATH_DELIMITER.join(route)
else:
raise ParsingError(
__('Did not get neither route nor path for looking up node, for row: %s') %
self.rows_objects_lookup.get(key, None)
)
try:
# there can be one or none
return self.parent.nodes.get(path=path)
except self.item_model.DoesNotExist as e:
# none found
ancestors_matches = self.blueprint.nodes.filter(path=path)
if len(ancestors_matches):
return ancestors_matches[0]
try:
if route is None and path:
route = path.split(self.PATH_DELIMITER)
_filter = {
'code': route[0]
}
if len(route) > 1:
_filter['parent__code'] = route[1]
return self.parent.nodes.get(**_filter)
except self.item_model.DoesNotExist as e:
return None
except self.item_model.MultipleObjectsReturned as e:
if self.dry:
self.throw(
ParentScopeError(
row=self.rows_objects_lookup.get(key, None)
)
)
return False
else:
raise e
except self.item_model.MultipleObjectsReturned as e:
#TODO: this indicates that the data is corrupted, need to handle better
if self.dry:
# more then one probably means there's PARENT_SCOPE missing
self.throw(
ParentScopeError(
row=self.rows_objects_lookup.get(key, None)
)
)
return False
else:
raise e
def _generate_container_name(self, container_dict):
name = _('Template of %(entity)s since %(year)s')
year = str(datetime.strptime(container_dict['period_start'], '%Y-%m-%d').year)
if 'entity' in container_dict and container_dict['entity']:
entity = container_dict['entity']
name = name % {'entity': entity.name, 'year': year}
del container_dict['entity']
elif 'divisions' in container_dict:
name = _('Official Template since %(year)') % {'year': year}
else:
self.throw(
DataValidationError(
reasons={'template': [__('Trying to create a template but no entity nor divisions found.')]}
)
)
container_dict['name'] = name
def _diff_node(self, obj, parent_node):
diff = []
different = False
if isinstance(obj, dict):
name = obj.get('name')
else:
name = obj.name
if name != parent_node.name:
different = True
diff.append('name')
return different, diff
def _diff_template(self):
print 'diffing'
#TODO: consider not declaring this tempalte as different if there are nodes left in parent cache
# and instead, perhaps instantiate these also - maybe without an item? Or null amounts?
#TODO: optimize process so container is only created at the end if it's different from its parent
if not self.template_is_different and not len(self.parent_cache):
print 'same'
# this should also delete all TemplateNodeRelations to this template
self.container_object.delete()
# since all nodes already relate to the parent no need to do anything
self.container_object = self.parent
else:
print 'nodes left in parent %s' % len(self.parent_cache)
for key in self.parent_cache:
print key
