def process_streets(self):
if not self.streets:
self.message = """
Street data not processed.
"""
return False
# retrieve all items
items = self.streets
# init empty lists
added_nodes = []
changed_nodes = []
unmodified_nodes = []
# retrieve a list of local nodes in DB
local_nodes_slug = Node.objects.filter(layer=self.layer).values_list('slug', flat=True)
# init empty list of slug of external nodes that will be needed to perform delete operations
external_nodes_slug = []
deleted_nodes_count = 0
try:
self.status = Status.objects.get(slug=self.config.get('status', None))
except Status.DoesNotExist:
self.status = None
# loop over every parsed item
for item in items:
# retrieve info in auxiliary variables
# readability counts!
pk = item['id']
name = item['properties'].get('LOCATION', '')[0:70]
address = name
slug = slugify(name)
number = 1
original_name = name
needed_different_name = False
while True:
# items might have the same name... so we add a number..
# check in DB too
if slug in external_nodes_slug or Node.objects.filter(slug__exact=slug).exclude(pk=pk).count() > 0:
needed_different_name = True
number = number + 1
name = "%s - %d" % (original_name, number)
slug = slug = slugify(name)
else:
if needed_different_name:
self.verbose('needed a different name for %s, trying "%s"' % (original_name, name))
break
# geometry object
geometry = GEOSGeometry(json.dumps(item["geometry"]))
# default values
added = False
changed = False
try:
# edit existing node
node = Node.objects.get(pk=pk)
except Node.DoesNotExist:
# add a new node
node = Node()
node.id = pk
node.layer = self.layer
node.status = self.status
node.data = {}
added = True
if node.name != name:
node.name = name
changed = True
if node.slug != slug:
node.slug = slug
changed = True
if added is True or node.geometry.equals(geometry) is False:
node.geometry = geometry
changed = True
if node.address != address:
node.address = address
changed = True
# perform save or update only if necessary
if added or changed:
try:
node.full_clean()
node.save()
except ValidationError as e:
raise Exception("%s errors: %s" % (name, e.messages))
if added:
added_nodes.append(node)
self.verbose('new node saved with name "%s"' % node.name)
elif changed:
changed_nodes.append(node)
self.verbose('node "%s" updated' % node.name)
else:
unmodified_nodes.append(node)
self.verbose('node "%s" unmodified' % node.name)
# fill node list container
external_nodes_slug.append(node.slug)
# delete old nodes
for local_node in local_nodes_slug:
# if local node not found in external nodes
if local_node not in external_nodes_slug:
node_name = node.name
# retrieve from DB and delete
node = Node.objects.get(slug=local_node)
node.delete()
# then increment count that will be included in message
deleted_nodes_count = deleted_nodes_count + 1
self.verbose('node "%s" deleted' % node_name)
self.config['last_time_streets_checked'] = str(date.today())
self.layer.external.config = self.config
self.layer.external.save()
# message that will be returned
self.message = """
%s streets added
%s streets changed
%s streets deleted
%s streets unmodified
%s total external records processed
%s total local records for this layer
""" % (
len(added_nodes),
len(changed_nodes),
deleted_nodes_count,
len(unmodified_nodes),
len(items),
Node.objects.filter(layer=self.layer).count()
)
def process_streets(self):
if not self.streets:
self.message = """
Street data not processed.
"""
return False
# retrieve all items
items = self.streets
# init empty lists
added_nodes = []
changed_nodes = []
unmodified_nodes = []
# retrieve a list of local nodes in DB
local_nodes_slug = Node.objects.filter(layer=self.layer).values_list(
'slug', flat=True)
# init empty list of slug of external nodes that will be needed to perform delete operations
external_nodes_slug = []
deleted_nodes_count = 0
try:
self.status = Status.objects.get(
slug=self.config.get('status', None))
except Status.DoesNotExist:
self.status = None
# loop over every parsed item
for item in items:
# retrieve info in auxiliary variables
# readability counts!
pk = item['id']
name = item['properties'].get('LOCATION', '')[0:70]
address = name
slug = slugify(name)
number = 1
original_name = name
needed_different_name = False
while True:
# items might have the same name... so we add a number..
# check in DB too
# TODO: this must be DRYED!!
if slug in external_nodes_slug or Node.objects.filter(
slug__exact=slug).exclude(pk=pk).count() > 0:
needed_different_name = True
number = number + 1
name = "%s - %d" % (original_name, number)
slug = slug = slugify(name)
else:
if needed_different_name:
self.verbose(
'needed a different name for %s, trying "%s"' %
(original_name, name))
break
# geometry object
geometry = GEOSGeometry(json.dumps(item["geometry"]))
# default values
added = False
changed = False
try:
# edit existing node
node = Node.objects.get(pk=pk)
except Node.DoesNotExist:
# add a new node
node = Node()
node.id = pk
node.layer = self.layer
node.status = self.status
node.data = {}
added = True
if node.name != name:
node.name = name
changed = True
if node.slug != slug:
node.slug = slug
changed = True
if added is True or node.geometry.equals(geometry) is False:
node.geometry = geometry
changed = True
if node.address != address:
node.address = address
changed = True
# perform save or update only if necessary
if added or changed:
try:
node.full_clean()
node.save()
except ValidationError as e:
# TODO: are we sure we want to interrupt the execution?
raise Exception("%s errors: %s" % (name, e.messages))
if added:
added_nodes.append(node)
self.verbose('new node saved with name "%s"' % node.name)
elif changed:
changed_nodes.append(node)
self.verbose('node "%s" updated' % node.name)
else:
unmodified_nodes.append(node)
self.verbose('node "%s" unmodified' % node.name)
# fill node list container
external_nodes_slug.append(node.slug)
# delete old nodes
for local_node in local_nodes_slug:
# if local node not found in external nodes
if not local_node in external_nodes_slug:
node_name = node.name
# retrieve from DB and delete
node = Node.objects.get(slug=local_node)
node.delete()
# then increment count that will be included in message
deleted_nodes_count = deleted_nodes_count + 1
self.verbose('node "%s" deleted' % node_name)
self.config['last_time_streets_checked'] = str(date.today())
self.layer.external.config = json.dumps(self.config,
indent=4,
sort_keys=True)
self.layer.external.save()
# message that will be returned
self.message = """
%s streets added
%s streets changed
%s streets deleted
%s streets unmodified
%s total external records processed
%s total local records for this layer
""" % (len(added_nodes), len(changed_nodes), deleted_nodes_count,
len(unmodified_nodes), len(items),
Node.objects.filter(layer=self.layer).count())
