def test_node_count_change(self):
""" when a new node is added nodes count should be updated """
new_node = Node(user_id=4, name='new node', slug='new_node', layer_id=1, coords="POINT (41.9720419277 12.5822391919)")
new_node.save()
self.assertEqual(new_node.user.stats.potential_nodes, 2, "Potential nodes increment count failed")
new_node.status = NODE_STATUS.get('active')
new_node.save()
self.assertEqual(new_node.user.stats.active_nodes, 4, "Active nodes increment count failed")
self.assertEqual(new_node.user.stats.potential_nodes, 1, "Potential nodes decrement count failed")
#new_node.is_hotspot = True
#new_node.save()
#self.assertEqual(new_node.user.stats.hotspots, 2, "Hotspot increment count failed")
new_node.delete()
user = User.objects.get(username='******')
self.assertEqual(user.stats.active_nodes, 3, "Active nodes decrement count failed")
def test_node_count_change(self):
""" when a new node is added nodes count should be updated """
new_node = Node(user_id=4,
name='new node',
slug='new_node',
layer_id=1,
coords="POINT (41.9720419277 12.5822391919)")
new_node.save()
self.assertEqual(new_node.user.stats.potential_nodes, 2,
"Potential nodes increment count failed")
new_node.status = NODE_STATUS.get('active')
new_node.save()
self.assertEqual(new_node.user.stats.active_nodes, 4,
"Active nodes increment count failed")
self.assertEqual(new_node.user.stats.potential_nodes, 1,
"Potential nodes decrement count failed")
#new_node.is_hotspot = True
#new_node.save()
#self.assertEqual(new_node.user.stats.hotspots, 2, "Hotspot increment count failed")
new_node.delete()
user = User.objects.get(username='******')
self.assertEqual(user.stats.active_nodes, 3,
"Active nodes decrement count failed")
def save(self):
""" synchronize DB """
# retrieve all items
items = self.parsed_data.getElementsByTagName('AccessPoint')
# init empty lists
added_nodes = []
changed_nodes = []
unmodified_nodes = []
# retrieve a list of local nodes in DB
local_nodes_slug = Node.objects.filter(layer=self.layer).values_list(
'slug', flat=True)
# init empty list of slug of external nodes that will be needed to perform delete operations
external_nodes_slug = []
deleted_nodes_count = 0
try:
self.status = Status.objects.get(
slug=self.config.get('status', None))
except Status.DoesNotExist:
self.status = None
# loop over every parsed item
for item in items:
# retrieve info in auxiliary variables
# readability counts!
name = self.get_text(item, 'Denominazione')[0:70]
slug = slugify(name)
number = 1
original_name = name
needed_different_name = False
while True:
# items might have the same name... so we add a number..
if slug in external_nodes_slug:
needed_different_name = True
number = number + 1
name = "%s - %d" % (original_name, number)
slug = slug = slugify(name)
else:
if needed_different_name:
self.verbose(
'needed a different name for %s, trying "%s"' %
(original_name, name))
break
lat = self.get_text(item, 'Latitudine')
lng = self.get_text(item, 'longitudine')
description = 'Indirizzo: %s, %s; Tipologia: %s' % (self.get_text(
item, 'Indirizzo'), self.get_text(
item, 'Comune'), self.get_text(item, 'Tipologia'))
address = '%s, %s' % (self.get_text(
item, 'Indirizzo'), self.get_text(item, 'Comune'))
# point object
point = Point(float(lng), float(lat))
# default values
added = False
changed = False
try:
# edit existing node
node = Node.objects.get(slug=slug)
except Node.DoesNotExist:
# add a new node
node = Node()
node.layer = self.layer
node.status = self.status
added = True
if node.name != name:
node.name = name
changed = True
if node.slug != slug:
node.slug = slug
changed = True
if added is True or node.geometry.equals(point) is False:
node.geometry = point
changed = True
if node.description != description:
node.description = description
changed = True
if node.address != address:
node.address = address # complete address
node.data = {
'address': self.get_text(item, 'Indirizzo'),
'city': self.get_text(item, 'Comune'),
'province': 'Roma',
'country': 'Italia'
}
changed = True
# perform save or update only if necessary
if added or changed:
try:
node.full_clean()
node.save()
except ValidationError as e:
# TODO: are we sure we want to interrupt the execution?
raise Exception("%s errors: %s" % (name, e.messages))
if added:
added_nodes.append(node)
self.verbose('new node saved with name "%s"' % node.name)
elif changed:
changed_nodes.append(node)
self.verbose('node "%s" updated' % node.name)
else:
unmodified_nodes.append(node)
self.verbose('node "%s" unmodified' % node.name)
# fill node list container
external_nodes_slug.append(node.slug)
# delete old nodes
for local_node in local_nodes_slug:
# if local node not found in external nodes
if not local_node in external_nodes_slug:
node_name = node.name
# retrieve from DB and delete
node = Node.objects.get(slug=local_node)
node.delete()
# then increment count that will be included in message
deleted_nodes_count = deleted_nodes_count + 1
self.verbose('node "%s" deleted' % node_name)
# message that will be returned
self.message = """
%s nodes added
%s nodes changed
%s nodes deleted
%s nodes unmodified
%s total external records processed
%s total local nodes for this layer
""" % (len(added_nodes), len(changed_nodes), deleted_nodes_count,
len(unmodified_nodes), len(items),
Node.objects.filter(layer=self.layer).count())
def process_borders(self,borders):
if not borders:
self.message = """
Borders data not processed.
"""
return False
# retrieve all items
items = borders
# init empty lists
added_nodes = []
changed_nodes = []
unmodified_nodes = []
# retrieve a list of local nodes in DB
local_nodes_slug = Node.objects.filter(layer=self.layer).values_list('slug', flat=True)
# init empty list of slug of external nodes that will be needed to perform delete operations
external_nodes_slug = []
deleted_nodes_count = 0
try:
self.status = Status.objects.get(slug=self.config.get('status', None))
except Status.DoesNotExist:
self.status = None
# loop over every parsed item
for item in items:
# retrieve info in auxiliary variables
# readability counts!
name = item['properties'].get('name', '')[0:70]
address = name
slug = slugify(name)
#print(slug)
number = 1
original_name = name
needed_different_name = False
while True:
# items might have the same name... so we add a number..
if slug in external_nodes_slug:
needed_different_name = True
number = number + 1
name = "%s - %d" % (original_name, number)
slug = slug = slugify(name)
else:
if needed_different_name:
self.verbose('needed a different name for %s, trying "%s"' % (original_name, name))
break
# geometry object
geometry = GEOSGeometry(json.dumps(item["geometry"]))
# default values
added = False
changed = False
try:
# edit existing node
node = Node.objects.get(slug=slug)
except Node.DoesNotExist:
# add a new node
node = Node()
node.layer = self.layer
node.status = self.status
node.data = {}
added = True
if node.name != name:
node.name = name
changed = True
if node.slug != slug:
node.slug = slug
changed = True
if added is True or node.geometry.equals(geometry) is False:
node.geometry = geometry
changed = True
if node.address != address:
node.address = address
changed = True
# perform save or update only if necessary
if added or changed:
try:
node.full_clean()
node.save()
except ValidationError as e:
# TODO: are we sure we want to interrupt the execution?
raise Exception("%s import errors: %s" % (name, e.messages))
if added:
added_nodes.append(node)
self.verbose('new node saved with name "%s"' % node.name)
elif changed:
changed_nodes.append(node)
self.verbose('node "%s" updated' % node.name)
else:
unmodified_nodes.append(node)
self.verbose('node "%s" unmodified' % node.name)
# fill node list container
external_nodes_slug.append(node.slug)
# delete old nodes
for local_node in local_nodes_slug:
# if local node not found in external nodes
if not local_node in external_nodes_slug:
node_name = node.name
# retrieve from DB and delete
node = Node.objects.get(slug=local_node)
node.delete()
# then increment count that will be included in message
deleted_nodes_count = deleted_nodes_count + 1
self.verbose('node "%s" deleted' % node_name)
self.layer.external.config = json.dumps(self.config, indent=4, sort_keys=True)
self.layer.external.save()
# message that will be returned
self.message = """
%s node added
%s node changed
%s node deleted
%s node unmodified
%s total external records processed
%s total local records for this layer
""" % (
len(added_nodes),
len(changed_nodes),
deleted_nodes_count,
len(unmodified_nodes),
len(items),
Node.objects.filter(layer=self.layer).count()
)
def save(self):
""" synchronize DB """
# retrieve all items
items = self.parsed_data.getElementsByTagName('AccessPoint')
# init empty lists
added_nodes = []
changed_nodes = []
unmodified_nodes = []
# retrieve a list of local nodes in DB
local_nodes_slug = Node.objects.filter(layer=self.layer).values_list('slug', flat=True)
# init empty list of slug of external nodes that will be needed to perform delete operations
external_nodes_slug = []
deleted_nodes_count = 0
try:
self.status = Status.objects.get(slug=self.config.get('status', None))
except Status.DoesNotExist:
self.status = None
# loop over every parsed item
for item in items:
# retrieve info in auxiliary variables
# readability counts!
name = self.get_text(item, 'Denominazione')[0:70]
slug = slugify(name)
number = 1
original_name = name
needed_different_name = False
while True:
# items might have the same name... so we add a number..
if slug in external_nodes_slug:
needed_different_name = True
number = number + 1
name = "%s - %d" % (original_name, number)
slug = slug = slugify(name)
else:
if needed_different_name:
self.verbose('needed a different name for %s, trying "%s"' % (original_name, name))
break
lat = self.get_text(item, 'Latitudine')
lng = self.get_text(item, 'longitudine')
description = 'Indirizzo: %s, %s; Tipologia: %s' % (
self.get_text(item, 'Indirizzo'),
self.get_text(item, 'Comune'),
self.get_text(item, 'Tipologia')
)
address = '%s, %s' % (
self.get_text(item, 'Indirizzo'),
self.get_text(item, 'Comune')
)
# point object
point = Point(float(lng), float(lat))
# default values
added = False
changed = False
try:
# edit existing node
node = Node.objects.get(slug=slug)
except Node.DoesNotExist:
# add a new node
node = Node()
node.layer = self.layer
node.status = self.status
added = True
if node.name != name:
node.name = name
changed = True
if node.slug != slug:
node.slug = slug
changed = True
if added is True or node.geometry.equals(point) is False:
node.geometry = point
changed = True
if node.description != description:
node.description = description
changed = True
if node.address != address:
node.address = address # complete address
node.data = {
'address': self.get_text(item, 'Indirizzo'),
'city': self.get_text(item, 'Comune'),
'province': 'Roma',
'country': 'Italia'
}
changed = True
# perform save or update only if necessary
if added or changed:
try:
node.full_clean()
node.save()
except ValidationError as e:
# TODO: are we sure we want to interrupt the execution?
raise Exception("%s errors: %s" % (name, e.messages))
if added:
added_nodes.append(node)
self.verbose('new node saved with name "%s"' % node.name)
elif changed:
changed_nodes.append(node)
self.verbose('node "%s" updated' % node.name)
else:
unmodified_nodes.append(node)
self.verbose('node "%s" unmodified' % node.name)
# fill node list container
external_nodes_slug.append(node.slug)
# delete old nodes
for local_node in local_nodes_slug:
# if local node not found in external nodes
if not local_node in external_nodes_slug:
node_name = node.name
# retrieve from DB and delete
node = Node.objects.get(slug=local_node)
node.delete()
# then increment count that will be included in message
deleted_nodes_count = deleted_nodes_count + 1
self.verbose('node "%s" deleted' % node_name)
# message that will be returned
self.message = """
%s nodes added
%s nodes changed
%s nodes deleted
%s nodes unmodified
%s total external records processed
%s total local nodes for this layer
""" % (
len(added_nodes),
len(changed_nodes),
deleted_nodes_count,
len(unmodified_nodes),
len(items),
Node.objects.filter(layer=self.layer).count()
)
def process_streets(self):
if not self.streets:
self.message = """
Street data not processed.
"""
return False
# retrieve all items
items = self.streets
# init empty lists
added_nodes = []
changed_nodes = []
unmodified_nodes = []
# retrieve a list of local nodes in DB
local_nodes_slug = Node.objects.filter(layer=self.layer).values_list(
'slug', flat=True)
# init empty list of slug of external nodes that will be needed to perform delete operations
external_nodes_slug = []
deleted_nodes_count = 0
try:
self.status = Status.objects.get(
slug=self.config.get('status', None))
except Status.DoesNotExist:
self.status = None
# loop over every parsed item
for item in items:
# retrieve info in auxiliary variables
# readability counts!
pk = item['id']
name = item['properties'].get('LOCATION', '')[0:70]
address = name
slug = slugify(name)
number = 1
original_name = name
needed_different_name = False
while True:
# items might have the same name... so we add a number..
# check in DB too
# TODO: this must be DRYED!!
if slug in external_nodes_slug or Node.objects.filter(
slug__exact=slug).exclude(pk=pk).count() > 0:
needed_different_name = True
number = number + 1
name = "%s - %d" % (original_name, number)
slug = slug = slugify(name)
else:
if needed_different_name:
self.verbose(
'needed a different name for %s, trying "%s"' %
(original_name, name))
break
# geometry object
geometry = GEOSGeometry(json.dumps(item["geometry"]))
# default values
added = False
changed = False
try:
# edit existing node
node = Node.objects.get(pk=pk)
except Node.DoesNotExist:
# add a new node
node = Node()
node.id = pk
node.layer = self.layer
node.status = self.status
node.data = {}
added = True
if node.name != name:
node.name = name
changed = True
if node.slug != slug:
node.slug = slug
changed = True
if added is True or node.geometry.equals(geometry) is False:
node.geometry = geometry
changed = True
if node.address != address:
node.address = address
changed = True
# perform save or update only if necessary
if added or changed:
try:
node.full_clean()
node.save()
except ValidationError as e:
# TODO: are we sure we want to interrupt the execution?
raise Exception("%s errors: %s" % (name, e.messages))
if added:
added_nodes.append(node)
self.verbose('new node saved with name "%s"' % node.name)
elif changed:
changed_nodes.append(node)
self.verbose('node "%s" updated' % node.name)
else:
unmodified_nodes.append(node)
self.verbose('node "%s" unmodified' % node.name)
# fill node list container
external_nodes_slug.append(node.slug)
# delete old nodes
for local_node in local_nodes_slug:
# if local node not found in external nodes
if not local_node in external_nodes_slug:
node_name = node.name
# retrieve from DB and delete
node = Node.objects.get(slug=local_node)
node.delete()
# then increment count that will be included in message
deleted_nodes_count = deleted_nodes_count + 1
self.verbose('node "%s" deleted' % node_name)
self.config['last_time_streets_checked'] = str(date.today())
self.layer.external.config = json.dumps(self.config,
indent=4,
sort_keys=True)
self.layer.external.save()
# message that will be returned
self.message = """
%s streets added
%s streets changed
%s streets deleted
%s streets unmodified
%s total external records processed
%s total local records for this layer
""" % (len(added_nodes), len(changed_nodes), deleted_nodes_count,
len(unmodified_nodes), len(items),
Node.objects.filter(layer=self.layer).count())
