def test_layer_nodes_minimum_distance(self):
""" ensure minimum distance settings works as expected """
layer = Layer.objects.get(slug='rome')
node = layer.node_set.all()[0]
# creating node with same coordinates should not be an issue
new_node = Node(**{
'name': 'new_node',
'slug': 'new_node',
'layer': layer,
'geometry': node.geometry
})
new_node.full_clean()
new_node.save()
layer.nodes_minimum_distance = 100
layer.save()
try:
new_node.full_clean()
except ValidationError as e:
self.assertIn(_('Distance between nodes cannot be less than %s meters') % layer.nodes_minimum_distance, e.messages)
return
self.assertTrue(False, 'validation not working as expected')
def test_layer_area_contains_node_validation(self):
""" ensure area validation works as expected """
layer = Layer.objects.get(slug='rome')
layer.area = GEOSGeometry('POLYGON ((12.19 41.92, 12.58 42.17, 12.82 41.86, 12.43 41.64, 12.43 41.65, 12.19 41.92))')
layer.full_clean()
layer.save()
# creating node with same coordinates should not be an issue
new_node = Node(**{
'name': 'new_node',
'slug': 'new_node',
'layer': layer,
'geometry': 'POINT (50.0 50.0)'
})
try:
new_node.full_clean()
except ValidationError as e:
self.assertIn(_('Node must be inside layer area'), e.messages)
else:
self.fail('validation not working as expected')
# if area is a point the contains check won't be done
layer.area = GEOSGeometry('POINT (30.0 30.0)')
layer.full_clean()
layer.save()
new_node.full_clean()
def test_layer_new_nodes_allowed(self):
layer = Layer.objects.get(pk=1)
layer.new_nodes_allowed = False
layer.full_clean()
layer.save()
# ensure changing an existing node works
node = layer.node_set.all()[0]
node.name = 'changed'
node.full_clean()
node.save()
# re-get from DB, just to be sure
node = Node.objects.get(pk=node.pk)
self.assertEqual(node.name, 'changed')
# ensure new node cannot be added
node = Node(**{
'name': 'test new node',
'slug': 'test-new-node',
'layer': layer,
'geometry': 'POINT (10.4389188797003565 43.7200020000987328)'
})
with self.assertRaises(ValidationError):
node.full_clean()
try:
node.full_clean()
assert()
except ValidationError as e:
self.assertIn(_('New nodes are not allowed for this layer'), e.messages)
def test_nodes_deletion(self):
for user in User.objects.all():
user.delete()
settings.DEFAULT_LAYER = 5
management.call_command('import_old_nodeshot', noinput=True)
node = Node.objects.get(pk=1)
self.assertIn('imported', node.data)
# --- delete oldnode --- #
OldNode.objects.get(pk=node.pk).delete()
# --- ensure node and user are deleted --- #
management.call_command('import_old_nodeshot', noinput=True)
self.assertEqual(Node.objects.filter(pk=node.pk).count(), 0)
self.assertEqual(User.objects.filter(email="*****@*****.**").count(), 0)
# --- add not imported node --- #
added_node = Node(name='added',
slug='added',
layer_id=5,
description='added',
geometry='POINT (12 42)')
added_node.full_clean()
added_node.save()
# --- ensure added_node is not deleted --- #
management.call_command('import_old_nodeshot', noinput=True)
self.assertEqual(Node.objects.filter(pk=added_node.pk).count(), 1)
def get_nodes(self, class_name, params):
""" get nodes """
# determine if response is going to be JSON or GeoJSON
if 'geojson' in class_name.lower():
response_format = 'geojson'
SerializerClass = OpenLaborGeoSerializer
else:
response_format = 'json'
SerializerClass = OpenLaborSerializer
layer_name = self.layer.name
cache_key = 'layer_%s_nodes.%s' % (self.layer.id, response_format)
serialized_nodes = cache.get(cache_key, False)
if serialized_nodes is False:
try:
response = requests.get(self.get_url,
verify=self.config.get(
'verify_SSL', True))
except requests.exceptions.ConnectionError as e:
return {
'error': _('external layer not reachable'),
'exception': list(e.message)
}
try:
response.data = json.loads(response.content)
except json.scanner.JSONDecodeError as e:
return {
'error':
_('external layer is experiencing some issues because it returned invalid data'
),
'exception':
list(e)
}
nodes = []
# loop over all the entries and convert to nodeshot format
for job in response.data:
# skip records which do not have geographic information
if not job.get('latitude', False) or not job.get(
'longitude', False):
continue
# convert response in nodeshot format
node_dictionary = self.to_nodeshot(job)
# create Node model instance (needed for rest_framework serializer)
node = Node(**node_dictionary)
node.layer_name = layer_name # hack to avoid too many queries to get layer name each time
nodes.append(node)
# serialize with rest framework to achieve consistency
serialized_nodes = SerializerClass(nodes, many=True).data
cache.set(cache_key, serialized_nodes, 86400) # cache for 1 day
return serialized_nodes
def test_openlabor_add_node(self):
layer = Layer.objects.external()[0]
layer.minimum_distance = 0
layer.area = None
layer.new_nodes_allowed = True
layer.save()
layer = Layer.objects.get(pk=layer.pk)
url = 'http://devopenlabor.lynxlab.com/api/v1'
external = LayerExternal(layer=layer)
external.interoperability = 'nodeshot.interoperability.synchronizers.OpenLabor'
external.config = json.dumps({
"open311_url": url,
"service_code_get": "001",
"service_code_post": "002",
"default_status": "active",
"api_key": "DEVO1395445966"
})
external.full_clean()
external.save()
node = Node()
node.name = 'offerta di lavoro di test'
node.description = 'altra offerta di lavoro inserita automaticamente tramite unit test'
node.geometry = 'POINT (12.5823391919000012 41.8721429276999820)'
node.layer = layer
node.user_id = 1
node.address = 'via del test'
node.data = {
"professional_profile": "professional_profile test",
"qualification_required": "qualification_required test",
"contract_type": "contract_type test",
"zip_code": "zip code test",
"city": "city test"
}
node.save()
self.assertIsNotNone(node.external.external_id)
def test_node_count_change(self):
""" when a new node is added nodes count should be updated """
new_node = Node(user_id=4,
name='new node',
slug='new_node',
layer_id=1,
coords="POINT (41.9720419277 12.5822391919)")
new_node.save()
self.assertEqual(new_node.user.stats.potential_nodes, 2,
"Potential nodes increment count failed")
new_node.status = NODE_STATUS.get('active')
new_node.save()
self.assertEqual(new_node.user.stats.active_nodes, 4,
"Active nodes increment count failed")
self.assertEqual(new_node.user.stats.potential_nodes, 1,
"Potential nodes decrement count failed")
#new_node.is_hotspot = True
#new_node.save()
#self.assertEqual(new_node.user.stats.hotspots, 2, "Hotspot increment count failed")
new_node.delete()
user = User.objects.get(username='******')
self.assertEqual(user.stats.active_nodes, 3,
"Active nodes decrement count failed")
def import_nodes(self):
""" import nodes into local DB """
self.message('saving nodes into local DB...')
saved_nodes = []
# loop over all old node and create new nodes
for old_node in self.old_nodes:
# if this old node is unconfirmed skip to next cycle
if old_node.status == 'u':
continue
try:
node = Node.objects.get(pk=old_node.id)
except Node.DoesNotExist:
node = Node(id=old_node.id)
node.data = {}
node.user_id = self.users_dict[old_node.email]['id']
node.name = old_node.name
node.slug = old_node.slug
node.geometry = Point(old_node.lng, old_node.lat)
node.elev = old_node.alt
node.description = old_node.description
node.notes = old_node.notes
node.added = old_node.added
node.updated = old_node.updated
intersecting_layers = node.intersecting_layers
# if more than one intersecting layer
if len(intersecting_layers) > 1:
# prompt user
answer = self.prompt_layer_selection(node, intersecting_layers)
if isinstance(answer, int):
node.layer_id = answer
elif answer == 'default' and self.default_layer is not False:
node.layer_id = self.default_layer
else:
self.message('Node %s discarded' % node.name)
continue
# if one intersecting layer select that
elif 2 > len(intersecting_layers) > 0:
node.layer = intersecting_layers[0]
# if no intersecting layers
else:
if self.default_layer is False:
# discard node if no default layer specified
self.message("""Node %s discarded because is not contained
in any specified layer and no default layer specified"""
% node.name)
continue
else:
node.layer_id = self.default_layer
if old_node.postal_code:
# additional info
node.data['postal_code'] = old_node.postal_code
# is it a hotspot?
if old_node.status in ['h', 'ah']:
node.data['is_hotspot'] = 'true'
# determine status according to settings
if self.status_mapping:
node.status_id = self.get_status(old_node.status)
try:
node.full_clean()
node.save(auto_update=False)
saved_nodes.append(node)
self.verbose('Saved node %s in layer %s with status %s' %
(node.name, node.layer, node.status.name))
except Exception:
tb = traceback.format_exc()
self.message('Could not save node %s, got exception:\n\n%s' %
(node.name, tb))
self.message('saved %d nodes into local DB' % len(saved_nodes))
self.saved_nodes = saved_nodes
def save(self):
""" synchronize DB """
# retrieve all items
items = self.parsed_data.getElementsByTagName('AccessPoint')
# init empty lists
added_nodes = []
changed_nodes = []
unmodified_nodes = []
# retrieve a list of local nodes in DB
local_nodes_slug = Node.objects.filter(layer=self.layer).values_list(
'slug', flat=True)
# init empty list of slug of external nodes that will be needed to perform delete operations
external_nodes_slug = []
deleted_nodes_count = 0
try:
self.status = Status.objects.get(
slug=self.config.get('status', None))
except Status.DoesNotExist:
self.status = None
# loop over every parsed item
for item in items:
# retrieve info in auxiliary variables
# readability counts!
name = self.get_text(item, 'Denominazione')[0:70]
slug = slugify(name)
number = 1
original_name = name
needed_different_name = False
while True:
# items might have the same name... so we add a number..
if slug in external_nodes_slug:
needed_different_name = True
number = number + 1
name = "%s - %d" % (original_name, number)
slug = slug = slugify(name)
else:
if needed_different_name:
self.verbose(
'needed a different name for %s, trying "%s"' %
(original_name, name))
break
lat = self.get_text(item, 'Latitudine')
lng = self.get_text(item, 'longitudine')
description = 'Indirizzo: %s, %s; Tipologia: %s' % (self.get_text(
item, 'Indirizzo'), self.get_text(
item, 'Comune'), self.get_text(item, 'Tipologia'))
address = '%s, %s' % (self.get_text(
item, 'Indirizzo'), self.get_text(item, 'Comune'))
# point object
point = Point(float(lng), float(lat))
# default values
added = False
changed = False
try:
# edit existing node
node = Node.objects.get(slug=slug)
except Node.DoesNotExist:
# add a new node
node = Node()
node.layer = self.layer
node.status = self.status
added = True
if node.name != name:
node.name = name
changed = True
if node.slug != slug:
node.slug = slug
changed = True
if added is True or node.geometry.equals(point) is False:
node.geometry = point
changed = True
if node.description != description:
node.description = description
changed = True
if node.address != address:
node.address = address # complete address
node.data = {
'address': self.get_text(item, 'Indirizzo'),
'city': self.get_text(item, 'Comune'),
'province': 'Roma',
'country': 'Italia'
}
changed = True
# perform save or update only if necessary
if added or changed:
try:
node.full_clean()
node.save()
except ValidationError as e:
# TODO: are we sure we want to interrupt the execution?
raise Exception("%s errors: %s" % (name, e.messages))
if added:
added_nodes.append(node)
self.verbose('new node saved with name "%s"' % node.name)
elif changed:
changed_nodes.append(node)
self.verbose('node "%s" updated' % node.name)
else:
unmodified_nodes.append(node)
self.verbose('node "%s" unmodified' % node.name)
# fill node list container
external_nodes_slug.append(node.slug)
# delete old nodes
for local_node in local_nodes_slug:
# if local node not found in external nodes
if not local_node in external_nodes_slug:
node_name = node.name
# retrieve from DB and delete
node = Node.objects.get(slug=local_node)
node.delete()
# then increment count that will be included in message
deleted_nodes_count = deleted_nodes_count + 1
self.verbose('node "%s" deleted' % node_name)
# message that will be returned
self.message = """
%s nodes added
%s nodes changed
%s nodes deleted
%s nodes unmodified
%s total external records processed
%s total local nodes for this layer
""" % (len(added_nodes), len(changed_nodes), deleted_nodes_count,
len(unmodified_nodes), len(items),
Node.objects.filter(layer=self.layer).count())
def save(self):
"""
save data into DB:
1. save new (missing) data
2. update only when needed
3. delete old data
4. generate report that will be printed
constraints:
* ensure new nodes do not take a name/slug which is already used
* validate through django before saving
* use good defaults
"""
self.key_mapping()
# retrieve all items
items = self.parsed_data
# init empty lists
added_nodes = []
changed_nodes = []
unmodified_nodes = []
# retrieve a list of all the slugs of this layer
layer_nodes_slug_list = Node.objects.filter(
layer=self.layer).values_list('slug', flat=True)
# keep a list of all the nodes of other layers
other_layers_slug_list = Node.objects.exclude(
layer=self.layer).values_list('slug', flat=True)
# init empty list of slug of external nodes that will be needed to perform delete operations
processed_slug_list = []
deleted_nodes_count = 0
# loop over every item
for item in items:
item = self._convert_item(item)
number = 1
original_name = item['name']
needed_different_name = False
while True:
# items might have the same name... so we add a number..
if item['slug'] in processed_slug_list or item[
'slug'] in other_layers_slug_list:
needed_different_name = True
number = number + 1
item['name'] = "%s - %d" % (original_name, number)
item['slug'] = slugify(item['name'])
else:
if needed_different_name:
self.verbose(
'needed a different name for %s, trying "%s"' %
(original_name, item['name']))
break
# default values
added = False
changed = False
try:
# edit existing node
node = Node.objects.get(slug=item['slug'], layer=self.layer)
except Node.DoesNotExist:
# add a new node
node = Node()
node.layer = self.layer
added = True
# loop over fields and store data only if necessary
for field in Node._meta.fields:
# geometry is a special case, skip
if field.name == 'geometry':
continue
# skip if field is not present in values
if field.name not in item.keys():
continue
# shortcut for value
value = item[field.name]
# if value is different than what we have
if value is not None and getattr(node, field.name) != value:
# set value
setattr(node, field.name, value)
# indicates that a DB query is necessary
changed = True
if added or (
node.geometry.equals(item['geometry']) is False
and node.geometry.equals_exact(item['geometry']) is False):
node.geometry = item['geometry']
changed = True
node.data = node.data or {}
# store any additional key/value in HStore data field
for key, value in item['data'].items():
if node.data[key] != value:
node.data[key] = value
changed = True
# perform save or update only if necessary
if added or changed:
try:
node.full_clean()
if None not in [node.added, node.updated]:
node.save(auto_update=False)
else:
node.save()
except Exception as e:
raise Exception('error while processing "%s": %s' %
(node.name, e))
if added:
added_nodes.append(node)
self.verbose('new node saved with name "%s"' % node.name)
elif changed:
changed_nodes.append(node)
self.verbose('node "%s" updated' % node.name)
else:
unmodified_nodes.append(node)
self.verbose('node "%s" unmodified' % node.name)
# fill node list container
processed_slug_list.append(node.slug)
# delete old nodes
for local_node in layer_nodes_slug_list:
# if local node not found in external nodes
if local_node not in processed_slug_list:
# retrieve from DB and delete
node = Node.objects.get(slug=local_node)
# store node name to print it later
node_name = node.name
node.delete()
# then increment count that will be included in message
deleted_nodes_count = deleted_nodes_count + 1
self.verbose('node "%s" deleted' % node_name)
# message that will be returned
self.message = """
%s nodes added
%s nodes changed
%s nodes deleted
%s nodes unmodified
%s total external records processed
%s total local nodes for this layer
""" % (len(added_nodes), len(changed_nodes), deleted_nodes_count,
len(unmodified_nodes), len(items),
Node.objects.filter(layer=self.layer).count())
def import_nodes(self):
""" import nodes into local DB """
self.message('saving nodes into local DB...')
saved_nodes = []
# loop over all old node and create new nodes
for old_node in self.old_nodes:
# if this old node is unconfirmed skip to next cycle
if old_node.status == 'u':
continue
node = Node(**{
"id": old_node.id,
"user_id": self.users_dict[old_node.email]['id'],
"name": old_node.name,
"slug": old_node.slug,
"geometry": Point(old_node.lng, old_node.lat),
"elev": old_node.alt,
"description": old_node.description,
"notes": old_node.notes,
"added": old_node.added,
"updated": old_node.updated,
"data": {}
})
if LAYER_APP_INSTALLED:
intersecting_layers = node.intersecting_layers
# if more than one intersecting layer
if len(intersecting_layers) > 1:
# prompt user
answer = self.prompt_layer_selection(node, intersecting_layers)
if isinstance(answer, int):
node.layer_id = answer
elif answer == 'default' and self.default_layer is not False:
node.layer_id = self.default_layer
else:
self.message('Node %s discarded' % node.name)
continue
# if one intersecting layer select that
elif 2 > len(intersecting_layers) > 0:
node.layer = intersecting_layers[0]
# if no intersecting layers
else:
if self.default_layer is False:
# discard node if no default layer specified
self.message("""Node %s discarded because is not contained
in any specified layer and no default layer specified""" % node.name)
continue
else:
node.layer_id = self.default_layer
if old_node.postal_code:
# additional info
node.data['postal_code'] = old_node.postal_code
# is it a hotspot?
if old_node.status in ['h', 'ah']:
node.data['is_hotspot'] = 'true'
# determine status according to settings
if self.status_mapping:
node.status_id = self.get_status(old_node.status)
try:
node.full_clean()
node.save(auto_update=False)
saved_nodes.append(node)
self.verbose('Saved node %s in layer %s with status %s' % (node.name, node.layer, node.status.name))
except Exception as e:
self.message('Could not save node %s, got exception: %s' % (node.name, e))
self.message('saved %d nodes into local DB' % len(saved_nodes))
self.saved_nodes = saved_nodes
def process_streets(self):
if not self.streets:
self.message = """
Street data not processed.
"""
return False
# retrieve all items
items = self.streets
# init empty lists
added_nodes = []
changed_nodes = []
unmodified_nodes = []
# retrieve a list of local nodes in DB
local_nodes_slug = Node.objects.filter(layer=self.layer).values_list(
'slug', flat=True)
# init empty list of slug of external nodes that will be needed to perform delete operations
external_nodes_slug = []
deleted_nodes_count = 0
try:
self.status = Status.objects.get(
slug=self.config.get('status', None))
except Status.DoesNotExist:
self.status = None
# loop over every parsed item
for item in items:
# retrieve info in auxiliary variables
# readability counts!
pk = item['id']
name = item['properties'].get('LOCATION', '')[0:70]
address = name
slug = slugify(name)
number = 1
original_name = name
needed_different_name = False
while True:
# items might have the same name... so we add a number..
# check in DB too
# TODO: this must be DRYED!!
if slug in external_nodes_slug or Node.objects.filter(
slug__exact=slug).exclude(pk=pk).count() > 0:
needed_different_name = True
number = number + 1
name = "%s - %d" % (original_name, number)
slug = slug = slugify(name)
else:
if needed_different_name:
self.verbose(
'needed a different name for %s, trying "%s"' %
(original_name, name))
break
# geometry object
geometry = GEOSGeometry(json.dumps(item["geometry"]))
# default values
added = False
changed = False
try:
# edit existing node
node = Node.objects.get(pk=pk)
except Node.DoesNotExist:
# add a new node
node = Node()
node.id = pk
node.layer = self.layer
node.status = self.status
node.data = {}
added = True
if node.name != name:
node.name = name
changed = True
if node.slug != slug:
node.slug = slug
changed = True
if added is True or node.geometry.equals(geometry) is False:
node.geometry = geometry
changed = True
if node.address != address:
node.address = address
changed = True
# perform save or update only if necessary
if added or changed:
try:
node.full_clean()
node.save()
except ValidationError as e:
# TODO: are we sure we want to interrupt the execution?
raise Exception("%s errors: %s" % (name, e.messages))
if added:
added_nodes.append(node)
self.verbose('new node saved with name "%s"' % node.name)
elif changed:
changed_nodes.append(node)
self.verbose('node "%s" updated' % node.name)
else:
unmodified_nodes.append(node)
self.verbose('node "%s" unmodified' % node.name)
# fill node list container
external_nodes_slug.append(node.slug)
# delete old nodes
for local_node in local_nodes_slug:
# if local node not found in external nodes
if not local_node in external_nodes_slug:
node_name = node.name
# retrieve from DB and delete
node = Node.objects.get(slug=local_node)
node.delete()
# then increment count that will be included in message
deleted_nodes_count = deleted_nodes_count + 1
self.verbose('node "%s" deleted' % node_name)
self.config['last_time_streets_checked'] = str(date.today())
self.layer.external.config = json.dumps(self.config,
indent=4,
sort_keys=True)
self.layer.external.save()
# message that will be returned
self.message = """
%s streets added
%s streets changed
%s streets deleted
%s streets unmodified
%s total external records processed
%s total local records for this layer
""" % (len(added_nodes), len(changed_nodes), deleted_nodes_count,
len(unmodified_nodes), len(items),
Node.objects.filter(layer=self.layer).count())
