def test_layer_area_contains_node_validation(self): """ ensure area validation works as expected """ layer = Layer.objects.get(slug='rome') layer.area = GEOSGeometry('POLYGON ((12.19 41.92, 12.58 42.17, 12.82 41.86, 12.43 41.64, 12.43 41.65, 12.19 41.92))') layer.full_clean() layer.save() # creating node with same coordinates should not be an issue new_node = Node(**{ 'name': 'new_node', 'slug': 'new_node', 'layer': layer, 'geometry': 'POINT (50.0 50.0)' }) try: new_node.full_clean() except ValidationError as e: self.assertIn(_('Node must be inside layer area'), e.messages) else: self.fail('validation not working as expected') # if area is a point the contains check won't be done layer.area = GEOSGeometry('POINT (30.0 30.0)') layer.full_clean() layer.save() new_node.full_clean()
def test_layer_new_nodes_allowed(self): layer = Layer.objects.get(pk=1) layer.new_nodes_allowed = False layer.full_clean() layer.save() # ensure changing an existing node works node = layer.node_set.all()[0] node.name = 'changed' node.full_clean() node.save() # re-get from DB, just to be sure node = Node.objects.get(pk=node.pk) self.assertEqual(node.name, 'changed') # ensure new node cannot be added node = Node(**{ 'name': 'test new node', 'slug': 'test-new-node', 'layer': layer, 'geometry': 'POINT (10.4389188797003565 43.7200020000987328)' }) with self.assertRaises(ValidationError): node.full_clean() try: node.full_clean() assert() except ValidationError as e: self.assertIn(_('New nodes are not allowed for this layer'), e.messages)
def get_nodes(self, class_name, params): """ get nodes """ # determine if response is going to be JSON or GeoJSON if 'geojson' in class_name.lower(): response_format = 'geojson' SerializerClass = OpenLaborGeoSerializer else: response_format = 'json' SerializerClass = OpenLaborSerializer layer_name = self.layer.name cache_key = 'layer_%s_nodes.%s' % (self.layer.id, response_format) serialized_nodes = cache.get(cache_key, False) if serialized_nodes is False: try: response = requests.get(self.get_url, verify=self.config.get( 'verify_SSL', True)) except requests.exceptions.ConnectionError as e: return { 'error': _('external layer not reachable'), 'exception': list(e.message) } try: response.data = json.loads(response.content) except json.scanner.JSONDecodeError as e: return { 'error': _('external layer is experiencing some issues because it returned invalid data' ), 'exception': list(e) } nodes = [] # loop over all the entries and convert to nodeshot format for job in response.data: # skip records which do not have geographic information if not job.get('latitude', False) or not job.get( 'longitude', False): continue # convert response in nodeshot format node_dictionary = self.to_nodeshot(job) # create Node model instance (needed for rest_framework serializer) node = Node(**node_dictionary) node.layer_name = layer_name # hack to avoid too many queries to get layer name each time nodes.append(node) # serialize with rest framework to achieve consistency serialized_nodes = SerializerClass(nodes, many=True).data cache.set(cache_key, serialized_nodes, 86400) # cache for 1 day return serialized_nodes
def get_nodes(self, class_name, params): """ get nodes """ # determine if response is going to be JSON or GeoJSON if 'geojson' in class_name.lower(): response_format = 'geojson' SerializerClass = OpenLaborGeoSerializer else: response_format = 'json' SerializerClass = OpenLaborSerializer layer_name = self.layer.name cache_key = 'layer_%s_nodes.%s' % (self.layer.id, response_format) serialized_nodes = cache.get(cache_key, False) if serialized_nodes is False: try: response = requests.get( self.get_url, verify=self.config.get('verify_SSL', True) ) except requests.exceptions.ConnectionError as e: return { 'error': _('external layer not reachable'), 'exception': list(e.message) } try: response.data = json.loads(response.content) except json.scanner.JSONDecodeError as e: return { 'error': _('external layer is experiencing some issues because it returned invalid data'), 'exception': list(e) } nodes = [] # loop over all the entries and convert to nodeshot format for job in response.data: # skip records which do not have geographic information if not job.get('latitude', False) or not job.get('longitude', False): continue # convert response in nodeshot format node_dictionary = self.to_nodeshot(job) # create Node model instance (needed for rest_framework serializer) node = Node(**node_dictionary) node.layer_name = layer_name # hack to avoid too many queries to get layer name each time nodes.append(node) # serialize with rest framework to achieve consistency serialized_nodes = SerializerClass(nodes, many=True).data cache.set(cache_key, serialized_nodes, 86400) # cache for 1 day return serialized_nodes
def test_layer_nodes_minimum_distance(self): """ ensure minimum distance settings works as expected """ layer = Layer.objects.get(slug='rome') node = layer.node_set.all()[0] # creating node with same coordinates should not be an issue new_node = Node(**{ 'name': 'new_node', 'slug': 'new_node', 'layer': layer, 'geometry': node.geometry }) new_node.full_clean() new_node.save() layer.nodes_minimum_distance = 100 layer.save() try: new_node.full_clean() except ValidationError as e: self.assertIn(_('Distance between nodes cannot be less than %s meters') % layer.nodes_minimum_distance, e.messages) return self.assertTrue(False, 'validation not working as expected')
def test_preexisting_name(self): """ test preexisting names """ layer = Layer.objects.external()[0] layer.new_nodes_allowed = False layer.save() layer = Layer.objects.get(pk=layer.pk) node = Node.first() self.assertNotEqual(layer.id, node.layer.id) node.name = 'simplejson' node.save() url = '%s/geojson1.json' % MOCK_URL_PREFIX responses.add(responses.GET, url, body=self._load('geojson1.json'), content_type='application/json') external = LayerExternal(layer=layer) external.synchronizer_path = 'nodeshot.interop.sync.synchronizers.GeoJson' external._reload_schema() external.config = {"url": url} external.full_clean() external.save() output = capture_output( management.call_command, ['sync', 'vienna'], kwargs={'verbosity': 0} ) # ensure following text is in output self.assertIn('2 nodes added', output) self.assertIn('0 nodes changed', output) self.assertIn('2 total external', output) self.assertIn('2 total local', output)
def test_preexisting_name(self): """ test preexisting names """ layer = Layer.objects.external()[0] layer.new_nodes_allowed = False layer.save() layer = Layer.objects.get(pk=layer.pk) node = Node.first() self.assertNotEqual(layer.id, node.layer.id) node.name = 'simplejson' node.save() url = '%s/geojson1.json' % MOCK_URL_PREFIX responses.add(responses.GET, url, body=self._load('geojson1.json'), content_type='application/json') external = LayerExternal(layer=layer) external.synchronizer_path = 'nodeshot.interop.sync.synchronizers.GeoJson' external._reload_schema() external.config = {"url": url} external.full_clean() external.save() output = capture_output(management.call_command, ['sync', 'vienna'], kwargs={'verbosity': 0}) # ensure following text is in output self.assertIn('2 nodes added', output) self.assertIn('0 nodes changed', output) self.assertIn('2 total external', output) self.assertIn('2 total local', output)
def test_preexisting_name(self): """ test preexisting names """ layer = Layer.objects.external()[0] layer.minimum_distance = 0 layer.area = None layer.new_nodes_allowed = False layer.save() layer = Layer.objects.get(pk=layer.pk) node = Node.first() self.assertNotEqual(layer.id, node.layer.id) node.name = 'simplejson' node.save() url = '%s/geojson1.json' % TEST_FILES_PATH external = LayerExternal(layer=layer) external.interoperability = 'nodeshot.interoperability.synchronizers.GeoJson' external.config = '{ "url": "%s", "map": {} }' % url external.full_clean() external.save() output = capture_output( management.call_command, ['synchronize', 'vienna'], kwargs={ 'verbosity': 0 } ) # ensure following text is in output self.assertIn('2 nodes added', output) self.assertIn('0 nodes changed', output) self.assertIn('2 total external', output) self.assertIn('2 total local', output)
def test_key_mappings(self): """ importing a file with different keys """ layer = Layer.objects.external()[0] layer.new_nodes_allowed = False layer.save() layer = Layer.objects.get(pk=layer.pk) node = Node.first() self.assertNotEqual(layer.id, node.layer.id) node.name = 'simplejson' node.save() url = '%s/geojson3.json' % MOCK_URL_PREFIX responses.add(responses.GET, url, body=self._load('geojson3.json'), content_type='application/json') external = LayerExternal(layer=layer) external.synchronizer_path = 'nodeshot.interop.sync.synchronizers.GeoJson' external._reload_schema() external.url = url external.field_name = "nome" external.field_description = "descrizione" external.field_address = "indirizzo" external.field_elev = "altitudine" external.full_clean() external.save() output = capture_output(management.call_command, ['sync', 'vienna'], kwargs={'verbosity': 0}) # ensure following text is in output self.assertIn('2 nodes added', output) self.assertIn('0 nodes changed', output) self.assertIn('2 total external', output) self.assertIn('2 total local', output) node = Node.objects.get(slug='verycool') self.assertEqual(node.name, 'veryCool') self.assertEqual(node.address, 'veryCool indirizzo') self.assertEqual(node.description, 'veryCool descrizione') self.assertEqual(node.elev, 10.0) node = Node.objects.get(slug='secondo') self.assertEqual(node.name, 'secondo') self.assertEqual(node.address, 'secondo indirizzo') self.assertEqual(node.description, 'secondo descrizione') self.assertEqual(node.elev, 20.0) output = capture_output(management.call_command, ['sync', 'vienna'], kwargs={'verbosity': 0}) # no changes self.assertIn('0 nodes added', output) self.assertIn('0 nodes changed', output) self.assertIn('0 nodes deleted', output) self.assertIn('2 nodes unmodified', output) self.assertIn('2 total external', output) self.assertIn('2 total local', output)
def test_key_mappings(self): """ importing a file with different keys """ layer = Layer.objects.external()[0] layer.minimum_distance = 0 layer.area = None layer.new_nodes_allowed = False layer.save() layer = Layer.objects.get(pk=layer.pk) node = Node.first() self.assertNotEqual(layer.id, node.layer.id) node.name = 'simplejson' node.save() url = '%snodeshot/testing/geojson3.json' % settings.STATIC_URL external = LayerExternal(layer=layer) external.interoperability = 'nodeshot.interoperability.synchronizers.GeoJson' external.config = json.dumps({ "url": url, "map": { "name": "nome", "description": "descrizione", "address": "indirizzo", "elevation": "altitudine" } }) external.full_clean() external.save() # start capturing print statements output = StringIO() sys.stdout = output # execute command management.call_command('synchronize', 'vienna', verbosity=0) # stop capturing print statements sys.stdout = sys.__stdout__ # ensure following text is in output self.assertIn('2 nodes added', output.getvalue()) self.assertIn('0 nodes changed', output.getvalue()) self.assertIn('2 total external', output.getvalue()) self.assertIn('2 total local', output.getvalue()) node = Node.objects.get(slug='verycool') self.assertEqual(node.name, 'veryCool') self.assertEqual(node.address, 'veryCool indirizzo') self.assertEqual(node.description, 'veryCool descrizione') self.assertEqual(node.elev, 10.0) node = Node.objects.get(slug='secondo') self.assertEqual(node.name, 'secondo') self.assertEqual(node.address, 'secondo indirizzo') self.assertEqual(node.description, 'secondo descrizione') self.assertEqual(node.elev, 20.0)
def test_layer_area_validation(self): """ ensure area validation works as expected """ layer = Layer.objects.get(slug='rome') layer.area = GEOSGeometry('POLYGON ((12.19 41.92, 12.58 42.17, 12.82 41.86, 12.43 41.64, 12.43 41.65, 12.19 41.92))') layer.save() # creating node with same coordinates should not be an issue new_node = Node(**{ 'name': 'new_node', 'slug': 'new_node', 'layer': layer, 'geometry': 'POINT (50.0 50.0)' }) try: new_node.full_clean() except ValidationError as e: self.assertIn(_('Node must be inside layer area'), e.messages) return self.assertTrue(False, 'validation not working as expected')
def test_key_mappings(self): """ importing a file with different keys """ layer = Layer.objects.external()[0] layer.minimum_distance = 0 layer.area = None layer.new_nodes_allowed = False layer.save() layer = Layer.objects.get(pk=layer.pk) node = Node.first() self.assertNotEqual(layer.id, node.layer.id) node.name = 'simplejson' node.save() url = '%s/geojson3.json' % TEST_FILES_PATH external = LayerExternal(layer=layer) external.interoperability = 'nodeshot.interoperability.synchronizers.GeoJson' external.config = json.dumps({ "url": url, "map": { "name": "nome", "description": "descrizione", "address": "indirizzo", "elevation": "altitudine" } }) external.full_clean() external.save() output = capture_output( management.call_command, ['synchronize', 'vienna'], kwargs={ 'verbosity': 0 } ) # ensure following text is in output self.assertIn('2 nodes added', output) self.assertIn('0 nodes changed', output) self.assertIn('2 total external', output) self.assertIn('2 total local', output) node = Node.objects.get(slug='verycool') self.assertEqual(node.name, 'veryCool') self.assertEqual(node.address, 'veryCool indirizzo') self.assertEqual(node.description, 'veryCool descrizione') self.assertEqual(node.elev, 10.0) node = Node.objects.get(slug='secondo') self.assertEqual(node.name, 'secondo') self.assertEqual(node.address, 'secondo indirizzo') self.assertEqual(node.description, 'secondo descrizione') self.assertEqual(node.elev, 20.0)
def test_nodes_deletion(self): for user in User.objects.all(): user.delete() settings.DEFAULT_LAYER = 5 management.call_command('import_old_nodeshot', noinput=True) node = Node.objects.get(pk=1) self.assertIn('imported', node.data) # --- delete oldnode --- # OldNode.objects.get(pk=node.pk).delete() # --- ensure node and user are deleted --- # management.call_command('import_old_nodeshot', noinput=True) self.assertEqual(Node.objects.filter(pk=node.pk).count(), 0) self.assertEqual(User.objects.filter(email="*****@*****.**").count(), 0) # --- add not imported node --- # added_node = Node(name='added', slug='added', layer_id=5, description='added', geometry='POINT (12 42)') added_node.full_clean() added_node.save() # --- ensure added_node is not deleted --- # management.call_command('import_old_nodeshot', noinput=True) self.assertEqual(Node.objects.filter(pk=added_node.pk).count(), 1)
def test_openlabor_add_node(self): layer = Layer.objects.external()[0] layer.minimum_distance = 0 layer.area = None layer.new_nodes_allowed = True layer.save() layer = Layer.objects.get(pk=layer.pk) url = 'http://devopenlabor.lynxlab.com/api/v1' external = LayerExternal(layer=layer) external.interoperability = 'nodeshot.interoperability.synchronizers.OpenLabor' external.config = json.dumps({ "open311_url": url, "service_code_get": "001", "service_code_post": "002", "default_status": "active", "api_key": "DEVO1395445966" }) external.full_clean() external.save() node = Node() node.name = 'offerta di lavoro di test' node.description = 'altra offerta di lavoro inserita automaticamente tramite unit test' node.geometry = 'POINT (12.5823391919000012 41.8721429276999820)' node.layer = layer node.user_id = 1 node.address = 'via del test' node.data = { "professional_profile": "professional_profile test", "qualification_required": "qualification_required test", "contract_type": "contract_type test", "zip_code": "zip code test", "city": "city test" } node.save() self.assertIsNotNone(node.external.external_id)
def test_node_count_change(self): """ when a new node is added nodes count should be updated """ new_node = Node(user_id=4, name='new node', slug='new_node', layer_id=1, coords="POINT (41.9720419277 12.5822391919)") new_node.save() self.assertEqual(new_node.user.stats.potential_nodes, 2, "Potential nodes increment count failed") new_node.status = NODE_STATUS.get('active') new_node.save() self.assertEqual(new_node.user.stats.active_nodes, 4, "Active nodes increment count failed") self.assertEqual(new_node.user.stats.potential_nodes, 1, "Potential nodes decrement count failed") #new_node.is_hotspot = True #new_node.save() #self.assertEqual(new_node.user.stats.hotspots, 2, "Hotspot increment count failed") new_node.delete() user = User.objects.get(username='******') self.assertEqual(user.stats.active_nodes, 3, "Active nodes decrement count failed")
def test_preexisting_name(self): """ test preexisting names """ layer = Layer.objects.external()[0] layer.minimum_distance = 0 layer.area = None layer.new_nodes_allowed = False layer.save() layer = Layer.objects.get(pk=layer.pk) node = Node.first() self.assertNotEqual(layer.id, node.layer.id) node.name = 'simplejson' node.save() url = '%snodeshot/testing/geojson1.json' % settings.STATIC_URL external = LayerExternal(layer=layer) external.interoperability = 'nodeshot.interoperability.synchronizers.GeoJson' external.config = '{ "url": "%s", "map": {} }' % url external.full_clean() external.save() # start capturing print statements output = StringIO() sys.stdout = output # execute command management.call_command('synchronize', 'vienna', verbosity=0) # stop capturing print statements sys.stdout = sys.__stdout__ # ensure following text is in output self.assertIn('2 nodes added', output.getvalue()) self.assertIn('0 nodes changed', output.getvalue()) self.assertIn('2 total external', output.getvalue()) self.assertIn('2 total local', output.getvalue())
def process_streets(self): if not self.streets: self.message = """ Street data not processed. """ return False # retrieve all items items = self.streets # init empty lists added_nodes = [] changed_nodes = [] unmodified_nodes = [] # retrieve a list of local nodes in DB local_nodes_slug = Node.objects.filter(layer=self.layer).values_list( 'slug', flat=True) # init empty list of slug of external nodes that will be needed to perform delete operations external_nodes_slug = [] deleted_nodes_count = 0 try: self.status = Status.objects.get( slug=self.config.get('status', None)) except Status.DoesNotExist: self.status = None # loop over every parsed item for item in items: # retrieve info in auxiliary variables # readability counts! pk = item['id'] name = item['properties'].get('LOCATION', '')[0:70] address = name slug = slugify(name) number = 1 original_name = name needed_different_name = False while True: # items might have the same name... so we add a number.. # check in DB too # TODO: this must be DRYED!! if slug in external_nodes_slug or Node.objects.filter( slug__exact=slug).exclude(pk=pk).count() > 0: needed_different_name = True number = number + 1 name = "%s - %d" % (original_name, number) slug = slug = slugify(name) else: if needed_different_name: self.verbose( 'needed a different name for %s, trying "%s"' % (original_name, name)) break # geometry object geometry = GEOSGeometry(json.dumps(item["geometry"])) # default values added = False changed = False try: # edit existing node node = Node.objects.get(pk=pk) except Node.DoesNotExist: # add a new node node = Node() node.id = pk node.layer = self.layer node.status = self.status node.data = {} added = True if node.name != name: node.name = name changed = True if node.slug != slug: node.slug = slug changed = True if added is True or node.geometry.equals(geometry) is False: node.geometry = geometry changed = True if node.address != address: node.address = address changed = True # perform save or update only if necessary if added or changed: try: node.full_clean() node.save() except ValidationError as e: # TODO: are we sure we want to interrupt the execution? raise Exception("%s errors: %s" % (name, e.messages)) if added: added_nodes.append(node) self.verbose('new node saved with name "%s"' % node.name) elif changed: changed_nodes.append(node) self.verbose('node "%s" updated' % node.name) else: unmodified_nodes.append(node) self.verbose('node "%s" unmodified' % node.name) # fill node list container external_nodes_slug.append(node.slug) # delete old nodes for local_node in local_nodes_slug: # if local node not found in external nodes if not local_node in external_nodes_slug: node_name = node.name # retrieve from DB and delete node = Node.objects.get(slug=local_node) node.delete() # then increment count that will be included in message deleted_nodes_count = deleted_nodes_count + 1 self.verbose('node "%s" deleted' % node_name) self.config['last_time_streets_checked'] = str(date.today()) self.layer.external.config = json.dumps(self.config, indent=4, sort_keys=True) self.layer.external.save() # message that will be returned self.message = """ %s streets added %s streets changed %s streets deleted %s streets unmodified %s total external records processed %s total local records for this layer """ % (len(added_nodes), len(changed_nodes), deleted_nodes_count, len(unmodified_nodes), len(items), Node.objects.filter(layer=self.layer).count())
def save(self): """ synchronize DB """ # retrieve all items items = self.parsed_data.getElementsByTagName('AccessPoint') # init empty lists added_nodes = [] changed_nodes = [] unmodified_nodes = [] # retrieve a list of local nodes in DB local_nodes_slug = Node.objects.filter(layer=self.layer).values_list('slug', flat=True) # init empty list of slug of external nodes that will be needed to perform delete operations external_nodes_slug = [] deleted_nodes_count = 0 try: self.status = Status.objects.get(slug=self.config.get('status', None)) except Status.DoesNotExist: self.status = None # loop over every parsed item for item in items: # retrieve info in auxiliary variables # readability counts! name = self.get_text(item, 'Denominazione')[0:70] slug = slugify(name) number = 1 original_name = name needed_different_name = False while True: # items might have the same name... so we add a number.. if slug in external_nodes_slug: needed_different_name = True number = number + 1 name = "%s - %d" % (original_name, number) slug = slug = slugify(name) else: if needed_different_name: self.verbose('needed a different name for %s, trying "%s"' % (original_name, name)) break lat = self.get_text(item, 'Latitudine') lng = self.get_text(item, 'longitudine') description = 'Indirizzo: %s, %s; Tipologia: %s' % ( self.get_text(item, 'Indirizzo'), self.get_text(item, 'Comune'), self.get_text(item, 'Tipologia') ) address = '%s, %s' % ( self.get_text(item, 'Indirizzo'), self.get_text(item, 'Comune') ) # point object point = Point(float(lng), float(lat)) # default values added = False changed = False try: # edit existing node node = Node.objects.get(slug=slug) except Node.DoesNotExist: # add a new node node = Node() node.layer = self.layer node.status = self.status added = True if node.name != name: node.name = name changed = True if node.slug != slug: node.slug = slug changed = True if added is True or node.geometry.equals(point) is False: node.geometry = point changed = True if node.description != description: node.description = description changed = True if node.address != address: node.address = address # complete address node.data = { 'address': self.get_text(item, 'Indirizzo'), 'city': self.get_text(item, 'Comune'), 'province': 'Roma', 'country': 'Italia' } changed = True # perform save or update only if necessary if added or changed: try: node.full_clean() node.save() except ValidationError as e: # TODO: are we sure we want to interrupt the execution? raise Exception("%s errors: %s" % (name, e.messages)) if added: added_nodes.append(node) self.verbose('new node saved with name "%s"' % node.name) elif changed: changed_nodes.append(node) self.verbose('node "%s" updated' % node.name) else: unmodified_nodes.append(node) self.verbose('node "%s" unmodified' % node.name) # fill node list container external_nodes_slug.append(node.slug) # delete old nodes for local_node in local_nodes_slug: # if local node not found in external nodes if not local_node in external_nodes_slug: node_name = node.name # retrieve from DB and delete node = Node.objects.get(slug=local_node) node.delete() # then increment count that will be included in message deleted_nodes_count = deleted_nodes_count + 1 self.verbose('node "%s" deleted' % node_name) # message that will be returned self.message = """ %s nodes added %s nodes changed %s nodes deleted %s nodes unmodified %s total external records processed %s total local nodes for this layer """ % ( len(added_nodes), len(changed_nodes), deleted_nodes_count, len(unmodified_nodes), len(items), Node.objects.filter(layer=self.layer).count() )
def nodes_minimum_distance_validation(self): """ if minimum distance is specified, ensure node is not too close to other nodes; """ if self.layer and self.layer.nodes_minimum_distance: minimum_distance = self.layer.nodes_minimum_distance # TODO - lower priority: do this check only when coordinates are changing near_nodes = Node.objects.exclude(pk=self.id).filter( geometry__distance_lte=(self.geometry, D(m=minimum_distance))).count() if near_nodes > 0: raise ValidationError( _('Distance between nodes cannot be less than %s meters') % minimum_distance) def node_contained_in_layer_area_validation(self): """ if layer defines an area, ensure node coordinates are contained in the area """ # if area is a polygon ensure it contains the node if self.layer and isinstance( self.layer.area, Polygon) and not self.layer.area.contains(self.geometry): raise ValidationError(_('Node must be inside layer area')) Node.add_validation_method(new_nodes_allowed_for_layer) Node.add_validation_method(nodes_minimum_distance_validation) Node.add_validation_method(node_contained_in_layer_area_validation)
def test_openlabor_add_node(self): layer = Layer.objects.external()[0] layer.minimum_distance = 0 layer.area = None layer.new_nodes_allowed = True layer.save() layer = Layer.objects.get(pk=layer.pk) url = 'http://devopenlabor.lynxlab.com/api/v1' external = LayerExternal(layer=layer) external.interoperability = 'nodeshot.interoperability.synchronizers.OpenLabor' external.config = json.dumps({ "open311_url": url, "service_code_get": "001", "service_code_post": "002", "api_key": "DEVO1395445966" }) external.full_clean() external.save() node = Node() node.name = 'offerta di lavoro di test' node.description = 'altra offerta di lavoro inserita automaticamente tramite unit test' node.geometry = 'POINT (12.5823391919000012 41.8721429276999820)' node.layer = layer node.user_id = 1 node.address = 'via del test' node.data = { "professional_profile": "professional_profile test", "qualification_required": "qualification_required test", "contract_type": "contract_type test", "zip_code": "zip code test", "city": "city test" } node.save() self.assertIsNotNone(node.external.external_id)
raise ValidationError(_('New nodes are not allowed for this layer')) except ObjectDoesNotExist: # this happens if node.layer is None return # TODO: thes features must be tested inside the layer's app code (nodeshot.core.layers.tests) def node_layer_validation(self): """ 1. if minimum distance is specified, ensure node is not too close to other nodes; 2. if layer defines an area, ensure node coordinates are contained in the area """ try: minimum_distance = self.layer.minimum_distance geometry = self.geometry layer_area = self.layer.area except ObjectDoesNotExist: # this happens if node.layer is None return # TODO - lower priority: do this check only when coordinates are changing if minimum_distance > 0: near_nodes = Node.objects.exclude(pk=self.id).filter(geometry__distance_lte=(geometry, D(m=minimum_distance))).count() if near_nodes > 0 : raise ValidationError(_('Distance between nodes cannot be less than %s meters') % minimum_distance) if layer_area is not None and not layer_area.contains(geometry): raise ValidationError(_('Node must be inside layer area')) Node.add_validation_method(new_nodes_allowed_for_layer) Node.add_validation_method(node_layer_validation)
if not self.pk and not self.layer.new_nodes_allowed: raise ValidationError(_('New nodes are not allowed for this layer')) except ObjectDoesNotExist: # this happens if node.layer is None return # TODO: thes features must be tested inside the layer's app code (nodeshot.core.layers.tests) def node_layer_validation(self): """ 1. if minimum distance is specified, ensure node is not too close to other nodes; 2. if layer defines an area, ensure node coordinates are contained in the area """ try: minimum_distance = self.layer.minimum_distance geometry = self.geometry layer_area = self.layer.area except ObjectDoesNotExist: # this happens if node.layer is None return # TODO - lower priority: do this check only when coordinates are changing if minimum_distance > 0: near_nodes = Node.objects.exclude(pk=self.id).filter(geometry__distance_lte=(geometry, D(m=minimum_distance))).count() if near_nodes > 0 : raise ValidationError(_('Distance between nodes cannot be less than %s meters') % minimum_distance) if layer_area is not None and not layer_area.contains(geometry): raise ValidationError(_('Node must be inside layer area')) Node.add_validation_method(new_nodes_allowed_for_layer) Node.add_validation_method(node_layer_validation)
ensure new nodes are allowed for this layer """ if not self.pk and self.layer and not self.layer.new_nodes_allowed: raise ValidationError(_('New nodes are not allowed for this layer')) def nodes_minimum_distance_validation(self): """ if minimum distance is specified, ensure node is not too close to other nodes; """ if self.layer and self.layer.nodes_minimum_distance: minimum_distance = self.layer.nodes_minimum_distance # TODO - lower priority: do this check only when coordinates are changing near_nodes = Node.objects.exclude(pk=self.id).filter(geometry__distance_lte=(self.geometry, D(m=minimum_distance))).count() if near_nodes > 0: raise ValidationError(_('Distance between nodes cannot be less than %s meters') % minimum_distance) def node_contained_in_layer_area_validation(self): """ if layer defines an area, ensure node coordinates are contained in the area """ # if area is a polygon ensure it contains the node if self.layer and isinstance(self.layer.area, Polygon) and not self.layer.area.contains(self.geometry): raise ValidationError(_('Node must be inside layer area')) Node.add_validation_method(new_nodes_allowed_for_layer) Node.add_validation_method(nodes_minimum_distance_validation) Node.add_validation_method(node_contained_in_layer_area_validation)
def save(self): """ synchronize DB """ # retrieve all items items = self.parsed_data.getElementsByTagName('AccessPoint') # init empty lists added_nodes = [] changed_nodes = [] unmodified_nodes = [] # retrieve a list of local nodes in DB local_nodes_slug = Node.objects.filter(layer=self.layer).values_list( 'slug', flat=True) # init empty list of slug of external nodes that will be needed to perform delete operations external_nodes_slug = [] deleted_nodes_count = 0 try: self.status = Status.objects.get( slug=self.config.get('status', None)) except Status.DoesNotExist: self.status = None # loop over every parsed item for item in items: # retrieve info in auxiliary variables # readability counts! name = self.get_text(item, 'Denominazione')[0:70] slug = slugify(name) number = 1 original_name = name needed_different_name = False while True: # items might have the same name... so we add a number.. if slug in external_nodes_slug: needed_different_name = True number = number + 1 name = "%s - %d" % (original_name, number) slug = slug = slugify(name) else: if needed_different_name: self.verbose( 'needed a different name for %s, trying "%s"' % (original_name, name)) break lat = self.get_text(item, 'Latitudine') lng = self.get_text(item, 'longitudine') description = 'Indirizzo: %s, %s; Tipologia: %s' % (self.get_text( item, 'Indirizzo'), self.get_text( item, 'Comune'), self.get_text(item, 'Tipologia')) address = '%s, %s' % (self.get_text( item, 'Indirizzo'), self.get_text(item, 'Comune')) # point object point = Point(float(lng), float(lat)) # default values added = False changed = False try: # edit existing node node = Node.objects.get(slug=slug) except Node.DoesNotExist: # add a new node node = Node() node.layer = self.layer node.status = self.status added = True if node.name != name: node.name = name changed = True if node.slug != slug: node.slug = slug changed = True if added is True or node.geometry.equals(point) is False: node.geometry = point changed = True if node.description != description: node.description = description changed = True if node.address != address: node.address = address # complete address node.data = { 'address': self.get_text(item, 'Indirizzo'), 'city': self.get_text(item, 'Comune'), 'province': 'Roma', 'country': 'Italia' } changed = True # perform save or update only if necessary if added or changed: try: node.full_clean() node.save() except ValidationError as e: # TODO: are we sure we want to interrupt the execution? raise Exception("%s errors: %s" % (name, e.messages)) if added: added_nodes.append(node) self.verbose('new node saved with name "%s"' % node.name) elif changed: changed_nodes.append(node) self.verbose('node "%s" updated' % node.name) else: unmodified_nodes.append(node) self.verbose('node "%s" unmodified' % node.name) # fill node list container external_nodes_slug.append(node.slug) # delete old nodes for local_node in local_nodes_slug: # if local node not found in external nodes if not local_node in external_nodes_slug: node_name = node.name # retrieve from DB and delete node = Node.objects.get(slug=local_node) node.delete() # then increment count that will be included in message deleted_nodes_count = deleted_nodes_count + 1 self.verbose('node "%s" deleted' % node_name) # message that will be returned self.message = """ %s nodes added %s nodes changed %s nodes deleted %s nodes unmodified %s total external records processed %s total local nodes for this layer """ % (len(added_nodes), len(changed_nodes), deleted_nodes_count, len(unmodified_nodes), len(items), Node.objects.filter(layer=self.layer).count())
def process_borders(self,borders): if not borders: self.message = """ Borders data not processed. """ return False # retrieve all items items = borders # init empty lists added_nodes = [] changed_nodes = [] unmodified_nodes = [] # retrieve a list of local nodes in DB local_nodes_slug = Node.objects.filter(layer=self.layer).values_list('slug', flat=True) # init empty list of slug of external nodes that will be needed to perform delete operations external_nodes_slug = [] deleted_nodes_count = 0 try: self.status = Status.objects.get(slug=self.config.get('status', None)) except Status.DoesNotExist: self.status = None # loop over every parsed item for item in items: # retrieve info in auxiliary variables # readability counts! name = item['properties'].get('name', '')[0:70] address = name slug = slugify(name) #print(slug) number = 1 original_name = name needed_different_name = False while True: # items might have the same name... so we add a number.. if slug in external_nodes_slug: needed_different_name = True number = number + 1 name = "%s - %d" % (original_name, number) slug = slug = slugify(name) else: if needed_different_name: self.verbose('needed a different name for %s, trying "%s"' % (original_name, name)) break # geometry object geometry = GEOSGeometry(json.dumps(item["geometry"])) # default values added = False changed = False try: # edit existing node node = Node.objects.get(slug=slug) except Node.DoesNotExist: # add a new node node = Node() node.layer = self.layer node.status = self.status node.data = {} added = True if node.name != name: node.name = name changed = True if node.slug != slug: node.slug = slug changed = True if added is True or node.geometry.equals(geometry) is False: node.geometry = geometry changed = True if node.address != address: node.address = address changed = True # perform save or update only if necessary if added or changed: try: node.full_clean() node.save() except ValidationError as e: # TODO: are we sure we want to interrupt the execution? raise Exception("%s import errors: %s" % (name, e.messages)) if added: added_nodes.append(node) self.verbose('new node saved with name "%s"' % node.name) elif changed: changed_nodes.append(node) self.verbose('node "%s" updated' % node.name) else: unmodified_nodes.append(node) self.verbose('node "%s" unmodified' % node.name) # fill node list container external_nodes_slug.append(node.slug) # delete old nodes for local_node in local_nodes_slug: # if local node not found in external nodes if not local_node in external_nodes_slug: node_name = node.name # retrieve from DB and delete node = Node.objects.get(slug=local_node) node.delete() # then increment count that will be included in message deleted_nodes_count = deleted_nodes_count + 1 self.verbose('node "%s" deleted' % node_name) self.layer.external.config = json.dumps(self.config, indent=4, sort_keys=True) self.layer.external.save() # message that will be returned self.message = """ %s node added %s node changed %s node deleted %s node unmodified %s total external records processed %s total local records for this layer """ % ( len(added_nodes), len(changed_nodes), deleted_nodes_count, len(unmodified_nodes), len(items), Node.objects.filter(layer=self.layer).count() )
def import_nodes(self): """ import nodes into local DB """ self.message('saving nodes into local DB...') saved_nodes = [] # loop over all old node and create new nodes for old_node in self.old_nodes: # if this old node is unconfirmed skip to next cycle if old_node.status == 'u': continue try: node = Node.objects.get(pk=old_node.id) except Node.DoesNotExist: node = Node(id=old_node.id) node.data = {} node.user_id = self.users_dict[old_node.email]['id'] node.name = old_node.name node.slug = old_node.slug node.geometry = Point(old_node.lng, old_node.lat) node.elev = old_node.alt node.description = old_node.description node.notes = old_node.notes node.added = old_node.added node.updated = old_node.updated node.data['imported'] = 'true' intersecting_layers = node.intersecting_layers # if more than one intersecting layer if len(intersecting_layers) > 1: # prompt user answer = self.prompt_layer_selection(node, intersecting_layers) if isinstance(answer, int): node.layer_id = answer elif answer == 'default' and self.default_layer is not False: node.layer_id = self.default_layer else: self.message('Node %s discarded' % node.name) continue # if one intersecting layer select that elif 2 > len(intersecting_layers) > 0: node.layer = intersecting_layers[0] # if no intersecting layers else: if self.default_layer is False: # discard node if no default layer specified self.message("""Node %s discarded because is not contained in any specified layer and no default layer specified""" % node.name) continue else: node.layer_id = self.default_layer if old_node.postal_code: # additional info node.data['postal_code'] = old_node.postal_code # is it a hotspot? if old_node.status in ['h', 'ah']: node.data['is_hotspot'] = 'true' # determine status according to settings if self.status_mapping: node.status_id = self.get_status(old_node.status) try: node.full_clean() node.save(auto_update=False) saved_nodes.append(node) self.verbose('Saved node %s in layer %s with status %s' % (node.name, node.layer, node.status.name)) except Exception: tb = traceback.format_exc() self.message('Could not save node %s, got exception:\n\n%s' % (node.name, tb)) self.message('saved %d nodes into local DB' % len(saved_nodes)) self.saved_nodes = saved_nodes
def test_key_mappings(self): """ importing a file with different keys """ layer = Layer.objects.external()[0] layer.new_nodes_allowed = False layer.save() layer = Layer.objects.get(pk=layer.pk) node = Node.first() self.assertNotEqual(layer.id, node.layer.id) node.name = 'simplejson' node.save() url = '%s/geojson3.json' % MOCK_URL_PREFIX responses.add(responses.GET, url, body=self._load('geojson3.json'), content_type='application/json') external = LayerExternal(layer=layer) external.synchronizer_path = 'nodeshot.interop.sync.synchronizers.GeoJson' external._reload_schema() external.url = url external.field_name = "nome" external.field_description = "descrizione" external.field_address = "indirizzo" external.field_elev = "altitudine" external.full_clean() external.save() output = capture_output( management.call_command, ['sync', 'vienna'], kwargs={'verbosity': 0} ) # ensure following text is in output self.assertIn('2 nodes added', output) self.assertIn('0 nodes changed', output) self.assertIn('2 total external', output) self.assertIn('2 total local', output) node = Node.objects.get(slug='verycool') self.assertEqual(node.name, 'veryCool') self.assertEqual(node.address, 'veryCool indirizzo') self.assertEqual(node.description, 'veryCool descrizione') self.assertEqual(node.elev, 10.0) node = Node.objects.get(slug='secondo') self.assertEqual(node.name, 'secondo') self.assertEqual(node.address, 'secondo indirizzo') self.assertEqual(node.description, 'secondo descrizione') self.assertEqual(node.elev, 20.0) output = capture_output( management.call_command, ['sync', 'vienna'], kwargs={'verbosity': 0} ) # no changes self.assertIn('0 nodes added', output) self.assertIn('0 nodes changed', output) self.assertIn('0 nodes deleted', output) self.assertIn('2 nodes unmodified', output) self.assertIn('2 total external', output) self.assertIn('2 total local', output)
def import_nodes(self): """ import nodes into local DB """ self.message('saving nodes into local DB...') saved_nodes = [] # loop over all old node and create new nodes for old_node in self.old_nodes: # if this old node is unconfirmed skip to next cycle if old_node.status == 'u': continue try: node = Node.objects.get(pk=old_node.id) except Node.DoesNotExist: node = Node(id=old_node.id) node.data = {} node.user_id = self.users_dict[old_node.email]['id'] node.name = old_node.name node.slug = old_node.slug node.geometry = Point(old_node.lng, old_node.lat) node.elev = old_node.alt node.description = old_node.description node.notes = old_node.notes node.added = old_node.added node.updated = old_node.updated intersecting_layers = node.intersecting_layers # if more than one intersecting layer if len(intersecting_layers) > 1: # prompt user answer = self.prompt_layer_selection(node, intersecting_layers) if isinstance(answer, int): node.layer_id = answer elif answer == 'default' and self.default_layer is not False: node.layer_id = self.default_layer else: self.message('Node %s discarded' % node.name) continue # if one intersecting layer select that elif 2 > len(intersecting_layers) > 0: node.layer = intersecting_layers[0] # if no intersecting layers else: if self.default_layer is False: # discard node if no default layer specified self.message("""Node %s discarded because is not contained in any specified layer and no default layer specified""" % node.name) continue else: node.layer_id = self.default_layer if old_node.postal_code: # additional info node.data['postal_code'] = old_node.postal_code # is it a hotspot? if old_node.status in ['h', 'ah']: node.data['is_hotspot'] = 'true' # determine status according to settings if self.status_mapping: node.status_id = self.get_status(old_node.status) try: node.full_clean() node.save(auto_update=False) saved_nodes.append(node) self.verbose('Saved node %s in layer %s with status %s' % (node.name, node.layer, node.status.name)) except Exception: tb = traceback.format_exc() self.message('Could not save node %s, got exception:\n\n%s' % (node.name, tb)) self.message('saved %d nodes into local DB' % len(saved_nodes)) self.saved_nodes = saved_nodes
def save(self): """ save data into DB: 1. save new (missing) data 2. update only when needed 3. delete old data 4. generate report that will be printed constraints: * ensure new nodes do not take a name/slug which is already used * validate through django before saving * use good defaults """ self.key_mapping() # retrieve all items items = self.parsed_data # init empty lists added_nodes = [] changed_nodes = [] unmodified_nodes = [] # retrieve a list of all the slugs of this layer layer_nodes_slug_list = Node.objects.filter( layer=self.layer).values_list('slug', flat=True) # keep a list of all the nodes of other layers other_layers_slug_list = Node.objects.exclude( layer=self.layer).values_list('slug', flat=True) # init empty list of slug of external nodes that will be needed to perform delete operations processed_slug_list = [] deleted_nodes_count = 0 # loop over every item for item in items: item = self._convert_item(item) number = 1 original_name = item['name'] needed_different_name = False while True: # items might have the same name... so we add a number.. if item['slug'] in processed_slug_list or item[ 'slug'] in other_layers_slug_list: needed_different_name = True number = number + 1 item['name'] = "%s - %d" % (original_name, number) item['slug'] = slugify(item['name']) else: if needed_different_name: self.verbose( 'needed a different name for %s, trying "%s"' % (original_name, item['name'])) break # default values added = False changed = False try: # edit existing node node = Node.objects.get(slug=item['slug'], layer=self.layer) except Node.DoesNotExist: # add a new node node = Node() node.layer = self.layer added = True # loop over fields and store data only if necessary for field in Node._meta.fields: # geometry is a special case, skip if field.name == 'geometry': continue # skip if field is not present in values if field.name not in item.keys(): continue # shortcut for value value = item[field.name] # if value is different than what we have if value is not None and getattr(node, field.name) != value: # set value setattr(node, field.name, value) # indicates that a DB query is necessary changed = True if added or ( node.geometry.equals(item['geometry']) is False and node.geometry.equals_exact(item['geometry']) is False): node.geometry = item['geometry'] changed = True node.data = node.data or {} # store any additional key/value in HStore data field for key, value in item['data'].items(): if node.data[key] != value: node.data[key] = value changed = True # perform save or update only if necessary if added or changed: try: node.full_clean() if None not in [node.added, node.updated]: node.save(auto_update=False) else: node.save() except Exception as e: raise Exception('error while processing "%s": %s' % (node.name, e)) if added: added_nodes.append(node) self.verbose('new node saved with name "%s"' % node.name) elif changed: changed_nodes.append(node) self.verbose('node "%s" updated' % node.name) else: unmodified_nodes.append(node) self.verbose('node "%s" unmodified' % node.name) # fill node list container processed_slug_list.append(node.slug) # delete old nodes for local_node in layer_nodes_slug_list: # if local node not found in external nodes if local_node not in processed_slug_list: # retrieve from DB and delete node = Node.objects.get(slug=local_node) # store node name to print it later node_name = node.name node.delete() # then increment count that will be included in message deleted_nodes_count = deleted_nodes_count + 1 self.verbose('node "%s" deleted' % node_name) # message that will be returned self.message = """ %s nodes added %s nodes changed %s nodes deleted %s nodes unmodified %s total external records processed %s total local nodes for this layer """ % (len(added_nodes), len(changed_nodes), deleted_nodes_count, len(unmodified_nodes), len(items), Node.objects.filter(layer=self.layer).count())
def import_nodes(self): """ import nodes into local DB """ self.message('saving nodes into local DB...') saved_nodes = [] # loop over all old node and create new nodes for old_node in self.old_nodes: # if this old node is unconfirmed skip to next cycle if old_node.status == 'u': continue node = Node(**{ "id": old_node.id, "user_id": self.users_dict[old_node.email]['id'], "name": old_node.name, "slug": old_node.slug, "geometry": Point(old_node.lng, old_node.lat), "elev": old_node.alt, "description": old_node.description, "notes": old_node.notes, "added": old_node.added, "updated": old_node.updated, "data": {} }) if LAYER_APP_INSTALLED: intersecting_layers = node.intersecting_layers # if more than one intersecting layer if len(intersecting_layers) > 1: # prompt user answer = self.prompt_layer_selection(node, intersecting_layers) if isinstance(answer, int): node.layer_id = answer elif answer == 'default' and self.default_layer is not False: node.layer_id = self.default_layer else: self.message('Node %s discarded' % node.name) continue # if one intersecting layer select that elif 2 > len(intersecting_layers) > 0: node.layer = intersecting_layers[0] # if no intersecting layers else: if self.default_layer is False: # discard node if no default layer specified self.message("""Node %s discarded because is not contained in any specified layer and no default layer specified""" % node.name) continue else: node.layer_id = self.default_layer if old_node.postal_code: # additional info node.data['postal_code'] = old_node.postal_code # is it a hotspot? if old_node.status in ['h', 'ah']: node.data['is_hotspot'] = 'true' # determine status according to settings if self.status_mapping: node.status_id = self.get_status(old_node.status) try: node.full_clean() node.save(auto_update=False) saved_nodes.append(node) self.verbose('Saved node %s in layer %s with status %s' % (node.name, node.layer, node.status.name)) except Exception as e: self.message('Could not save node %s, got exception: %s' % (node.name, e)) self.message('saved %d nodes into local DB' % len(saved_nodes)) self.saved_nodes = saved_nodes
def save(self): """ save data into DB: 1. save new (missing) data 2. update only when needed 3. delete old data 4. generate report that will be printed constraints: * ensure new nodes do not take a name/slug which is already used * validate through django before saving * use good defaults """ self.key_mapping() # retrieve all items items = self.parsed_data # init empty lists added_nodes = [] changed_nodes = [] unmodified_nodes = [] # retrieve a list of all the slugs of this layer layer_nodes_slug_list = Node.objects.filter(layer=self.layer).values_list('slug', flat=True) # keep a list of all the nodes of other layers other_layers_slug_list = Node.objects.exclude(layer=self.layer).values_list('slug', flat=True) # init empty list of slug of external nodes that will be needed to perform delete operations processed_slug_list = [] deleted_nodes_count = 0 # loop over every item for item in items: item = self._convert_item(item) number = 1 original_name = item['name'] needed_different_name = False while True: # items might have the same name... so we add a number.. if item['slug'] in processed_slug_list or item['slug'] in other_layers_slug_list: needed_different_name = True number = number + 1 item['name'] = "%s - %d" % (original_name, number) item['slug'] = slugify(item['name']) else: if needed_different_name: self.verbose('needed a different name for %s, trying "%s"' % (original_name, item['name'])) break # default values added = False changed = False try: # edit existing node node = Node.objects.get(slug=item['slug'], layer=self.layer) except Node.DoesNotExist: # add a new node node = Node() node.layer = self.layer added = True # loop over fields and store data only if necessary for field in Node._meta.fields: # geometry is a special case, skip if field.name == 'geometry': continue # skip if field is not present in values if field.name not in item.keys(): continue # shortcut for value value = item[field.name] # if value is different than what we have if getattr(node, field.name) != value and value is not None: # set value setattr(node, field.name, value) # indicates that a DB query is necessary changed = True if added or (node.geometry.equals(item['geometry']) is False and node.geometry.equals_exact(item['geometry']) is False): node.geometry = item['geometry'] changed = True node.data = node.data or {} # store any additional key/value in HStore data field for key, value in item['data'].items(): if node.data[key] != value: node.data[key] = value changed = True # perform save or update only if necessary if added or changed: try: node.full_clean() if None not in [node.added, node.updated]: node.save(auto_update=False) else: node.save() except Exception as e: raise Exception('error while processing "%s": %s' % (node.name, e)) if added: added_nodes.append(node) self.verbose('new node saved with name "%s"' % node.name) elif changed: changed_nodes.append(node) self.verbose('node "%s" updated' % node.name) else: unmodified_nodes.append(node) self.verbose('node "%s" unmodified' % node.name) # fill node list container processed_slug_list.append(node.slug) # delete old nodes for local_node in layer_nodes_slug_list: # if local node not found in external nodes if local_node not in processed_slug_list: # retrieve from DB and delete node = Node.objects.get(slug=local_node) # store node name to print it later node_name = node.name node.delete() # then increment count that will be included in message deleted_nodes_count = deleted_nodes_count + 1 self.verbose('node "%s" deleted' % node_name) # message that will be returned self.message = """ %s nodes added %s nodes changed %s nodes deleted %s nodes unmodified %s total external records processed %s total local nodes for this layer """ % ( len(added_nodes), len(changed_nodes), deleted_nodes_count, len(unmodified_nodes), len(items), Node.objects.filter(layer=self.layer).count() )