def test_node_count_change(self): """ when a new node is added nodes count should be updated """ new_node = Node(user_id=4, name='new node', slug='new_node', layer_id=1, coords="POINT (41.9720419277 12.5822391919)") new_node.save() self.assertEqual(new_node.user.stats.potential_nodes, 2, "Potential nodes increment count failed") new_node.status = NODE_STATUS.get('active') new_node.save() self.assertEqual(new_node.user.stats.active_nodes, 4, "Active nodes increment count failed") self.assertEqual(new_node.user.stats.potential_nodes, 1, "Potential nodes decrement count failed") #new_node.is_hotspot = True #new_node.save() #self.assertEqual(new_node.user.stats.hotspots, 2, "Hotspot increment count failed") new_node.delete() user = User.objects.get(username='******') self.assertEqual(user.stats.active_nodes, 3, "Active nodes decrement count failed")
def save(self): """ synchronize DB """ # retrieve all items items = self.parsed_data.getElementsByTagName('AccessPoint') # init empty lists added_nodes = [] changed_nodes = [] unmodified_nodes = [] # retrieve a list of local nodes in DB local_nodes_slug = Node.objects.filter(layer=self.layer).values_list( 'slug', flat=True) # init empty list of slug of external nodes that will be needed to perform delete operations external_nodes_slug = [] deleted_nodes_count = 0 try: self.status = Status.objects.get( slug=self.config.get('status', None)) except Status.DoesNotExist: self.status = None # loop over every parsed item for item in items: # retrieve info in auxiliary variables # readability counts! name = self.get_text(item, 'Denominazione')[0:70] slug = slugify(name) number = 1 original_name = name needed_different_name = False while True: # items might have the same name... so we add a number.. if slug in external_nodes_slug: needed_different_name = True number = number + 1 name = "%s - %d" % (original_name, number) slug = slug = slugify(name) else: if needed_different_name: self.verbose( 'needed a different name for %s, trying "%s"' % (original_name, name)) break lat = self.get_text(item, 'Latitudine') lng = self.get_text(item, 'longitudine') description = 'Indirizzo: %s, %s; Tipologia: %s' % (self.get_text( item, 'Indirizzo'), self.get_text( item, 'Comune'), self.get_text(item, 'Tipologia')) address = '%s, %s' % (self.get_text( item, 'Indirizzo'), self.get_text(item, 'Comune')) # point object point = Point(float(lng), float(lat)) # default values added = False changed = False try: # edit existing node node = Node.objects.get(slug=slug) except Node.DoesNotExist: # add a new node node = Node() node.layer = self.layer node.status = self.status added = True if node.name != name: node.name = name changed = True if node.slug != slug: node.slug = slug changed = True if added is True or node.geometry.equals(point) is False: node.geometry = point changed = True if node.description != description: node.description = description changed = True if node.address != address: node.address = address # complete address node.data = { 'address': self.get_text(item, 'Indirizzo'), 'city': self.get_text(item, 'Comune'), 'province': 'Roma', 'country': 'Italia' } changed = True # perform save or update only if necessary if added or changed: try: node.full_clean() node.save() except ValidationError as e: # TODO: are we sure we want to interrupt the execution? raise Exception("%s errors: %s" % (name, e.messages)) if added: added_nodes.append(node) self.verbose('new node saved with name "%s"' % node.name) elif changed: changed_nodes.append(node) self.verbose('node "%s" updated' % node.name) else: unmodified_nodes.append(node) self.verbose('node "%s" unmodified' % node.name) # fill node list container external_nodes_slug.append(node.slug) # delete old nodes for local_node in local_nodes_slug: # if local node not found in external nodes if not local_node in external_nodes_slug: node_name = node.name # retrieve from DB and delete node = Node.objects.get(slug=local_node) node.delete() # then increment count that will be included in message deleted_nodes_count = deleted_nodes_count + 1 self.verbose('node "%s" deleted' % node_name) # message that will be returned self.message = """ %s nodes added %s nodes changed %s nodes deleted %s nodes unmodified %s total external records processed %s total local nodes for this layer """ % (len(added_nodes), len(changed_nodes), deleted_nodes_count, len(unmodified_nodes), len(items), Node.objects.filter(layer=self.layer).count())
def save(self): """ save data into DB: 1. save new (missing) data 2. update only when needed 3. delete old data 4. generate report that will be printed constraints: * ensure new nodes do not take a name/slug which is already used * validate through django before saving * use good defaults """ self.key_mapping() # retrieve all items items = self.parsed_data # init empty lists added_nodes = [] changed_nodes = [] unmodified_nodes = [] # retrieve a list of all the slugs of this layer layer_nodes_slug_list = Node.objects.filter(layer=self.layer).values_list('slug', flat=True) # keep a list of all the nodes of other layers other_layers_slug_list = Node.objects.exclude(layer=self.layer).values_list('slug', flat=True) # init empty list of slug of external nodes that will be needed to perform delete operations processed_slug_list = [] deleted_nodes_count = 0 # loop over every item for item in items: item = self._convert_item(item) number = 1 original_name = item['name'] needed_different_name = False while True: # items might have the same name... so we add a number.. if item['slug'] in processed_slug_list or item['slug'] in other_layers_slug_list: needed_different_name = True number = number + 1 item['name'] = "%s - %d" % (original_name, number) item['slug'] = slugify(item['name']) else: if needed_different_name: self.verbose('needed a different name for %s, trying "%s"' % (original_name, item['name'])) break # default values added = False changed = False try: # edit existing node node = Node.objects.get(slug=item['slug'], layer=self.layer) except Node.DoesNotExist: # add a new node node = Node() node.layer = self.layer added = True # loop over fields and store data only if necessary for field in Node._meta.fields: # geometry is a special case, skip if field.name == 'geometry': continue # skip if field is not present in values if field.name not in item.keys(): continue # shortcut for value value = item[field.name] # if value is different than what we have if getattr(node, field.name) != value and value is not None: # set value setattr(node, field.name, value) # indicates that a DB query is necessary changed = True if added or (node.geometry.equals(item['geometry']) is False and node.geometry.equals_exact(item['geometry']) is False): node.geometry = item['geometry'] changed = True node.data = node.data or {} # store any additional key/value in HStore data field for key, value in item['data'].items(): if node.data[key] != value: node.data[key] = value changed = True # perform save or update only if necessary if added or changed: try: node.full_clean() if None not in [node.added, node.updated]: node.save(auto_update=False) else: node.save() except Exception as e: raise Exception('error while processing "%s": %s' % (node.name, e)) if added: added_nodes.append(node) self.verbose('new node saved with name "%s"' % node.name) elif changed: changed_nodes.append(node) self.verbose('node "%s" updated' % node.name) else: unmodified_nodes.append(node) self.verbose('node "%s" unmodified' % node.name) # fill node list container processed_slug_list.append(node.slug) # delete old nodes for local_node in layer_nodes_slug_list: # if local node not found in external nodes if local_node not in processed_slug_list: # retrieve from DB and delete node = Node.objects.get(slug=local_node) # store node name to print it later node_name = node.name node.delete() # then increment count that will be included in message deleted_nodes_count = deleted_nodes_count + 1 self.verbose('node "%s" deleted' % node_name) # message that will be returned self.message = """ %s nodes added %s nodes changed %s nodes deleted %s nodes unmodified %s total external records processed %s total local nodes for this layer """ % ( len(added_nodes), len(changed_nodes), deleted_nodes_count, len(unmodified_nodes), len(items), Node.objects.filter(layer=self.layer).count() )
def save(self): """ save data into DB: 1. save new (missing) data 2. update only when needed 3. delete old data 4. generate report that will be printed constraints: * ensure new nodes do not take a name/slug which is already used * validate through django before saving * use good defaults """ self.key_mapping() # retrieve all items items = self.parsed_data # init empty lists added_nodes = [] changed_nodes = [] unmodified_nodes = [] # retrieve a list of all the slugs of this layer layer_nodes_slug_list = Node.objects.filter( layer=self.layer).values_list('slug', flat=True) # keep a list of all the nodes of other layers other_layers_slug_list = Node.objects.exclude( layer=self.layer).values_list('slug', flat=True) # init empty list of slug of external nodes that will be needed to perform delete operations processed_slug_list = [] deleted_nodes_count = 0 # loop over every item for item in items: item = self._convert_item(item) number = 1 original_name = item['name'] needed_different_name = False while True: # items might have the same name... so we add a number.. if item['slug'] in processed_slug_list or item[ 'slug'] in other_layers_slug_list: needed_different_name = True number = number + 1 item['name'] = "%s - %d" % (original_name, number) item['slug'] = slugify(item['name']) else: if needed_different_name: self.verbose( 'needed a different name for %s, trying "%s"' % (original_name, item['name'])) break # default values added = False changed = False try: # edit existing node node = Node.objects.get(slug=item['slug'], layer=self.layer) except Node.DoesNotExist: # add a new node node = Node() node.layer = self.layer added = True # loop over fields and store data only if necessary for field in Node._meta.fields: # geometry is a special case, skip if field.name == 'geometry': continue # skip if field is not present in values if field.name not in item.keys(): continue # shortcut for value value = item[field.name] # if value is different than what we have if value is not None and getattr(node, field.name) != value: # set value setattr(node, field.name, value) # indicates that a DB query is necessary changed = True if added or ( node.geometry.equals(item['geometry']) is False and node.geometry.equals_exact(item['geometry']) is False): node.geometry = item['geometry'] changed = True node.data = node.data or {} # store any additional key/value in HStore data field for key, value in item['data'].items(): if node.data[key] != value: node.data[key] = value changed = True # perform save or update only if necessary if added or changed: try: node.full_clean() if None not in [node.added, node.updated]: node.save(auto_update=False) else: node.save() except Exception as e: raise Exception('error while processing "%s": %s' % (node.name, e)) if added: added_nodes.append(node) self.verbose('new node saved with name "%s"' % node.name) elif changed: changed_nodes.append(node) self.verbose('node "%s" updated' % node.name) else: unmodified_nodes.append(node) self.verbose('node "%s" unmodified' % node.name) # fill node list container processed_slug_list.append(node.slug) # delete old nodes for local_node in layer_nodes_slug_list: # if local node not found in external nodes if local_node not in processed_slug_list: # retrieve from DB and delete node = Node.objects.get(slug=local_node) # store node name to print it later node_name = node.name node.delete() # then increment count that will be included in message deleted_nodes_count = deleted_nodes_count + 1 self.verbose('node "%s" deleted' % node_name) # message that will be returned self.message = """ %s nodes added %s nodes changed %s nodes deleted %s nodes unmodified %s total external records processed %s total local nodes for this layer """ % (len(added_nodes), len(changed_nodes), deleted_nodes_count, len(unmodified_nodes), len(items), Node.objects.filter(layer=self.layer).count())
def process_borders(self,borders): if not borders: self.message = """ Borders data not processed. """ return False # retrieve all items items = borders # init empty lists added_nodes = [] changed_nodes = [] unmodified_nodes = [] # retrieve a list of local nodes in DB local_nodes_slug = Node.objects.filter(layer=self.layer).values_list('slug', flat=True) # init empty list of slug of external nodes that will be needed to perform delete operations external_nodes_slug = [] deleted_nodes_count = 0 try: self.status = Status.objects.get(slug=self.config.get('status', None)) except Status.DoesNotExist: self.status = None # loop over every parsed item for item in items: # retrieve info in auxiliary variables # readability counts! name = item['properties'].get('name', '')[0:70] address = name slug = slugify(name) #print(slug) number = 1 original_name = name needed_different_name = False while True: # items might have the same name... so we add a number.. if slug in external_nodes_slug: needed_different_name = True number = number + 1 name = "%s - %d" % (original_name, number) slug = slug = slugify(name) else: if needed_different_name: self.verbose('needed a different name for %s, trying "%s"' % (original_name, name)) break # geometry object geometry = GEOSGeometry(json.dumps(item["geometry"])) # default values added = False changed = False try: # edit existing node node = Node.objects.get(slug=slug) except Node.DoesNotExist: # add a new node node = Node() node.layer = self.layer node.status = self.status node.data = {} added = True if node.name != name: node.name = name changed = True if node.slug != slug: node.slug = slug changed = True if added is True or node.geometry.equals(geometry) is False: node.geometry = geometry changed = True if node.address != address: node.address = address changed = True # perform save or update only if necessary if added or changed: try: node.full_clean() node.save() except ValidationError as e: # TODO: are we sure we want to interrupt the execution? raise Exception("%s import errors: %s" % (name, e.messages)) if added: added_nodes.append(node) self.verbose('new node saved with name "%s"' % node.name) elif changed: changed_nodes.append(node) self.verbose('node "%s" updated' % node.name) else: unmodified_nodes.append(node) self.verbose('node "%s" unmodified' % node.name) # fill node list container external_nodes_slug.append(node.slug) # delete old nodes for local_node in local_nodes_slug: # if local node not found in external nodes if not local_node in external_nodes_slug: node_name = node.name # retrieve from DB and delete node = Node.objects.get(slug=local_node) node.delete() # then increment count that will be included in message deleted_nodes_count = deleted_nodes_count + 1 self.verbose('node "%s" deleted' % node_name) self.layer.external.config = json.dumps(self.config, indent=4, sort_keys=True) self.layer.external.save() # message that will be returned self.message = """ %s node added %s node changed %s node deleted %s node unmodified %s total external records processed %s total local records for this layer """ % ( len(added_nodes), len(changed_nodes), deleted_nodes_count, len(unmodified_nodes), len(items), Node.objects.filter(layer=self.layer).count() )
def save(self): """ synchronize DB """ # retrieve all items items = self.parsed_data.getElementsByTagName('AccessPoint') # init empty lists added_nodes = [] changed_nodes = [] unmodified_nodes = [] # retrieve a list of local nodes in DB local_nodes_slug = Node.objects.filter(layer=self.layer).values_list('slug', flat=True) # init empty list of slug of external nodes that will be needed to perform delete operations external_nodes_slug = [] deleted_nodes_count = 0 try: self.status = Status.objects.get(slug=self.config.get('status', None)) except Status.DoesNotExist: self.status = None # loop over every parsed item for item in items: # retrieve info in auxiliary variables # readability counts! name = self.get_text(item, 'Denominazione')[0:70] slug = slugify(name) number = 1 original_name = name needed_different_name = False while True: # items might have the same name... so we add a number.. if slug in external_nodes_slug: needed_different_name = True number = number + 1 name = "%s - %d" % (original_name, number) slug = slug = slugify(name) else: if needed_different_name: self.verbose('needed a different name for %s, trying "%s"' % (original_name, name)) break lat = self.get_text(item, 'Latitudine') lng = self.get_text(item, 'longitudine') description = 'Indirizzo: %s, %s; Tipologia: %s' % ( self.get_text(item, 'Indirizzo'), self.get_text(item, 'Comune'), self.get_text(item, 'Tipologia') ) address = '%s, %s' % ( self.get_text(item, 'Indirizzo'), self.get_text(item, 'Comune') ) # point object point = Point(float(lng), float(lat)) # default values added = False changed = False try: # edit existing node node = Node.objects.get(slug=slug) except Node.DoesNotExist: # add a new node node = Node() node.layer = self.layer node.status = self.status added = True if node.name != name: node.name = name changed = True if node.slug != slug: node.slug = slug changed = True if added is True or node.geometry.equals(point) is False: node.geometry = point changed = True if node.description != description: node.description = description changed = True if node.address != address: node.address = address # complete address node.data = { 'address': self.get_text(item, 'Indirizzo'), 'city': self.get_text(item, 'Comune'), 'province': 'Roma', 'country': 'Italia' } changed = True # perform save or update only if necessary if added or changed: try: node.full_clean() node.save() except ValidationError as e: # TODO: are we sure we want to interrupt the execution? raise Exception("%s errors: %s" % (name, e.messages)) if added: added_nodes.append(node) self.verbose('new node saved with name "%s"' % node.name) elif changed: changed_nodes.append(node) self.verbose('node "%s" updated' % node.name) else: unmodified_nodes.append(node) self.verbose('node "%s" unmodified' % node.name) # fill node list container external_nodes_slug.append(node.slug) # delete old nodes for local_node in local_nodes_slug: # if local node not found in external nodes if not local_node in external_nodes_slug: node_name = node.name # retrieve from DB and delete node = Node.objects.get(slug=local_node) node.delete() # then increment count that will be included in message deleted_nodes_count = deleted_nodes_count + 1 self.verbose('node "%s" deleted' % node_name) # message that will be returned self.message = """ %s nodes added %s nodes changed %s nodes deleted %s nodes unmodified %s total external records processed %s total local nodes for this layer """ % ( len(added_nodes), len(changed_nodes), deleted_nodes_count, len(unmodified_nodes), len(items), Node.objects.filter(layer=self.layer).count() )
def process_streets(self): if not self.streets: self.message = """ Street data not processed. """ return False # retrieve all items items = self.streets # init empty lists added_nodes = [] changed_nodes = [] unmodified_nodes = [] # retrieve a list of local nodes in DB local_nodes_slug = Node.objects.filter(layer=self.layer).values_list( 'slug', flat=True) # init empty list of slug of external nodes that will be needed to perform delete operations external_nodes_slug = [] deleted_nodes_count = 0 try: self.status = Status.objects.get( slug=self.config.get('status', None)) except Status.DoesNotExist: self.status = None # loop over every parsed item for item in items: # retrieve info in auxiliary variables # readability counts! pk = item['id'] name = item['properties'].get('LOCATION', '')[0:70] address = name slug = slugify(name) number = 1 original_name = name needed_different_name = False while True: # items might have the same name... so we add a number.. # check in DB too # TODO: this must be DRYED!! if slug in external_nodes_slug or Node.objects.filter( slug__exact=slug).exclude(pk=pk).count() > 0: needed_different_name = True number = number + 1 name = "%s - %d" % (original_name, number) slug = slug = slugify(name) else: if needed_different_name: self.verbose( 'needed a different name for %s, trying "%s"' % (original_name, name)) break # geometry object geometry = GEOSGeometry(json.dumps(item["geometry"])) # default values added = False changed = False try: # edit existing node node = Node.objects.get(pk=pk) except Node.DoesNotExist: # add a new node node = Node() node.id = pk node.layer = self.layer node.status = self.status node.data = {} added = True if node.name != name: node.name = name changed = True if node.slug != slug: node.slug = slug changed = True if added is True or node.geometry.equals(geometry) is False: node.geometry = geometry changed = True if node.address != address: node.address = address changed = True # perform save or update only if necessary if added or changed: try: node.full_clean() node.save() except ValidationError as e: # TODO: are we sure we want to interrupt the execution? raise Exception("%s errors: %s" % (name, e.messages)) if added: added_nodes.append(node) self.verbose('new node saved with name "%s"' % node.name) elif changed: changed_nodes.append(node) self.verbose('node "%s" updated' % node.name) else: unmodified_nodes.append(node) self.verbose('node "%s" unmodified' % node.name) # fill node list container external_nodes_slug.append(node.slug) # delete old nodes for local_node in local_nodes_slug: # if local node not found in external nodes if not local_node in external_nodes_slug: node_name = node.name # retrieve from DB and delete node = Node.objects.get(slug=local_node) node.delete() # then increment count that will be included in message deleted_nodes_count = deleted_nodes_count + 1 self.verbose('node "%s" deleted' % node_name) self.config['last_time_streets_checked'] = str(date.today()) self.layer.external.config = json.dumps(self.config, indent=4, sort_keys=True) self.layer.external.save() # message that will be returned self.message = """ %s streets added %s streets changed %s streets deleted %s streets unmodified %s total external records processed %s total local records for this layer """ % (len(added_nodes), len(changed_nodes), deleted_nodes_count, len(unmodified_nodes), len(items), Node.objects.filter(layer=self.layer).count())