def test_openlabor_add_node(self): layer = Layer.objects.external()[0] layer.minimum_distance = 0 layer.area = None layer.new_nodes_allowed = True layer.save() layer = Layer.objects.get(pk=layer.pk) url = 'http://devopenlabor.lynxlab.com/api/v1' external = LayerExternal(layer=layer) external.interoperability = 'nodeshot.interoperability.synchronizers.OpenLabor' external.config = json.dumps({ "open311_url": url, "service_code_get": "001", "service_code_post": "002", "default_status": "active", "api_key": "DEVO1395445966" }) external.full_clean() external.save() node = Node() node.name = 'offerta di lavoro di test' node.description = 'altra offerta di lavoro inserita automaticamente tramite unit test' node.geometry = 'POINT (12.5823391919000012 41.8721429276999820)' node.layer = layer node.user_id = 1 node.address = 'via del test' node.data = { "professional_profile": "professional_profile test", "qualification_required": "qualification_required test", "contract_type": "contract_type test", "zip_code": "zip code test", "city": "city test" } node.save() self.assertIsNotNone(node.external.external_id)
def import_nodes(self): """ import nodes into local DB """ self.message('saving nodes into local DB...') saved_nodes = [] # loop over all old node and create new nodes for old_node in self.old_nodes: # if this old node is unconfirmed skip to next cycle if old_node.status == 'u': continue try: node = Node.objects.get(pk=old_node.id) except Node.DoesNotExist: node = Node(id=old_node.id) node.data = {} node.user_id = self.users_dict[old_node.email]['id'] node.name = old_node.name node.slug = old_node.slug node.geometry = Point(old_node.lng, old_node.lat) node.elev = old_node.alt node.description = old_node.description node.notes = old_node.notes node.added = old_node.added node.updated = old_node.updated intersecting_layers = node.intersecting_layers # if more than one intersecting layer if len(intersecting_layers) > 1: # prompt user answer = self.prompt_layer_selection(node, intersecting_layers) if isinstance(answer, int): node.layer_id = answer elif answer == 'default' and self.default_layer is not False: node.layer_id = self.default_layer else: self.message('Node %s discarded' % node.name) continue # if one intersecting layer select that elif 2 > len(intersecting_layers) > 0: node.layer = intersecting_layers[0] # if no intersecting layers else: if self.default_layer is False: # discard node if no default layer specified self.message("""Node %s discarded because is not contained in any specified layer and no default layer specified""" % node.name) continue else: node.layer_id = self.default_layer if old_node.postal_code: # additional info node.data['postal_code'] = old_node.postal_code # is it a hotspot? if old_node.status in ['h', 'ah']: node.data['is_hotspot'] = 'true' # determine status according to settings if self.status_mapping: node.status_id = self.get_status(old_node.status) try: node.full_clean() node.save(auto_update=False) saved_nodes.append(node) self.verbose('Saved node %s in layer %s with status %s' % (node.name, node.layer, node.status.name)) except Exception: tb = traceback.format_exc() self.message('Could not save node %s, got exception:\n\n%s' % (node.name, tb)) self.message('saved %d nodes into local DB' % len(saved_nodes)) self.saved_nodes = saved_nodes
def save(self): """ synchronize DB """ # retrieve all items items = self.parsed_data.getElementsByTagName('AccessPoint') # init empty lists added_nodes = [] changed_nodes = [] unmodified_nodes = [] # retrieve a list of local nodes in DB local_nodes_slug = Node.objects.filter(layer=self.layer).values_list( 'slug', flat=True) # init empty list of slug of external nodes that will be needed to perform delete operations external_nodes_slug = [] deleted_nodes_count = 0 try: self.status = Status.objects.get( slug=self.config.get('status', None)) except Status.DoesNotExist: self.status = None # loop over every parsed item for item in items: # retrieve info in auxiliary variables # readability counts! name = self.get_text(item, 'Denominazione')[0:70] slug = slugify(name) number = 1 original_name = name needed_different_name = False while True: # items might have the same name... so we add a number.. if slug in external_nodes_slug: needed_different_name = True number = number + 1 name = "%s - %d" % (original_name, number) slug = slug = slugify(name) else: if needed_different_name: self.verbose( 'needed a different name for %s, trying "%s"' % (original_name, name)) break lat = self.get_text(item, 'Latitudine') lng = self.get_text(item, 'longitudine') description = 'Indirizzo: %s, %s; Tipologia: %s' % (self.get_text( item, 'Indirizzo'), self.get_text( item, 'Comune'), self.get_text(item, 'Tipologia')) address = '%s, %s' % (self.get_text( item, 'Indirizzo'), self.get_text(item, 'Comune')) # point object point = Point(float(lng), float(lat)) # default values added = False changed = False try: # edit existing node node = Node.objects.get(slug=slug) except Node.DoesNotExist: # add a new node node = Node() node.layer = self.layer node.status = self.status added = True if node.name != name: node.name = name changed = True if node.slug != slug: node.slug = slug changed = True if added is True or node.geometry.equals(point) is False: node.geometry = point changed = True if node.description != description: node.description = description changed = True if node.address != address: node.address = address # complete address node.data = { 'address': self.get_text(item, 'Indirizzo'), 'city': self.get_text(item, 'Comune'), 'province': 'Roma', 'country': 'Italia' } changed = True # perform save or update only if necessary if added or changed: try: node.full_clean() node.save() except ValidationError as e: # TODO: are we sure we want to interrupt the execution? raise Exception("%s errors: %s" % (name, e.messages)) if added: added_nodes.append(node) self.verbose('new node saved with name "%s"' % node.name) elif changed: changed_nodes.append(node) self.verbose('node "%s" updated' % node.name) else: unmodified_nodes.append(node) self.verbose('node "%s" unmodified' % node.name) # fill node list container external_nodes_slug.append(node.slug) # delete old nodes for local_node in local_nodes_slug: # if local node not found in external nodes if not local_node in external_nodes_slug: node_name = node.name # retrieve from DB and delete node = Node.objects.get(slug=local_node) node.delete() # then increment count that will be included in message deleted_nodes_count = deleted_nodes_count + 1 self.verbose('node "%s" deleted' % node_name) # message that will be returned self.message = """ %s nodes added %s nodes changed %s nodes deleted %s nodes unmodified %s total external records processed %s total local nodes for this layer """ % (len(added_nodes), len(changed_nodes), deleted_nodes_count, len(unmodified_nodes), len(items), Node.objects.filter(layer=self.layer).count())
def save(self): """ save data into DB: 1. save new (missing) data 2. update only when needed 3. delete old data 4. generate report that will be printed constraints: * ensure new nodes do not take a name/slug which is already used * validate through django before saving * use good defaults """ self.key_mapping() # retrieve all items items = self.parsed_data # init empty lists added_nodes = [] changed_nodes = [] unmodified_nodes = [] # retrieve a list of all the slugs of this layer layer_nodes_slug_list = Node.objects.filter(layer=self.layer).values_list('slug', flat=True) # keep a list of all the nodes of other layers other_layers_slug_list = Node.objects.exclude(layer=self.layer).values_list('slug', flat=True) # init empty list of slug of external nodes that will be needed to perform delete operations processed_slug_list = [] deleted_nodes_count = 0 # loop over every item for item in items: item = self._convert_item(item) number = 1 original_name = item['name'] needed_different_name = False while True: # items might have the same name... so we add a number.. if item['slug'] in processed_slug_list or item['slug'] in other_layers_slug_list: needed_different_name = True number = number + 1 item['name'] = "%s - %d" % (original_name, number) item['slug'] = slugify(item['name']) else: if needed_different_name: self.verbose('needed a different name for %s, trying "%s"' % (original_name, item['name'])) break # default values added = False changed = False try: # edit existing node node = Node.objects.get(slug=item['slug'], layer=self.layer) except Node.DoesNotExist: # add a new node node = Node() node.layer = self.layer added = True # loop over fields and store data only if necessary for field in Node._meta.fields: # geometry is a special case, skip if field.name == 'geometry': continue # skip if field is not present in values if field.name not in item.keys(): continue # shortcut for value value = item[field.name] # if value is different than what we have if getattr(node, field.name) != value and value is not None: # set value setattr(node, field.name, value) # indicates that a DB query is necessary changed = True if added or (node.geometry.equals(item['geometry']) is False and node.geometry.equals_exact(item['geometry']) is False): node.geometry = item['geometry'] changed = True node.data = node.data or {} # store any additional key/value in HStore data field for key, value in item['data'].items(): if node.data[key] != value: node.data[key] = value changed = True # perform save or update only if necessary if added or changed: try: node.full_clean() if None not in [node.added, node.updated]: node.save(auto_update=False) else: node.save() except Exception as e: raise Exception('error while processing "%s": %s' % (node.name, e)) if added: added_nodes.append(node) self.verbose('new node saved with name "%s"' % node.name) elif changed: changed_nodes.append(node) self.verbose('node "%s" updated' % node.name) else: unmodified_nodes.append(node) self.verbose('node "%s" unmodified' % node.name) # fill node list container processed_slug_list.append(node.slug) # delete old nodes for local_node in layer_nodes_slug_list: # if local node not found in external nodes if local_node not in processed_slug_list: # retrieve from DB and delete node = Node.objects.get(slug=local_node) # store node name to print it later node_name = node.name node.delete() # then increment count that will be included in message deleted_nodes_count = deleted_nodes_count + 1 self.verbose('node "%s" deleted' % node_name) # message that will be returned self.message = """ %s nodes added %s nodes changed %s nodes deleted %s nodes unmodified %s total external records processed %s total local nodes for this layer """ % ( len(added_nodes), len(changed_nodes), deleted_nodes_count, len(unmodified_nodes), len(items), Node.objects.filter(layer=self.layer).count() )
def save(self): """ save data into DB: 1. save new (missing) data 2. update only when needed 3. delete old data 4. generate report that will be printed constraints: * ensure new nodes do not take a name/slug which is already used * validate through django before saving * use good defaults """ self.key_mapping() # retrieve all items items = self.parsed_data # init empty lists added_nodes = [] changed_nodes = [] unmodified_nodes = [] # retrieve a list of all the slugs of this layer layer_nodes_slug_list = Node.objects.filter( layer=self.layer).values_list('slug', flat=True) # keep a list of all the nodes of other layers other_layers_slug_list = Node.objects.exclude( layer=self.layer).values_list('slug', flat=True) # init empty list of slug of external nodes that will be needed to perform delete operations processed_slug_list = [] deleted_nodes_count = 0 # loop over every item for item in items: item = self._convert_item(item) number = 1 original_name = item['name'] needed_different_name = False while True: # items might have the same name... so we add a number.. if item['slug'] in processed_slug_list or item[ 'slug'] in other_layers_slug_list: needed_different_name = True number = number + 1 item['name'] = "%s - %d" % (original_name, number) item['slug'] = slugify(item['name']) else: if needed_different_name: self.verbose( 'needed a different name for %s, trying "%s"' % (original_name, item['name'])) break # default values added = False changed = False try: # edit existing node node = Node.objects.get(slug=item['slug'], layer=self.layer) except Node.DoesNotExist: # add a new node node = Node() node.layer = self.layer added = True # loop over fields and store data only if necessary for field in Node._meta.fields: # geometry is a special case, skip if field.name == 'geometry': continue # skip if field is not present in values if field.name not in item.keys(): continue # shortcut for value value = item[field.name] # if value is different than what we have if value is not None and getattr(node, field.name) != value: # set value setattr(node, field.name, value) # indicates that a DB query is necessary changed = True if added or ( node.geometry.equals(item['geometry']) is False and node.geometry.equals_exact(item['geometry']) is False): node.geometry = item['geometry'] changed = True node.data = node.data or {} # store any additional key/value in HStore data field for key, value in item['data'].items(): if node.data[key] != value: node.data[key] = value changed = True # perform save or update only if necessary if added or changed: try: node.full_clean() if None not in [node.added, node.updated]: node.save(auto_update=False) else: node.save() except Exception as e: raise Exception('error while processing "%s": %s' % (node.name, e)) if added: added_nodes.append(node) self.verbose('new node saved with name "%s"' % node.name) elif changed: changed_nodes.append(node) self.verbose('node "%s" updated' % node.name) else: unmodified_nodes.append(node) self.verbose('node "%s" unmodified' % node.name) # fill node list container processed_slug_list.append(node.slug) # delete old nodes for local_node in layer_nodes_slug_list: # if local node not found in external nodes if local_node not in processed_slug_list: # retrieve from DB and delete node = Node.objects.get(slug=local_node) # store node name to print it later node_name = node.name node.delete() # then increment count that will be included in message deleted_nodes_count = deleted_nodes_count + 1 self.verbose('node "%s" deleted' % node_name) # message that will be returned self.message = """ %s nodes added %s nodes changed %s nodes deleted %s nodes unmodified %s total external records processed %s total local nodes for this layer """ % (len(added_nodes), len(changed_nodes), deleted_nodes_count, len(unmodified_nodes), len(items), Node.objects.filter(layer=self.layer).count())
def process_borders(self,borders): if not borders: self.message = """ Borders data not processed. """ return False # retrieve all items items = borders # init empty lists added_nodes = [] changed_nodes = [] unmodified_nodes = [] # retrieve a list of local nodes in DB local_nodes_slug = Node.objects.filter(layer=self.layer).values_list('slug', flat=True) # init empty list of slug of external nodes that will be needed to perform delete operations external_nodes_slug = [] deleted_nodes_count = 0 try: self.status = Status.objects.get(slug=self.config.get('status', None)) except Status.DoesNotExist: self.status = None # loop over every parsed item for item in items: # retrieve info in auxiliary variables # readability counts! name = item['properties'].get('name', '')[0:70] address = name slug = slugify(name) #print(slug) number = 1 original_name = name needed_different_name = False while True: # items might have the same name... so we add a number.. if slug in external_nodes_slug: needed_different_name = True number = number + 1 name = "%s - %d" % (original_name, number) slug = slug = slugify(name) else: if needed_different_name: self.verbose('needed a different name for %s, trying "%s"' % (original_name, name)) break # geometry object geometry = GEOSGeometry(json.dumps(item["geometry"])) # default values added = False changed = False try: # edit existing node node = Node.objects.get(slug=slug) except Node.DoesNotExist: # add a new node node = Node() node.layer = self.layer node.status = self.status node.data = {} added = True if node.name != name: node.name = name changed = True if node.slug != slug: node.slug = slug changed = True if added is True or node.geometry.equals(geometry) is False: node.geometry = geometry changed = True if node.address != address: node.address = address changed = True # perform save or update only if necessary if added or changed: try: node.full_clean() node.save() except ValidationError as e: # TODO: are we sure we want to interrupt the execution? raise Exception("%s import errors: %s" % (name, e.messages)) if added: added_nodes.append(node) self.verbose('new node saved with name "%s"' % node.name) elif changed: changed_nodes.append(node) self.verbose('node "%s" updated' % node.name) else: unmodified_nodes.append(node) self.verbose('node "%s" unmodified' % node.name) # fill node list container external_nodes_slug.append(node.slug) # delete old nodes for local_node in local_nodes_slug: # if local node not found in external nodes if not local_node in external_nodes_slug: node_name = node.name # retrieve from DB and delete node = Node.objects.get(slug=local_node) node.delete() # then increment count that will be included in message deleted_nodes_count = deleted_nodes_count + 1 self.verbose('node "%s" deleted' % node_name) self.layer.external.config = json.dumps(self.config, indent=4, sort_keys=True) self.layer.external.save() # message that will be returned self.message = """ %s node added %s node changed %s node deleted %s node unmodified %s total external records processed %s total local records for this layer """ % ( len(added_nodes), len(changed_nodes), deleted_nodes_count, len(unmodified_nodes), len(items), Node.objects.filter(layer=self.layer).count() )
def import_nodes(self): """ import nodes into local DB """ self.message('saving nodes into local DB...') saved_nodes = [] # loop over all old node and create new nodes for old_node in self.old_nodes: # if this old node is unconfirmed skip to next cycle if old_node.status == 'u': continue try: node = Node.objects.get(pk=old_node.id) except Node.DoesNotExist: node = Node(id=old_node.id) node.data = {} node.user_id = self.users_dict[old_node.email]['id'] node.name = old_node.name node.slug = old_node.slug node.geometry = Point(old_node.lng, old_node.lat) node.elev = old_node.alt node.description = old_node.description node.notes = old_node.notes node.added = old_node.added node.updated = old_node.updated node.data['imported'] = 'true' intersecting_layers = node.intersecting_layers # if more than one intersecting layer if len(intersecting_layers) > 1: # prompt user answer = self.prompt_layer_selection(node, intersecting_layers) if isinstance(answer, int): node.layer_id = answer elif answer == 'default' and self.default_layer is not False: node.layer_id = self.default_layer else: self.message('Node %s discarded' % node.name) continue # if one intersecting layer select that elif 2 > len(intersecting_layers) > 0: node.layer = intersecting_layers[0] # if no intersecting layers else: if self.default_layer is False: # discard node if no default layer specified self.message("""Node %s discarded because is not contained in any specified layer and no default layer specified""" % node.name) continue else: node.layer_id = self.default_layer if old_node.postal_code: # additional info node.data['postal_code'] = old_node.postal_code # is it a hotspot? if old_node.status in ['h', 'ah']: node.data['is_hotspot'] = 'true' # determine status according to settings if self.status_mapping: node.status_id = self.get_status(old_node.status) try: node.full_clean() node.save(auto_update=False) saved_nodes.append(node) self.verbose('Saved node %s in layer %s with status %s' % (node.name, node.layer, node.status.name)) except Exception: tb = traceback.format_exc() self.message('Could not save node %s, got exception:\n\n%s' % (node.name, tb)) self.message('saved %d nodes into local DB' % len(saved_nodes)) self.saved_nodes = saved_nodes
def import_nodes(self): """ import nodes into local DB """ self.message('saving nodes into local DB...') saved_nodes = [] # loop over all old node and create new nodes for old_node in self.old_nodes: # if this old node is unconfirmed skip to next cycle if old_node.status == 'u': continue node = Node(**{ "id": old_node.id, "user_id": self.users_dict[old_node.email]['id'], "name": old_node.name, "slug": old_node.slug, "geometry": Point(old_node.lng, old_node.lat), "elev": old_node.alt, "description": old_node.description, "notes": old_node.notes, "added": old_node.added, "updated": old_node.updated, "data": {} }) if LAYER_APP_INSTALLED: intersecting_layers = node.intersecting_layers # if more than one intersecting layer if len(intersecting_layers) > 1: # prompt user answer = self.prompt_layer_selection(node, intersecting_layers) if isinstance(answer, int): node.layer_id = answer elif answer == 'default' and self.default_layer is not False: node.layer_id = self.default_layer else: self.message('Node %s discarded' % node.name) continue # if one intersecting layer select that elif 2 > len(intersecting_layers) > 0: node.layer = intersecting_layers[0] # if no intersecting layers else: if self.default_layer is False: # discard node if no default layer specified self.message("""Node %s discarded because is not contained in any specified layer and no default layer specified""" % node.name) continue else: node.layer_id = self.default_layer if old_node.postal_code: # additional info node.data['postal_code'] = old_node.postal_code # is it a hotspot? if old_node.status in ['h', 'ah']: node.data['is_hotspot'] = 'true' # determine status according to settings if self.status_mapping: node.status_id = self.get_status(old_node.status) try: node.full_clean() node.save(auto_update=False) saved_nodes.append(node) self.verbose('Saved node %s in layer %s with status %s' % (node.name, node.layer, node.status.name)) except Exception as e: self.message('Could not save node %s, got exception: %s' % (node.name, e)) self.message('saved %d nodes into local DB' % len(saved_nodes)) self.saved_nodes = saved_nodes
def save(self): """ synchronize DB """ # retrieve all items items = self.parsed_data.getElementsByTagName('AccessPoint') # init empty lists added_nodes = [] changed_nodes = [] unmodified_nodes = [] # retrieve a list of local nodes in DB local_nodes_slug = Node.objects.filter(layer=self.layer).values_list('slug', flat=True) # init empty list of slug of external nodes that will be needed to perform delete operations external_nodes_slug = [] deleted_nodes_count = 0 try: self.status = Status.objects.get(slug=self.config.get('status', None)) except Status.DoesNotExist: self.status = None # loop over every parsed item for item in items: # retrieve info in auxiliary variables # readability counts! name = self.get_text(item, 'Denominazione')[0:70] slug = slugify(name) number = 1 original_name = name needed_different_name = False while True: # items might have the same name... so we add a number.. if slug in external_nodes_slug: needed_different_name = True number = number + 1 name = "%s - %d" % (original_name, number) slug = slug = slugify(name) else: if needed_different_name: self.verbose('needed a different name for %s, trying "%s"' % (original_name, name)) break lat = self.get_text(item, 'Latitudine') lng = self.get_text(item, 'longitudine') description = 'Indirizzo: %s, %s; Tipologia: %s' % ( self.get_text(item, 'Indirizzo'), self.get_text(item, 'Comune'), self.get_text(item, 'Tipologia') ) address = '%s, %s' % ( self.get_text(item, 'Indirizzo'), self.get_text(item, 'Comune') ) # point object point = Point(float(lng), float(lat)) # default values added = False changed = False try: # edit existing node node = Node.objects.get(slug=slug) except Node.DoesNotExist: # add a new node node = Node() node.layer = self.layer node.status = self.status added = True if node.name != name: node.name = name changed = True if node.slug != slug: node.slug = slug changed = True if added is True or node.geometry.equals(point) is False: node.geometry = point changed = True if node.description != description: node.description = description changed = True if node.address != address: node.address = address # complete address node.data = { 'address': self.get_text(item, 'Indirizzo'), 'city': self.get_text(item, 'Comune'), 'province': 'Roma', 'country': 'Italia' } changed = True # perform save or update only if necessary if added or changed: try: node.full_clean() node.save() except ValidationError as e: # TODO: are we sure we want to interrupt the execution? raise Exception("%s errors: %s" % (name, e.messages)) if added: added_nodes.append(node) self.verbose('new node saved with name "%s"' % node.name) elif changed: changed_nodes.append(node) self.verbose('node "%s" updated' % node.name) else: unmodified_nodes.append(node) self.verbose('node "%s" unmodified' % node.name) # fill node list container external_nodes_slug.append(node.slug) # delete old nodes for local_node in local_nodes_slug: # if local node not found in external nodes if not local_node in external_nodes_slug: node_name = node.name # retrieve from DB and delete node = Node.objects.get(slug=local_node) node.delete() # then increment count that will be included in message deleted_nodes_count = deleted_nodes_count + 1 self.verbose('node "%s" deleted' % node_name) # message that will be returned self.message = """ %s nodes added %s nodes changed %s nodes deleted %s nodes unmodified %s total external records processed %s total local nodes for this layer """ % ( len(added_nodes), len(changed_nodes), deleted_nodes_count, len(unmodified_nodes), len(items), Node.objects.filter(layer=self.layer).count() )
def process_streets(self): if not self.streets: self.message = """ Street data not processed. """ return False # retrieve all items items = self.streets # init empty lists added_nodes = [] changed_nodes = [] unmodified_nodes = [] # retrieve a list of local nodes in DB local_nodes_slug = Node.objects.filter(layer=self.layer).values_list( 'slug', flat=True) # init empty list of slug of external nodes that will be needed to perform delete operations external_nodes_slug = [] deleted_nodes_count = 0 try: self.status = Status.objects.get( slug=self.config.get('status', None)) except Status.DoesNotExist: self.status = None # loop over every parsed item for item in items: # retrieve info in auxiliary variables # readability counts! pk = item['id'] name = item['properties'].get('LOCATION', '')[0:70] address = name slug = slugify(name) number = 1 original_name = name needed_different_name = False while True: # items might have the same name... so we add a number.. # check in DB too # TODO: this must be DRYED!! if slug in external_nodes_slug or Node.objects.filter( slug__exact=slug).exclude(pk=pk).count() > 0: needed_different_name = True number = number + 1 name = "%s - %d" % (original_name, number) slug = slug = slugify(name) else: if needed_different_name: self.verbose( 'needed a different name for %s, trying "%s"' % (original_name, name)) break # geometry object geometry = GEOSGeometry(json.dumps(item["geometry"])) # default values added = False changed = False try: # edit existing node node = Node.objects.get(pk=pk) except Node.DoesNotExist: # add a new node node = Node() node.id = pk node.layer = self.layer node.status = self.status node.data = {} added = True if node.name != name: node.name = name changed = True if node.slug != slug: node.slug = slug changed = True if added is True or node.geometry.equals(geometry) is False: node.geometry = geometry changed = True if node.address != address: node.address = address changed = True # perform save or update only if necessary if added or changed: try: node.full_clean() node.save() except ValidationError as e: # TODO: are we sure we want to interrupt the execution? raise Exception("%s errors: %s" % (name, e.messages)) if added: added_nodes.append(node) self.verbose('new node saved with name "%s"' % node.name) elif changed: changed_nodes.append(node) self.verbose('node "%s" updated' % node.name) else: unmodified_nodes.append(node) self.verbose('node "%s" unmodified' % node.name) # fill node list container external_nodes_slug.append(node.slug) # delete old nodes for local_node in local_nodes_slug: # if local node not found in external nodes if not local_node in external_nodes_slug: node_name = node.name # retrieve from DB and delete node = Node.objects.get(slug=local_node) node.delete() # then increment count that will be included in message deleted_nodes_count = deleted_nodes_count + 1 self.verbose('node "%s" deleted' % node_name) self.config['last_time_streets_checked'] = str(date.today()) self.layer.external.config = json.dumps(self.config, indent=4, sort_keys=True) self.layer.external.save() # message that will be returned self.message = """ %s streets added %s streets changed %s streets deleted %s streets unmodified %s total external records processed %s total local records for this layer """ % (len(added_nodes), len(changed_nodes), deleted_nodes_count, len(unmodified_nodes), len(items), Node.objects.filter(layer=self.layer).count())