def import_nodes(self): """ import nodes into local DB """ self.message('saving nodes into local DB...') saved_nodes = [] # loop over all old node and create new nodes for old_node in self.old_nodes: # if this old node is unconfirmed skip to next cycle if old_node.status == 'u': continue try: node = Node.objects.get(pk=old_node.id) except Node.DoesNotExist: node = Node(id=old_node.id) node.data = {} node.user_id = self.users_dict[old_node.email]['id'] node.name = old_node.name node.slug = old_node.slug node.geometry = Point(old_node.lng, old_node.lat) node.elev = old_node.alt node.description = old_node.description node.notes = old_node.notes node.added = old_node.added node.updated = old_node.updated intersecting_layers = node.intersecting_layers # if more than one intersecting layer if len(intersecting_layers) > 1: # prompt user answer = self.prompt_layer_selection(node, intersecting_layers) if isinstance(answer, int): node.layer_id = answer elif answer == 'default' and self.default_layer is not False: node.layer_id = self.default_layer else: self.message('Node %s discarded' % node.name) continue # if one intersecting layer select that elif 2 > len(intersecting_layers) > 0: node.layer = intersecting_layers[0] # if no intersecting layers else: if self.default_layer is False: # discard node if no default layer specified self.message("""Node %s discarded because is not contained in any specified layer and no default layer specified""" % node.name) continue else: node.layer_id = self.default_layer if old_node.postal_code: # additional info node.data['postal_code'] = old_node.postal_code # is it a hotspot? if old_node.status in ['h', 'ah']: node.data['is_hotspot'] = 'true' # determine status according to settings if self.status_mapping: node.status_id = self.get_status(old_node.status) try: node.full_clean() node.save(auto_update=False) saved_nodes.append(node) self.verbose('Saved node %s in layer %s with status %s' % (node.name, node.layer, node.status.name)) except Exception: tb = traceback.format_exc() self.message('Could not save node %s, got exception:\n\n%s' % (node.name, tb)) self.message('saved %d nodes into local DB' % len(saved_nodes)) self.saved_nodes = saved_nodes
def save(self): """ synchronize DB """ # retrieve all items items = self.parsed_data.getElementsByTagName('AccessPoint') # init empty lists added_nodes = [] changed_nodes = [] unmodified_nodes = [] # retrieve a list of local nodes in DB local_nodes_slug = Node.objects.filter(layer=self.layer).values_list( 'slug', flat=True) # init empty list of slug of external nodes that will be needed to perform delete operations external_nodes_slug = [] deleted_nodes_count = 0 try: self.status = Status.objects.get( slug=self.config.get('status', None)) except Status.DoesNotExist: self.status = None # loop over every parsed item for item in items: # retrieve info in auxiliary variables # readability counts! name = self.get_text(item, 'Denominazione')[0:70] slug = slugify(name) number = 1 original_name = name needed_different_name = False while True: # items might have the same name... so we add a number.. if slug in external_nodes_slug: needed_different_name = True number = number + 1 name = "%s - %d" % (original_name, number) slug = slug = slugify(name) else: if needed_different_name: self.verbose( 'needed a different name for %s, trying "%s"' % (original_name, name)) break lat = self.get_text(item, 'Latitudine') lng = self.get_text(item, 'longitudine') description = 'Indirizzo: %s, %s; Tipologia: %s' % (self.get_text( item, 'Indirizzo'), self.get_text( item, 'Comune'), self.get_text(item, 'Tipologia')) address = '%s, %s' % (self.get_text( item, 'Indirizzo'), self.get_text(item, 'Comune')) # point object point = Point(float(lng), float(lat)) # default values added = False changed = False try: # edit existing node node = Node.objects.get(slug=slug) except Node.DoesNotExist: # add a new node node = Node() node.layer = self.layer node.status = self.status added = True if node.name != name: node.name = name changed = True if node.slug != slug: node.slug = slug changed = True if added is True or node.geometry.equals(point) is False: node.geometry = point changed = True if node.description != description: node.description = description changed = True if node.address != address: node.address = address # complete address node.data = { 'address': self.get_text(item, 'Indirizzo'), 'city': self.get_text(item, 'Comune'), 'province': 'Roma', 'country': 'Italia' } changed = True # perform save or update only if necessary if added or changed: try: node.full_clean() node.save() except ValidationError as e: # TODO: are we sure we want to interrupt the execution? raise Exception("%s errors: %s" % (name, e.messages)) if added: added_nodes.append(node) self.verbose('new node saved with name "%s"' % node.name) elif changed: changed_nodes.append(node) self.verbose('node "%s" updated' % node.name) else: unmodified_nodes.append(node) self.verbose('node "%s" unmodified' % node.name) # fill node list container external_nodes_slug.append(node.slug) # delete old nodes for local_node in local_nodes_slug: # if local node not found in external nodes if not local_node in external_nodes_slug: node_name = node.name # retrieve from DB and delete node = Node.objects.get(slug=local_node) node.delete() # then increment count that will be included in message deleted_nodes_count = deleted_nodes_count + 1 self.verbose('node "%s" deleted' % node_name) # message that will be returned self.message = """ %s nodes added %s nodes changed %s nodes deleted %s nodes unmodified %s total external records processed %s total local nodes for this layer """ % (len(added_nodes), len(changed_nodes), deleted_nodes_count, len(unmodified_nodes), len(items), Node.objects.filter(layer=self.layer).count())
def process_borders(self,borders): if not borders: self.message = """ Borders data not processed. """ return False # retrieve all items items = borders # init empty lists added_nodes = [] changed_nodes = [] unmodified_nodes = [] # retrieve a list of local nodes in DB local_nodes_slug = Node.objects.filter(layer=self.layer).values_list('slug', flat=True) # init empty list of slug of external nodes that will be needed to perform delete operations external_nodes_slug = [] deleted_nodes_count = 0 try: self.status = Status.objects.get(slug=self.config.get('status', None)) except Status.DoesNotExist: self.status = None # loop over every parsed item for item in items: # retrieve info in auxiliary variables # readability counts! name = item['properties'].get('name', '')[0:70] address = name slug = slugify(name) #print(slug) number = 1 original_name = name needed_different_name = False while True: # items might have the same name... so we add a number.. if slug in external_nodes_slug: needed_different_name = True number = number + 1 name = "%s - %d" % (original_name, number) slug = slug = slugify(name) else: if needed_different_name: self.verbose('needed a different name for %s, trying "%s"' % (original_name, name)) break # geometry object geometry = GEOSGeometry(json.dumps(item["geometry"])) # default values added = False changed = False try: # edit existing node node = Node.objects.get(slug=slug) except Node.DoesNotExist: # add a new node node = Node() node.layer = self.layer node.status = self.status node.data = {} added = True if node.name != name: node.name = name changed = True if node.slug != slug: node.slug = slug changed = True if added is True or node.geometry.equals(geometry) is False: node.geometry = geometry changed = True if node.address != address: node.address = address changed = True # perform save or update only if necessary if added or changed: try: node.full_clean() node.save() except ValidationError as e: # TODO: are we sure we want to interrupt the execution? raise Exception("%s import errors: %s" % (name, e.messages)) if added: added_nodes.append(node) self.verbose('new node saved with name "%s"' % node.name) elif changed: changed_nodes.append(node) self.verbose('node "%s" updated' % node.name) else: unmodified_nodes.append(node) self.verbose('node "%s" unmodified' % node.name) # fill node list container external_nodes_slug.append(node.slug) # delete old nodes for local_node in local_nodes_slug: # if local node not found in external nodes if not local_node in external_nodes_slug: node_name = node.name # retrieve from DB and delete node = Node.objects.get(slug=local_node) node.delete() # then increment count that will be included in message deleted_nodes_count = deleted_nodes_count + 1 self.verbose('node "%s" deleted' % node_name) self.layer.external.config = json.dumps(self.config, indent=4, sort_keys=True) self.layer.external.save() # message that will be returned self.message = """ %s node added %s node changed %s node deleted %s node unmodified %s total external records processed %s total local records for this layer """ % ( len(added_nodes), len(changed_nodes), deleted_nodes_count, len(unmodified_nodes), len(items), Node.objects.filter(layer=self.layer).count() )
def import_nodes(self): """ import nodes into local DB """ self.message('saving nodes into local DB...') saved_nodes = [] # loop over all old node and create new nodes for old_node in self.old_nodes: # if this old node is unconfirmed skip to next cycle if old_node.status == 'u': continue try: node = Node.objects.get(pk=old_node.id) except Node.DoesNotExist: node = Node(id=old_node.id) node.data = {} node.user_id = self.users_dict[old_node.email]['id'] node.name = old_node.name node.slug = old_node.slug node.geometry = Point(old_node.lng, old_node.lat) node.elev = old_node.alt node.description = old_node.description node.notes = old_node.notes node.added = old_node.added node.updated = old_node.updated node.data['imported'] = 'true' intersecting_layers = node.intersecting_layers # if more than one intersecting layer if len(intersecting_layers) > 1: # prompt user answer = self.prompt_layer_selection(node, intersecting_layers) if isinstance(answer, int): node.layer_id = answer elif answer == 'default' and self.default_layer is not False: node.layer_id = self.default_layer else: self.message('Node %s discarded' % node.name) continue # if one intersecting layer select that elif 2 > len(intersecting_layers) > 0: node.layer = intersecting_layers[0] # if no intersecting layers else: if self.default_layer is False: # discard node if no default layer specified self.message("""Node %s discarded because is not contained in any specified layer and no default layer specified""" % node.name) continue else: node.layer_id = self.default_layer if old_node.postal_code: # additional info node.data['postal_code'] = old_node.postal_code # is it a hotspot? if old_node.status in ['h', 'ah']: node.data['is_hotspot'] = 'true' # determine status according to settings if self.status_mapping: node.status_id = self.get_status(old_node.status) try: node.full_clean() node.save(auto_update=False) saved_nodes.append(node) self.verbose('Saved node %s in layer %s with status %s' % (node.name, node.layer, node.status.name)) except Exception: tb = traceback.format_exc() self.message('Could not save node %s, got exception:\n\n%s' % (node.name, tb)) self.message('saved %d nodes into local DB' % len(saved_nodes)) self.saved_nodes = saved_nodes
def save(self): """ synchronize DB """ # retrieve all items items = self.parsed_data.getElementsByTagName('AccessPoint') # init empty lists added_nodes = [] changed_nodes = [] unmodified_nodes = [] # retrieve a list of local nodes in DB local_nodes_slug = Node.objects.filter(layer=self.layer).values_list('slug', flat=True) # init empty list of slug of external nodes that will be needed to perform delete operations external_nodes_slug = [] deleted_nodes_count = 0 try: self.status = Status.objects.get(slug=self.config.get('status', None)) except Status.DoesNotExist: self.status = None # loop over every parsed item for item in items: # retrieve info in auxiliary variables # readability counts! name = self.get_text(item, 'Denominazione')[0:70] slug = slugify(name) number = 1 original_name = name needed_different_name = False while True: # items might have the same name... so we add a number.. if slug in external_nodes_slug: needed_different_name = True number = number + 1 name = "%s - %d" % (original_name, number) slug = slug = slugify(name) else: if needed_different_name: self.verbose('needed a different name for %s, trying "%s"' % (original_name, name)) break lat = self.get_text(item, 'Latitudine') lng = self.get_text(item, 'longitudine') description = 'Indirizzo: %s, %s; Tipologia: %s' % ( self.get_text(item, 'Indirizzo'), self.get_text(item, 'Comune'), self.get_text(item, 'Tipologia') ) address = '%s, %s' % ( self.get_text(item, 'Indirizzo'), self.get_text(item, 'Comune') ) # point object point = Point(float(lng), float(lat)) # default values added = False changed = False try: # edit existing node node = Node.objects.get(slug=slug) except Node.DoesNotExist: # add a new node node = Node() node.layer = self.layer node.status = self.status added = True if node.name != name: node.name = name changed = True if node.slug != slug: node.slug = slug changed = True if added is True or node.geometry.equals(point) is False: node.geometry = point changed = True if node.description != description: node.description = description changed = True if node.address != address: node.address = address # complete address node.data = { 'address': self.get_text(item, 'Indirizzo'), 'city': self.get_text(item, 'Comune'), 'province': 'Roma', 'country': 'Italia' } changed = True # perform save or update only if necessary if added or changed: try: node.full_clean() node.save() except ValidationError as e: # TODO: are we sure we want to interrupt the execution? raise Exception("%s errors: %s" % (name, e.messages)) if added: added_nodes.append(node) self.verbose('new node saved with name "%s"' % node.name) elif changed: changed_nodes.append(node) self.verbose('node "%s" updated' % node.name) else: unmodified_nodes.append(node) self.verbose('node "%s" unmodified' % node.name) # fill node list container external_nodes_slug.append(node.slug) # delete old nodes for local_node in local_nodes_slug: # if local node not found in external nodes if not local_node in external_nodes_slug: node_name = node.name # retrieve from DB and delete node = Node.objects.get(slug=local_node) node.delete() # then increment count that will be included in message deleted_nodes_count = deleted_nodes_count + 1 self.verbose('node "%s" deleted' % node_name) # message that will be returned self.message = """ %s nodes added %s nodes changed %s nodes deleted %s nodes unmodified %s total external records processed %s total local nodes for this layer """ % ( len(added_nodes), len(changed_nodes), deleted_nodes_count, len(unmodified_nodes), len(items), Node.objects.filter(layer=self.layer).count() )
def process_streets(self): if not self.streets: self.message = """ Street data not processed. """ return False # retrieve all items items = self.streets # init empty lists added_nodes = [] changed_nodes = [] unmodified_nodes = [] # retrieve a list of local nodes in DB local_nodes_slug = Node.objects.filter(layer=self.layer).values_list( 'slug', flat=True) # init empty list of slug of external nodes that will be needed to perform delete operations external_nodes_slug = [] deleted_nodes_count = 0 try: self.status = Status.objects.get( slug=self.config.get('status', None)) except Status.DoesNotExist: self.status = None # loop over every parsed item for item in items: # retrieve info in auxiliary variables # readability counts! pk = item['id'] name = item['properties'].get('LOCATION', '')[0:70] address = name slug = slugify(name) number = 1 original_name = name needed_different_name = False while True: # items might have the same name... so we add a number.. # check in DB too # TODO: this must be DRYED!! if slug in external_nodes_slug or Node.objects.filter( slug__exact=slug).exclude(pk=pk).count() > 0: needed_different_name = True number = number + 1 name = "%s - %d" % (original_name, number) slug = slug = slugify(name) else: if needed_different_name: self.verbose( 'needed a different name for %s, trying "%s"' % (original_name, name)) break # geometry object geometry = GEOSGeometry(json.dumps(item["geometry"])) # default values added = False changed = False try: # edit existing node node = Node.objects.get(pk=pk) except Node.DoesNotExist: # add a new node node = Node() node.id = pk node.layer = self.layer node.status = self.status node.data = {} added = True if node.name != name: node.name = name changed = True if node.slug != slug: node.slug = slug changed = True if added is True or node.geometry.equals(geometry) is False: node.geometry = geometry changed = True if node.address != address: node.address = address changed = True # perform save or update only if necessary if added or changed: try: node.full_clean() node.save() except ValidationError as e: # TODO: are we sure we want to interrupt the execution? raise Exception("%s errors: %s" % (name, e.messages)) if added: added_nodes.append(node) self.verbose('new node saved with name "%s"' % node.name) elif changed: changed_nodes.append(node) self.verbose('node "%s" updated' % node.name) else: unmodified_nodes.append(node) self.verbose('node "%s" unmodified' % node.name) # fill node list container external_nodes_slug.append(node.slug) # delete old nodes for local_node in local_nodes_slug: # if local node not found in external nodes if not local_node in external_nodes_slug: node_name = node.name # retrieve from DB and delete node = Node.objects.get(slug=local_node) node.delete() # then increment count that will be included in message deleted_nodes_count = deleted_nodes_count + 1 self.verbose('node "%s" deleted' % node_name) self.config['last_time_streets_checked'] = str(date.today()) self.layer.external.config = json.dumps(self.config, indent=4, sort_keys=True) self.layer.external.save() # message that will be returned self.message = """ %s streets added %s streets changed %s streets deleted %s streets unmodified %s total external records processed %s total local records for this layer """ % (len(added_nodes), len(changed_nodes), deleted_nodes_count, len(unmodified_nodes), len(items), Node.objects.filter(layer=self.layer).count())