def test_openlabor_add_node(self): layer = Layer.objects.external()[0] layer.minimum_distance = 0 layer.area = None layer.new_nodes_allowed = True layer.save() layer = Layer.objects.get(pk=layer.pk) url = 'http://devopenlabor.lynxlab.com/api/v1' external = LayerExternal(layer=layer) external.interoperability = 'nodeshot.interoperability.synchronizers.OpenLabor' external.config = json.dumps({ "open311_url": url, "service_code_get": "001", "service_code_post": "002", "default_status": "active", "api_key": "DEVO1395445966" }) external.full_clean() external.save() node = Node() node.name = 'offerta di lavoro di test' node.description = 'altra offerta di lavoro inserita automaticamente tramite unit test' node.geometry = 'POINT (12.5823391919000012 41.8721429276999820)' node.layer = layer node.user_id = 1 node.address = 'via del test' node.data = { "professional_profile": "professional_profile test", "qualification_required": "qualification_required test", "contract_type": "contract_type test", "zip_code": "zip code test", "city": "city test" } node.save() self.assertIsNotNone(node.external.external_id)
def save(self): """ synchronize DB """ # retrieve all items items = self.parsed_data.getElementsByTagName('AccessPoint') # init empty lists added_nodes = [] changed_nodes = [] unmodified_nodes = [] # retrieve a list of local nodes in DB local_nodes_slug = Node.objects.filter(layer=self.layer).values_list( 'slug', flat=True) # init empty list of slug of external nodes that will be needed to perform delete operations external_nodes_slug = [] deleted_nodes_count = 0 try: self.status = Status.objects.get( slug=self.config.get('status', None)) except Status.DoesNotExist: self.status = None # loop over every parsed item for item in items: # retrieve info in auxiliary variables # readability counts! name = self.get_text(item, 'Denominazione')[0:70] slug = slugify(name) number = 1 original_name = name needed_different_name = False while True: # items might have the same name... so we add a number.. if slug in external_nodes_slug: needed_different_name = True number = number + 1 name = "%s - %d" % (original_name, number) slug = slug = slugify(name) else: if needed_different_name: self.verbose( 'needed a different name for %s, trying "%s"' % (original_name, name)) break lat = self.get_text(item, 'Latitudine') lng = self.get_text(item, 'longitudine') description = 'Indirizzo: %s, %s; Tipologia: %s' % (self.get_text( item, 'Indirizzo'), self.get_text( item, 'Comune'), self.get_text(item, 'Tipologia')) address = '%s, %s' % (self.get_text( item, 'Indirizzo'), self.get_text(item, 'Comune')) # point object point = Point(float(lng), float(lat)) # default values added = False changed = False try: # edit existing node node = Node.objects.get(slug=slug) except Node.DoesNotExist: # add a new node node = Node() node.layer = self.layer node.status = self.status added = True if node.name != name: node.name = name changed = True if node.slug != slug: node.slug = slug changed = True if added is True or node.geometry.equals(point) is False: node.geometry = point changed = True if node.description != description: node.description = description changed = True if node.address != address: node.address = address # complete address node.data = { 'address': self.get_text(item, 'Indirizzo'), 'city': self.get_text(item, 'Comune'), 'province': 'Roma', 'country': 'Italia' } changed = True # perform save or update only if necessary if added or changed: try: node.full_clean() node.save() except ValidationError as e: # TODO: are we sure we want to interrupt the execution? raise Exception("%s errors: %s" % (name, e.messages)) if added: added_nodes.append(node) self.verbose('new node saved with name "%s"' % node.name) elif changed: changed_nodes.append(node) self.verbose('node "%s" updated' % node.name) else: unmodified_nodes.append(node) self.verbose('node "%s" unmodified' % node.name) # fill node list container external_nodes_slug.append(node.slug) # delete old nodes for local_node in local_nodes_slug: # if local node not found in external nodes if not local_node in external_nodes_slug: node_name = node.name # retrieve from DB and delete node = Node.objects.get(slug=local_node) node.delete() # then increment count that will be included in message deleted_nodes_count = deleted_nodes_count + 1 self.verbose('node "%s" deleted' % node_name) # message that will be returned self.message = """ %s nodes added %s nodes changed %s nodes deleted %s nodes unmodified %s total external records processed %s total local nodes for this layer """ % (len(added_nodes), len(changed_nodes), deleted_nodes_count, len(unmodified_nodes), len(items), Node.objects.filter(layer=self.layer).count())
def process_borders(self,borders): if not borders: self.message = """ Borders data not processed. """ return False # retrieve all items items = borders # init empty lists added_nodes = [] changed_nodes = [] unmodified_nodes = [] # retrieve a list of local nodes in DB local_nodes_slug = Node.objects.filter(layer=self.layer).values_list('slug', flat=True) # init empty list of slug of external nodes that will be needed to perform delete operations external_nodes_slug = [] deleted_nodes_count = 0 try: self.status = Status.objects.get(slug=self.config.get('status', None)) except Status.DoesNotExist: self.status = None # loop over every parsed item for item in items: # retrieve info in auxiliary variables # readability counts! name = item['properties'].get('name', '')[0:70] address = name slug = slugify(name) #print(slug) number = 1 original_name = name needed_different_name = False while True: # items might have the same name... so we add a number.. if slug in external_nodes_slug: needed_different_name = True number = number + 1 name = "%s - %d" % (original_name, number) slug = slug = slugify(name) else: if needed_different_name: self.verbose('needed a different name for %s, trying "%s"' % (original_name, name)) break # geometry object geometry = GEOSGeometry(json.dumps(item["geometry"])) # default values added = False changed = False try: # edit existing node node = Node.objects.get(slug=slug) except Node.DoesNotExist: # add a new node node = Node() node.layer = self.layer node.status = self.status node.data = {} added = True if node.name != name: node.name = name changed = True if node.slug != slug: node.slug = slug changed = True if added is True or node.geometry.equals(geometry) is False: node.geometry = geometry changed = True if node.address != address: node.address = address changed = True # perform save or update only if necessary if added or changed: try: node.full_clean() node.save() except ValidationError as e: # TODO: are we sure we want to interrupt the execution? raise Exception("%s import errors: %s" % (name, e.messages)) if added: added_nodes.append(node) self.verbose('new node saved with name "%s"' % node.name) elif changed: changed_nodes.append(node) self.verbose('node "%s" updated' % node.name) else: unmodified_nodes.append(node) self.verbose('node "%s" unmodified' % node.name) # fill node list container external_nodes_slug.append(node.slug) # delete old nodes for local_node in local_nodes_slug: # if local node not found in external nodes if not local_node in external_nodes_slug: node_name = node.name # retrieve from DB and delete node = Node.objects.get(slug=local_node) node.delete() # then increment count that will be included in message deleted_nodes_count = deleted_nodes_count + 1 self.verbose('node "%s" deleted' % node_name) self.layer.external.config = json.dumps(self.config, indent=4, sort_keys=True) self.layer.external.save() # message that will be returned self.message = """ %s node added %s node changed %s node deleted %s node unmodified %s total external records processed %s total local records for this layer """ % ( len(added_nodes), len(changed_nodes), deleted_nodes_count, len(unmodified_nodes), len(items), Node.objects.filter(layer=self.layer).count() )
def save(self): """ synchronize DB """ # retrieve all items items = self.parsed_data.getElementsByTagName('AccessPoint') # init empty lists added_nodes = [] changed_nodes = [] unmodified_nodes = [] # retrieve a list of local nodes in DB local_nodes_slug = Node.objects.filter(layer=self.layer).values_list('slug', flat=True) # init empty list of slug of external nodes that will be needed to perform delete operations external_nodes_slug = [] deleted_nodes_count = 0 try: self.status = Status.objects.get(slug=self.config.get('status', None)) except Status.DoesNotExist: self.status = None # loop over every parsed item for item in items: # retrieve info in auxiliary variables # readability counts! name = self.get_text(item, 'Denominazione')[0:70] slug = slugify(name) number = 1 original_name = name needed_different_name = False while True: # items might have the same name... so we add a number.. if slug in external_nodes_slug: needed_different_name = True number = number + 1 name = "%s - %d" % (original_name, number) slug = slug = slugify(name) else: if needed_different_name: self.verbose('needed a different name for %s, trying "%s"' % (original_name, name)) break lat = self.get_text(item, 'Latitudine') lng = self.get_text(item, 'longitudine') description = 'Indirizzo: %s, %s; Tipologia: %s' % ( self.get_text(item, 'Indirizzo'), self.get_text(item, 'Comune'), self.get_text(item, 'Tipologia') ) address = '%s, %s' % ( self.get_text(item, 'Indirizzo'), self.get_text(item, 'Comune') ) # point object point = Point(float(lng), float(lat)) # default values added = False changed = False try: # edit existing node node = Node.objects.get(slug=slug) except Node.DoesNotExist: # add a new node node = Node() node.layer = self.layer node.status = self.status added = True if node.name != name: node.name = name changed = True if node.slug != slug: node.slug = slug changed = True if added is True or node.geometry.equals(point) is False: node.geometry = point changed = True if node.description != description: node.description = description changed = True if node.address != address: node.address = address # complete address node.data = { 'address': self.get_text(item, 'Indirizzo'), 'city': self.get_text(item, 'Comune'), 'province': 'Roma', 'country': 'Italia' } changed = True # perform save or update only if necessary if added or changed: try: node.full_clean() node.save() except ValidationError as e: # TODO: are we sure we want to interrupt the execution? raise Exception("%s errors: %s" % (name, e.messages)) if added: added_nodes.append(node) self.verbose('new node saved with name "%s"' % node.name) elif changed: changed_nodes.append(node) self.verbose('node "%s" updated' % node.name) else: unmodified_nodes.append(node) self.verbose('node "%s" unmodified' % node.name) # fill node list container external_nodes_slug.append(node.slug) # delete old nodes for local_node in local_nodes_slug: # if local node not found in external nodes if not local_node in external_nodes_slug: node_name = node.name # retrieve from DB and delete node = Node.objects.get(slug=local_node) node.delete() # then increment count that will be included in message deleted_nodes_count = deleted_nodes_count + 1 self.verbose('node "%s" deleted' % node_name) # message that will be returned self.message = """ %s nodes added %s nodes changed %s nodes deleted %s nodes unmodified %s total external records processed %s total local nodes for this layer """ % ( len(added_nodes), len(changed_nodes), deleted_nodes_count, len(unmodified_nodes), len(items), Node.objects.filter(layer=self.layer).count() )
def process_streets(self): if not self.streets: self.message = """ Street data not processed. """ return False # retrieve all items items = self.streets # init empty lists added_nodes = [] changed_nodes = [] unmodified_nodes = [] # retrieve a list of local nodes in DB local_nodes_slug = Node.objects.filter(layer=self.layer).values_list( 'slug', flat=True) # init empty list of slug of external nodes that will be needed to perform delete operations external_nodes_slug = [] deleted_nodes_count = 0 try: self.status = Status.objects.get( slug=self.config.get('status', None)) except Status.DoesNotExist: self.status = None # loop over every parsed item for item in items: # retrieve info in auxiliary variables # readability counts! pk = item['id'] name = item['properties'].get('LOCATION', '')[0:70] address = name slug = slugify(name) number = 1 original_name = name needed_different_name = False while True: # items might have the same name... so we add a number.. # check in DB too # TODO: this must be DRYED!! if slug in external_nodes_slug or Node.objects.filter( slug__exact=slug).exclude(pk=pk).count() > 0: needed_different_name = True number = number + 1 name = "%s - %d" % (original_name, number) slug = slug = slugify(name) else: if needed_different_name: self.verbose( 'needed a different name for %s, trying "%s"' % (original_name, name)) break # geometry object geometry = GEOSGeometry(json.dumps(item["geometry"])) # default values added = False changed = False try: # edit existing node node = Node.objects.get(pk=pk) except Node.DoesNotExist: # add a new node node = Node() node.id = pk node.layer = self.layer node.status = self.status node.data = {} added = True if node.name != name: node.name = name changed = True if node.slug != slug: node.slug = slug changed = True if added is True or node.geometry.equals(geometry) is False: node.geometry = geometry changed = True if node.address != address: node.address = address changed = True # perform save or update only if necessary if added or changed: try: node.full_clean() node.save() except ValidationError as e: # TODO: are we sure we want to interrupt the execution? raise Exception("%s errors: %s" % (name, e.messages)) if added: added_nodes.append(node) self.verbose('new node saved with name "%s"' % node.name) elif changed: changed_nodes.append(node) self.verbose('node "%s" updated' % node.name) else: unmodified_nodes.append(node) self.verbose('node "%s" unmodified' % node.name) # fill node list container external_nodes_slug.append(node.slug) # delete old nodes for local_node in local_nodes_slug: # if local node not found in external nodes if not local_node in external_nodes_slug: node_name = node.name # retrieve from DB and delete node = Node.objects.get(slug=local_node) node.delete() # then increment count that will be included in message deleted_nodes_count = deleted_nodes_count + 1 self.verbose('node "%s" deleted' % node_name) self.config['last_time_streets_checked'] = str(date.today()) self.layer.external.config = json.dumps(self.config, indent=4, sort_keys=True) self.layer.external.save() # message that will be returned self.message = """ %s streets added %s streets changed %s streets deleted %s streets unmodified %s total external records processed %s total local records for this layer """ % (len(added_nodes), len(changed_nodes), deleted_nodes_count, len(unmodified_nodes), len(items), Node.objects.filter(layer=self.layer).count())