def partial_postcode(request, postcode, format='json'): postcode = re.sub(r'\s+', '', postcode.upper()) if is_valid_postcode(postcode): postcode = re.sub(r'\d[A-Z]{2}$', '', postcode) if not is_valid_partial_postcode(postcode): raise ViewException(format, "Partial postcode '%s' is not valid." % postcode, 400) location = Postcode.objects.filter(postcode__startswith=postcode).extra( where=['length(postcode) = %d' % (len(postcode) + 3)]).aggregate( Collect('location'))['location__collect'] if not location: raise ViewException(format, 'Postcode not found', 404) postcode = Postcode(postcode=postcode, location=location.centroid) if format == 'html': return render( request, 'mapit/postcode.html', { 'postcode': postcode.as_dict(), 'json_view': 'mapit-postcode-partial', }) return output_json(postcode.as_dict())
def _get_centre_point(self, postcode): geom = Address.objects.filter( **{self.geom_query: postcode} ).aggregate(Collect('point'))['point__collect'] if geom: centre = geom.centroid.coords return centre
def _area_geometry(area_id): area = get_object_or_404(Area, id=area_id) all_areas = area.polygons.all().aggregate( Collect('polygon'))['polygon__collect'] if not all_areas: return output_json({'error': _('No polygons found')}, code=404) out = { 'parts': all_areas.num_geom, } if settings.MAPIT_AREA_SRID != 4326: out['srid_en'] = settings.MAPIT_AREA_SRID out['area'] = all_areas.area out['min_e'], out['min_n'], out['max_e'], out[ 'max_n'] = all_areas.extent out['centre_e'], out['centre_n'] = all_areas.centroid all_areas.transform(4326) out['min_lon'], out['min_lat'], out['max_lon'], out[ 'max_lat'] = all_areas.extent out['centre_lon'], out['centre_lat'] = all_areas.centroid else: out['min_lon'], out['min_lat'], out['max_lon'], out[ 'max_lat'] = all_areas.extent out['centre_lon'], out['centre_lat'] = all_areas.centroid if hasattr(countries, 'area_geometry_srid'): srid = countries.area_geometry_srid all_areas.transform(srid) out['srid_en'] = srid out['area'] = all_areas.area out['min_e'], out['min_n'], out['max_e'], out[ 'max_n'] = all_areas.extent out['centre_e'], out['centre_n'] = all_areas.centroid return out
def communities_occurring_in_area(self, queryset, name, value): """Return Communities occurring in the given Area. * The filter returns a list of Area objects as ``value`` * We need to extract their PKs to create a queryset equivalent to the list of objects ``value``. Only querysets allow agggregation, not lists. * A search_area Multipolygon is collected from the geoms of Areas in ``value`` * The Taxon PKs are calculated from occurrences (CommunityAreaEncounters) ``intersect``ing the search_area * The queryset is filtered by the list of Community PKs with occurrences in the matching areas """ if value: area_pks = [area.pk for area in value] search_area = Area.objects.filter(pk__in=area_pks).aggregate( Collect('geom'))["geom__collect"] pks = set([ x["community__pk"] for x in occ_models.CommunityAreaEncounter.objects.filter( Q(point__intersects=search_area) | Q(geom__intersects=search_area)).values("community__pk") ]) return queryset.filter(pk__in=pks) else: return queryset
def __collect_polygons(self, area): all_polygons = area.polygons.all() if len(all_polygons) > 1: all_polygons = all_polygons.aggregate(Collect('polygon'))['polygon__collect'] elif len(all_polygons) == 1: all_polygons = all_polygons[0].polygon else: return None return all_polygons
def test14_collect(self): "Testing the `collect` GeoQuerySet method and `Collect` aggregate." # Reference query: # SELECT AsText(ST_Collect("relatedapp_location"."point")) FROM "relatedapp_city" LEFT OUTER JOIN # "relatedapp_location" ON ("relatedapp_city"."location_id" = "relatedapp_location"."id") # WHERE "relatedapp_city"."state" = 'TX'; ref_geom = fromstr('MULTIPOINT(-97.516111 33.058333,-96.801611 32.782057,-95.363151 29.763374,-96.801611 32.782057)') c1 = City.objects.filter(state='TX').collect(field_name='location__point') c2 = City.objects.filter(state='TX').aggregate(Collect('location__point'))['location__point__collect'] for coll in (c1, c2): # Even though Dallas and Ft. Worth share same point, Collect doesn't # consolidate -- that's why 4 points in MultiPoint. self.assertEqual(4, len(coll)) self.assertEqual(ref_geom, coll)
def infer_practice_boundaries(): practices = Practice.objects.filter(location__isnull=False, setting=4).exclude(status_code__in=( Practice.STATUS_RETIRED, Practice.STATUS_DORMANT, Practice.STATUS_CLOSED, )) partition = practices.aggregate(voronoi=Func( Collect("location"), function="ST_VoronoiPolygons"))["voronoi"] national_boundary = get_national_boundary() practice_regions = get_practice_code_to_region_map(partition, national_boundary) with transaction.atomic(): for practice in practices: practice.boundary = practice_regions[practice.code] practice.save(update_fields=["boundary"])
def get_queryset(self): """ - group all geometries together (st_collect) by same variable & value """ qs = NsemPsaContour.objects.filter(nsem_psa_variable__nsem=self.nsem) qs = qs.values(*[ 'value', 'color', 'date', 'nsem_psa_variable__name', 'nsem_psa_variable__data_type', 'nsem_psa_variable__display_name', 'nsem_psa_variable__units', ]) qs = qs.annotate(geom=Collect(Cast('geo', GeometryField()))) qs = qs.order_by('nsem_psa_variable__name') return qs
def get_context_data(self, **kwargs): context = super(SAPlaceDetailSub, self).get_context_data(**kwargs) context['child_place_template'] = self.child_place_template context['child_place_list_template'] = self.child_place_list_template context['subcontent_title'] = 'Constituency Offices' if self.object.kind.slug == 'province': context['child_places'] = ( ZAPlace.objects.filter( kind__slug__in=CONSTITUENCY_OFFICE_PLACE_KIND_SLUGS). filter( # From https://github.com/mysociety/mapit/blob/e79689499cade74bed2d016cb1291c6849c0e8b7/mapit/geometryserialiser.py#L58 location__coveredby=self.object.mapit_area.polygons. aggregate(Collect('polygon'))['polygon__collect'])) return context
def test_collect(self): """ Testing the `Collect` aggregate. """ # Reference query: # SELECT AsText(ST_Collect("relatedapp_location"."point")) FROM "relatedapp_city" LEFT OUTER JOIN # "relatedapp_location" ON ("relatedapp_city"."location_id" = "relatedapp_location"."id") # WHERE "relatedapp_city"."state" = 'TX'; ref_geom = GEOSGeometry( 'MULTIPOINT(-97.516111 33.058333,-96.801611 32.782057,' '-95.363151 29.763374,-96.801611 32.782057)' ) coll = City.objects.filter(state='TX').aggregate(Collect('location__point'))['location__point__collect'] # Even though Dallas and Ft. Worth share same point, Collect doesn't # consolidate -- that's why 4 points in MultiPoint. self.assertEqual(4, len(coll)) self.assertTrue(ref_geom.equals(coll))
def occurring_in_area(self, queryset, name, value): """Return Taxa occurring in the given Area. * The filter returns a list of Area objects as ``value`` * We need to extract their PKs to create a queryset equivalent to the list of objects ``value``. Only querysets allow agggregation, not lists. * A search_area Multipolygon is collected from the geoms of Areas in ``value`` * The queryset is filtered by intersection of its point or geom with the search area """ if value: search_area = Area.objects.filter( pk__in=[area.pk for area in value]).aggregate( Collect('geom'))["geom__collect"] return queryset.filter( Q(point__intersects=search_area) | Q(geom__intersects=search_area)) else: return queryset
def _get_extent_summary(self, object_list): ids = [o.spatial_id for o in object_list] queryset = self.model.objects.filter(spatial_id__in=ids) summary = queryset.aggregate( Collect('outline'), Extent('outline'), ) extents = { 'count': queryset.count(), } if queryset.count(): extents.update({ 'collect': json.loads(summary['outline__collect'].geojson), 'convex_hull': json.loads(summary['outline__collect'].convex_hull.geojson), 'extent': { 'xmin': summary['outline__extent'][0], 'ymin': summary['outline__extent'][1], 'xmax': summary['outline__extent'][2], 'ymax': summary['outline__extent'][3], }, }) return extents
def handle_label(self, directory_name, **options): current_generation = Generation.objects.current() new_generation = Generation.objects.new() if not new_generation: raise Exception("No new generation to be used for import!") if not os.path.isdir(directory_name): raise Exception("'%s' is not a directory" % (directory_name,)) os.chdir(directory_name) mapit_type_glob = smart_text("[A-Z0-9][A-Z0-9][A-Z0-9]") if not glob(mapit_type_glob): raise Exception( "'%s' did not contain any directories that look like MapIt types (e.g. O11, OWA, etc.)" % ( directory_name,)) def verbose(s): if int(options['verbosity']) > 1: print(smart_str(s)) verbose("Loading any admin boundaries from " + directory_name) verbose("Finding language codes...") language_code_to_name = {} code_keys = ('two_letter', 'three_letter') for row in get_iso639_2_table(): english_name = getattr(row, 'english_name') for k in code_keys: code = getattr(row, k) if not code: continue language_code_to_name[code] = english_name global_country = Country.objects.get(code='G') # print json.dumps(language_code_to_name, sort_keys=True, indent=4) skip_up_to = None # skip_up_to = 'relation-80370' skipping = bool(skip_up_to) for type_directory in sorted(glob(mapit_type_glob)): verbose("Loading type " + type_directory) if not os.path.exists(type_directory): verbose("Skipping the non-existent " + type_directory) continue verbose("Loading all KML in " + type_directory) files = sorted(os.listdir(type_directory)) total_files = len(files) for i, e in enumerate(files): progress = "[%d%% complete] " % ((i * 100) / total_files,) if skipping: if skip_up_to in e: skipping = False else: continue if not e.endswith('.kml'): verbose("Ignoring non-KML file: " + e) continue m = re.search(r'^(way|relation)-(\d+)-', e) if not m: raise Exception("Couldn't extract OSM element type and ID from: " + e) osm_type, osm_id = m.groups() kml_filename = os.path.join(type_directory, e) verbose(progress + "Loading " + os.path.realpath(kml_filename)) # Need to parse the KML manually to get the ExtendedData kml_data = KML() xml.sax.parse(smart_str(kml_filename), kml_data) useful_names = [n for n in kml_data.data.keys() if not n.startswith('Boundaries for')] if len(useful_names) == 0: raise Exception("No useful names found in KML data") elif len(useful_names) > 1: raise Exception("Multiple useful names found in KML data") name = useful_names[0] print(smart_str(" %s" % name)) if osm_type == 'relation': code_type_osm = CodeType.objects.get(code='osm_rel') elif osm_type == 'way': code_type_osm = CodeType.objects.get(code='osm_way') else: raise Exception("Unknown OSM element type: " + osm_type) ds = DataSource(kml_filename) layer = ds[0] if len(layer) != 1: raise Exception("We only expect one feature in each layer") feat = layer[1] g = feat.geom.transform(4326, clone=True) if g.geom_count == 0: # Just ignore any KML files that have no polygons in them: verbose(' Ignoring that file - it contained no polygons') continue # Nowadays, in generating the data we should have # excluded any "polygons" with less than four points # (the final one being the same as the first), but # just in case: polygons_too_small = 0 for polygon in g: if polygon.num_points < 4: polygons_too_small += 1 if polygons_too_small: message = "%d out of %d polygon(s) were too small" % (polygons_too_small, g.geom_count) verbose(' Skipping, since ' + message) continue g_geos = g.geos if not g_geos.valid: verbose(" Invalid KML:" + kml_filename) fixed_multipolygon = fix_invalid_geos_multipolygon(g_geos) if len(fixed_multipolygon) == 0: verbose(" Invalid polygons couldn't be fixed") continue g = fixed_multipolygon.ogr area_type = Type.objects.get(code=type_directory) try: osm_code = Code.objects.get(type=code_type_osm, code=osm_id, area__generation_high__lte=current_generation, area__generation_high__gte=current_generation) except Code.DoesNotExist: verbose(' No area existed in the current generation with that OSM element type and ID') osm_code = None was_the_same_in_current = False if osm_code: m = osm_code.area # First, we need to check if the polygons are # still the same as in the previous generation: previous_geos_geometry = m.polygons.aggregate(Collect('polygon'))['polygon__collect'] if previous_geos_geometry is None: verbose(' In the current generation, that area was empty - skipping') else: # Simplify it to make sure the polygons are valid: previous_geos_geometry = shapely.wkb.loads( str(previous_geos_geometry.simplify(tolerance=0).ewkb)) new_geos_geometry = shapely.wkb.loads(str(g.geos.simplify(tolerance=0).ewkb)) if previous_geos_geometry.almost_equals(new_geos_geometry, decimal=7): was_the_same_in_current = True else: verbose(' In the current generation, the boundary was different') if was_the_same_in_current: # Extend the high generation to the new one: verbose(' The boundary was identical in the previous generation; raising generation_high') m.generation_high = new_generation else: # Otherwise, create a completely new area: m = Area( name=name, type=area_type, country=global_country, parent_area=None, generation_low=new_generation, generation_high=new_generation, ) poly = [g] if options['commit']: m.save() verbose(' Area ID: ' + str(m.id)) if name not in kml_data.data: print(json.dumps(kml_data.data, sort_keys=True, indent=4)) raise Exception("Will fail to find '%s' in the dictionary" % (name,)) old_lang_codes = set(n.type.code for n in m.names.all()) for k, translated_name in kml_data.data[name].items(): language_name = None if k == 'name': lang = 'default' language_name = "OSM Default" else: name_match = re.search(r'^name:(.+)$', k) if name_match: lang = name_match.group(1) if lang in language_code_to_name: language_name = language_code_to_name[lang] if not language_name: continue old_lang_codes.discard(lang) # Otherwise, make sure that a NameType for this language exists: NameType.objects.update_or_create(code=lang, defaults={'description': language_name}) name_type = NameType.objects.get(code=lang) m.names.update_or_create(type=name_type, defaults={'name': translated_name}) if old_lang_codes: verbose('Removing deleted languages codes: ' + ' '.join(old_lang_codes)) m.names.filter(type__code__in=old_lang_codes).delete() # If the boundary was the same, the old Code # object will still be pointing to the same Area, # which just had its generation_high incremented. # In every other case, there's a new area object, # so create a new Code and save it: if not was_the_same_in_current: new_code = Code(area=m, type=code_type_osm, code=osm_id) new_code.save() save_polygons({'dummy': (m, poly)})
def handle_label(self, filename, **options): missing_options = [] for k in [ 'generation_id', 'area_type_code', 'name_type_code', 'country_code' ]: if options[k]: continue else: missing_options.append(k) if missing_options: message_start = "Missing arguments " if len( missing_options) > 1 else "Missing argument " message = message_start + " ".join('--{0}'.format(k) for k in missing_options) raise CommandError(message) generation_id = options['generation_id'] area_type_code = options['area_type_code'] name_type_code = options['name_type_code'] country_code = options['country_code'] override_name = options['override_name'] name_field = options['name_field'] if not (override_name or name_field): name_field = 'Name' override_code = options['override_code'] code_field = options['code_field'] code_type_code = options['code_type'] encoding = options['encoding'] or 'utf-8' if name_field and override_name: raise CommandError( "You must not specify both --name_field and --override_name") if code_field and override_code: raise CommandError( "You must not specify both --code_field and --override_code") using_code = (code_field or override_code) if (using_code and not code_type_code) or (not using_code and code_type_code): raise CommandError( "If you want to save a code, specify --code_type and either --code_field or --override_code" ) try: area_type = Type.objects.get(code=area_type_code) except: type_desc = input( 'Please give a description for area type code %s: ' % area_type_code) area_type = Type(code=area_type_code, description=type_desc) if options['commit']: area_type.save() try: name_type = NameType.objects.get(code=name_type_code) except: name_desc = input( 'Please give a description for name type code %s: ' % name_type_code) name_type = NameType(code=name_type_code, description=name_desc) if options['commit']: name_type.save() if country_code != 'first-letter': try: country = Country.objects.get(code=country_code) except: country_name = input( 'Please give the name for country code %s: ' % country_code) country = Country(code=country_code, name=country_name) if options['commit']: country.save() if code_type_code: try: code_type = CodeType.objects.get(code=code_type_code) except: code_desc = input( 'Please give a description for code type %s: ' % code_type_code) code_type = CodeType(code=code_type_code, description=code_desc) if options['commit']: code_type.save() self.stdout.write("Importing from %s" % filename) if not options['commit']: self.stdout.write( '(will not save to db as --commit not specified)') current_generation = Generation.objects.current() new_generation = Generation.objects.get(id=generation_id) def verbose(*args): if int(options['verbosity']) > 1: self.stdout.write(" ".join(str(a) for a in args)) ds = DataSource(filename) layer = ds[0] if (override_name or override_code) and len(layer) > 1: message = ( "Warning: you have specified an override %s and this file contains more than one feature; " "multiple areas with the same %s will be created") if override_name: self.stdout.write(message % ('name', 'name')) if override_code: self.stdout.write(message % ('code', 'code')) for feat in layer: if override_name: name = override_name else: name = None for nf in name_field.split(','): try: name = feat[nf].value break except: pass if name is None: choices = ', '.join(layer.fields) raise CommandError( "Could not find name using name field '%s' - should it be something else? " "It will be one of these: %s. Specify which with --name_field" % (name_field, choices)) try: if not isinstance(name, six.text_type): name = name.decode(encoding) except: raise CommandError( "Could not decode name using encoding '%s' - is it in another encoding? " "Specify one with --encoding" % encoding) name = re.sub(r'\s+', ' ', name) if not name: if options['ignore_blank']: continue raise Exception("Could not find a name to use for area") code = None if override_code: code = override_code elif code_field: try: code = feat[code_field].value except: choices = ', '.join(layer.fields) raise CommandError( "Could not find code using code field '%s' - should it be something else? " "It will be one of these: %s. Specify which with --code_field" % (code_field, choices)) self.stdout.write(" looking at '%s'%s" % (name, (' (%s)' % code) if code else '')) if country_code == 'first-letter' and code: try: country = Country.objects.get(code=code[0]) except Country.DoesNotExist: self.stdout.write(" No country found from first-letter") country = None g = None if hasattr(feat, 'geom'): g = feat.geom.transform(settings.MAPIT_AREA_SRID, clone=True) try: if options['new']: # Always want a new area raise Area.DoesNotExist if code: matching_message = "code %s of code type %s" % (code, code_type) areas = Area.objects.filter( codes__code=code, codes__type=code_type).order_by('-generation_high') else: matching_message = "name %s of area type %s" % (name, area_type) areas = Area.objects.filter( name=name, type=area_type).order_by('-generation_high') if len(areas) == 0: verbose(" the area was not found - creating a new one") raise Area.DoesNotExist m = areas[0] verbose(" found the area") if options['preserve']: # Find whether we need to create a new Area: previous_geos_geometry = m.polygons.aggregate( Collect('polygon'))['polygon__collect'] if m.generation_high < current_generation.id: # Then it was missing in current_generation: verbose( " area existed previously, but was missing from", current_generation) raise Area.DoesNotExist elif g is None: if previous_geos_geometry is not None: verbose(" area is now empty") raise Area.DoesNotExist else: verbose(" the area has remained empty") elif previous_geos_geometry is None: # It was empty in the previous generation: verbose(" area was empty in", current_generation) raise Area.DoesNotExist else: # Otherwise, create a new Area unless the # polygons were the same in current_generation: previous_geos_geometry = previous_geos_geometry.simplify( tolerance=0) new_geos_geometry = g.geos.simplify(tolerance=0) create_new_area = not previous_geos_geometry.equals( new_geos_geometry) p = previous_geos_geometry.sym_difference( new_geos_geometry ).area / previous_geos_geometry.area verbose(" change in area is:", "%.03f%%" % (100 * p, )) if create_new_area: verbose( " the area", m, "has changed, creating a new area due to --preserve" ) raise Area.DoesNotExist else: verbose(" the area remained the same") else: # If --preserve is not specified, the code or the name must be unique: if len(areas) > 1: raise Area.MultipleObjectsReturned( "There was more than one area with %s, and --preserve was not specified" % (matching_message, )) except Area.DoesNotExist: m = Area( name=name, type=area_type, country=country, # parent_area=parent_area, generation_low=new_generation, generation_high=new_generation, ) if options['use_code_as_id'] and code: m.id = int(code) # check that we are not about to skip a generation if m.generation_high and current_generation and m.generation_high.id < current_generation.id: raise Exception( "Area %s found, but not in current generation %s" % (m, current_generation)) m.generation_high = new_generation if options['fix_invalid_polygons'] and g is not None: # Make a GEOS geometry only to check for validity: geos_g = g.geos if not geos_g.valid: geos_g = fix_invalid_geos_geometry(geos_g) if geos_g is None: self.stdout.write( "The geometry for area %s was invalid and couldn't be fixed" % name) g = None else: g = geos_g.ogr poly = [g] if g is not None else [] if options['commit']: m.save() m.names.update_or_create(type=name_type, defaults={'name': name}) if code: m.codes.update_or_create(type=code_type, defaults={'code': code}) save_polygons({m.id: (m, poly)})
def getSpeciesMetadata(self): """ Returns metadata for all species at a specific genus """ species = self.QuerySet.filter(genus_id=self.id).values('species_id').annotate(points=Collect('geom'),ab=Count('species_id'),name=Min('scientific_name')) return species
def generate_clusters(cls, precision, operator=None): if not cls.objects.filter(precision=precision, operator=operator).exists(): # Generate clusters from smaller clusters if 1 <= precision < MAX_CLUSTER_PRECISION_SIZE: print("Generating clusters...") # Get smaller clusters and annotate the new geohash for the bigger clusters smaller_precision = precision + 1 smaller_clusters = cls.objects.filter(precision=smaller_precision, operator=operator)\ .annotate(bigger_geohash=GeoHash('point', precision=precision)) # Group by bigger geohash clusters_hashes = smaller_clusters.values( 'bigger_geohash').distinct() total = clusters_hashes.count() if not total: raise ValueError( "No clusters found for precision {}".format(precision + 1)) print("Saving data for {} clusters...".format(total)) loop_counter = 0 percentage = 0 cluster_array = [] for cluster_dict in clusters_hashes: geohash = cluster_dict['bigger_geohash'] # Get data from smaller clusters sub_clusters = smaller_clusters.filter( bigger_geohash=geohash).values('point', 'count', 'data') count = reduce((lambda acc, cl: acc + cl['count']), sub_clusters, 0) point = Point( reduce((lambda acc, cl: acc + (cl['point'].x * float(cl['count']))), sub_clusters, 0.0) / float(count), reduce((lambda acc, cl: acc + (cl['point'].y * float(cl['count']))), sub_clusters, 0.0) / float(count)) data = '' if count != 1 else sub_clusters[0]['data'] cluster = cls(point=point, precision=precision, count=count, data=data, operator=operator) cluster_array.append(cluster) if len(cluster_array) >= DATABASE_COMMIT_SIZE: cls.objects.bulk_create(cluster_array) cluster_array = [] loop_counter += 1 prev_percentage = percentage percentage = 100 * loop_counter // total if percentage > prev_percentage: print(" {}% done ({} clusters)".format( percentage, loop_counter)) if len(cluster_array) > 0: cls.objects.bulk_create(cluster_array) return # Generate clusters from base stations elif precision == MAX_CLUSTER_PRECISION_SIZE: print("Generating clusters...") # Add geohash to all base stations base_stations = BS_MODEL.objects.annotate( geohash=GeoHash('point', precision=precision)) # Filter by operator if operator: mnc_list = [m.value for m in operator.mnc_set.all()] base_stations = base_stations.filter(mnc__in=mnc_list) # Group by geohash and get cluster MultiPoint and count clusters_values = base_stations.values('geohash').annotate( count=Count('point'), geom=Collect('point')) total = clusters_values.count() if not total: raise ValueError( "No base stations found for precision {}".format( precision)) print("Saving data for {} clusters...".format(total)) loop_counter = 0 percentage = 0 cluster_array = [] for cluster_dict in clusters_values: count = cluster_dict['count'] point = cluster_dict['geom'].centroid data = '' if count != 1 else base_stations.get( geohash=cluster_dict['geohash']).data cluster = cls(point=point, precision=precision, count=count, data=data, operator=operator) cluster_array.append(cluster) if len(cluster_array) >= DATABASE_COMMIT_SIZE: cls.objects.bulk_create(cluster_array) cluster_array = [] loop_counter += 1 prev_percentage = percentage percentage = 100 * loop_counter // total if percentage > prev_percentage: print(" {}% done ({} clusters)".format( percentage, loop_counter)) if len(cluster_array) > 0: cls.objects.bulk_create(cluster_array) return else: raise ValueError( "precision must be in the [1, {}] interval".format( MAX_CLUSTER_PRECISION_SIZE)) else: operator_string = ' and operator {}'.format( operator) if operator else '' raise ValueError( "There are already clusters for precision {}{}".format( precision, operator_string))
def getGenusMetadata(self): """ Returns metadata for all genus of a specific family """ genera = self.QuerySet.filter(family_id=self.id).values('genus_id').annotate(points=Collect('geom'),ab=Count('genus_id'),name=Min('genus')) return genera
def getFamiliesMetadata(self): """ Returns metadata for all families of a specific class """ families = self.QuerySet.filter(order_id=self.id).values('family_id').annotate(points=Collect('geom'),ab=Count('family_id'),name=Min('family')) return families
def getOrdersMetadata(self): """ Returns metadata for all classes of a specific order """ orders = self.QuerySet.filter(class_id=self.id).values('order_id').annotate(points=Collect('geom'),ab=Count('order_id'),name=Min('_order')) return orders
def getClassesMetadata(self): """ Returns metadata for all classes of a specific order """ classes = self.QuerySet.filter(phylum_id=self.id).values('class_id').annotate(points=Collect('geom'),ab=Count('class_id'),name=Min('_class')) return classes
def handle(self, **options): # from http://colorbrewer2.org/index.php?type=diverging&scheme=Spectral&n=11 party_to_colour = { 'Party of National Unity': '#9e0142', 'Kenya African National Union': '#d53e4f', 'Orange Democratic Movement': '#f46d43', 'Orange Democratic Movement Party Of Kenya': '#fdae61', 'NARC - Kenya': '#fee08b', 'Safina Party Of Kenya': '#ffffbf', 'National Rainbow Coalition': '#e6f598', 'Ford People': '#abdda4', 'Democratic Party': '#66c2a5', 'spare': '#3288bd', # not used 'other': '#5e4fa2', } rows = [] fieldnames = ['name', 'person', 'party', 'color', 'location'] # get all the constituencies constituencies = models.Place.objects.all().filter(kind__slug='constituency'); for con in constituencies: row = { 'name': con.name, 'person': '', 'party': '', 'location': '', 'color': '', } # get the person and party data pos = con.current_politician_position() if pos: if pos.person: person = pos.person row['person'] = person.name; parties = [] for party in person.parties(): parties.append( party.name) row['party'] = ', '.join(parties) row['color'] = party_to_colour.get( row['party'] ) or party_to_colour['other']; # get the kml positions area = con.mapit_area if area: all_areas = area.polygons.all() if len(all_areas) > 1: all_areas = all_areas.aggregate(Collect('polygon'))['polygon__collect'] elif len(all_areas) == 1: all_areas = all_areas[0].polygon # else: # return output_json({ 'error': 'No polygons found' }, code=404) # Note - the following commented out as it causes issues for # some constiuncies - see # https://github.com/mysociety/pombola/issues/443 for details. # Not using this only results in the CS produced bein much larger # (7MB as opposed to 1MB) but for us this is not really an issue. # apply a simplify_tolerance to make CSV smaller # all_areas = all_areas.simplify(0.001) row['location'] = all_areas.kml rows.append(row) csv_output = StringIO.StringIO() writer = csv.DictWriter( csv_output, fieldnames ) fieldname_dict = {} for key in fieldnames: fieldname_dict[key] = key writer.writerow( fieldname_dict ) for data in rows: writer.writerow( data ) print csv_output.getvalue()
def handle_label(self, directory_name, **options): if not os.path.isdir(directory_name): raise Exception("'%s' is not a directory" % (directory_name,)) os.chdir(directory_name) skip_up_to = None # skip_up_to = 'relation-80370' skipping = bool(skip_up_to) osm_elements_seen_in_new_data = set([]) with open("/home/mark/difference-results.csv", 'w') as fp: csv_writer = csv.writer(fp) csv_writer.writerow(["ElementType", "ElementID", "ExistedPreviously", "PreviousEmpty", "PreviousArea", "NewEmpty", "NewArea", "SymmetricDifferenceArea", "GEOSEquals", "GEOSEqualsExact"]) for admin_directory in sorted(x for x in os.listdir('.') if os.path.isdir(x)): if not re.search('^[A-Z0-9]{3}$', admin_directory): print("Skipping a directory that doesn't look like a MapIt type:", admin_directory) if not os.path.exists(admin_directory): continue files = sorted(os.listdir(admin_directory)) for i, e in enumerate(files): if skipping: if skip_up_to in e: skipping = False else: continue if not e.endswith('.kml'): continue m = re.search(r'^(way|relation)-(\d+)-', e) if not m: raise Exception("Couldn't extract OSM element type and ID from: " + e) osm_type, osm_id = m.groups() osm_elements_seen_in_new_data.add((osm_type, osm_id)) kml_filename = os.path.join(admin_directory, e) # Need to parse the KML manually to get the ExtendedData kml_data = KML() print("parsing", kml_filename) xml.sax.parse(kml_filename, kml_data) useful_names = [n for n in kml_data.data.keys() if not n.startswith('Boundaries for')] if len(useful_names) == 0: raise Exception("No useful names found in KML data") elif len(useful_names) > 1: raise Exception("Multiple useful names found in KML data") name = useful_names[0] print(" ", smart_str(name)) if osm_type == 'relation': code_type_osm = CodeType.objects.get(code='osm_rel') elif osm_type == 'way': code_type_osm = CodeType.objects.get(code='osm_way') else: raise Exception("Unknown OSM element type: " + osm_type) ds = DataSource(kml_filename) if len(ds) != 1: raise Exception("We only expect one layer in a DataSource") layer = ds[0] if len(layer) != 1: raise Exception("We only expect one feature in each layer") feat = layer[0] osm_codes = list(Code.objects.filter(type=code_type_osm, code=osm_id)) osm_codes.sort(key=lambda e: e.area.generation_high.created) new_area = None new_empty = None previous_area = None previous_empty = None symmetric_difference_area = None g = feat.geom.transform(4326, clone=True) for polygon in g: if polygon.point_count < 4: new_empty = True if not new_empty: new_geos_geometry = g.geos.simplify(tolerance=0) new_area = new_geos_geometry.area new_empty = new_geos_geometry.empty geos_equals = None geos_equals_exact = None most_recent_osm_code = None if osm_codes: most_recent_osm_code = osm_codes[-1] previous_geos_geometry = most_recent_osm_code.area.polygons.aggregate(Collect('polygon')) previous_geos_geometry = previous_geos_geometry['polygon__collect'] previous_empty = previous_geos_geometry is None if not previous_empty: previous_geos_geometry = previous_geos_geometry.simplify(tolerance=0) previous_area = previous_geos_geometry.area if not new_empty: symmetric_difference_area = previous_geos_geometry.sym_difference( new_geos_geometry).area geos_equals = previous_geos_geometry.equals(new_geos_geometry) geos_equals_exact = previous_geos_geometry.equals_exact(new_geos_geometry) csv_writer.writerow([osm_type, osm_id, bool(osm_codes), # ExistedPreviously empty_if_none(previous_empty), empty_if_none(previous_area), empty_if_none(new_empty), empty_if_none(new_area), empty_if_none(symmetric_difference_area), empty_if_none(geos_equals), empty_if_none(geos_equals_exact)])
def get_context_data(self, **kwargs): context = super(UserPlaceListView, self).get_context_data(**kwargs) context['center'] = self.get_queryset().aggregate( position_center=Collect('position'))['position_center'].centroid return context
def getPhylaMetadata(self): """ Returns metadata for all phyla of a specific kingdom """ phyla = self.QuerySet.filter(kingdom_id=self.id).values('phylum_id').annotate(points=Collect('geom'),ab=Count('phylum_id'),name=Min('phylum')) return phyla
def handle_label(self, directory_name, **options): current_generation = Generation.objects.current() if not os.path.isdir(directory_name): raise Exception("'%s' is not a directory" % (directory_name,)) os.chdir(directory_name) mapit_type_glob = smart_text("[A-Z0-9][A-Z0-9][A-Z0-9]") if not glob(mapit_type_glob): raise Exception( "'%s' did not contain any directories that look like MapIt types (e.g. O11, OWA, etc.)" % ( directory_name,)) def verbose(s): if int(options['verbosity']) > 1: print(smart_str(s)) verbose("Loading any admin boundaries from " + directory_name) for type_directory in sorted(glob(mapit_type_glob)): verbose("Loading type " + type_directory) if not os.path.exists(type_directory): verbose("Skipping the non-existent " + type_directory) continue verbose("Loading all KML in " + type_directory) files = sorted(os.listdir(type_directory)) total_files = len(files) for i, e in enumerate(files): progress = "[%d%% complete] " % ((i * 100) / total_files,) if not e.endswith('.kml'): verbose("Ignoring non-KML file: " + e) continue m = re.search(r'^(way|relation)-(\d+)-', e) if not m: raise Exception("Couldn't extract OSM element type and ID from: " + e) osm_type, osm_id = m.groups() kml_filename = os.path.join(type_directory, e) verbose(progress + "Loading " + os.path.realpath(kml_filename)) if osm_type == 'relation': code_type_osm = CodeType.objects.get(code='osm_rel') elif osm_type == 'way': code_type_osm = CodeType.objects.get(code='osm_way') else: raise Exception("Unknown OSM element type: " + osm_type) ds = DataSource(kml_filename) layer = ds[0] if len(layer) != 1: raise Exception("We only expect one feature in each layer") feat = layer[1] g = feat.geom.transform(4326, clone=True) if g.geom_count == 0: verbose(' Ignoring that file - it contained no polygons') continue polygons_too_small = 0 for polygon in g: if polygon.num_points < 4: polygons_too_small += 1 if polygons_too_small: message = "%d out of %d polygon(s) were too small" % (polygons_too_small, g.geom_count) verbose(' Skipping, since ' + message) continue g_geos = g.geos if not g_geos.valid: verbose(" Invalid KML:" + kml_filename) fixed_multipolygon = fix_invalid_geos_multipolygon(g_geos) if len(fixed_multipolygon) == 0: verbose(" Invalid polygons couldn't be fixed") continue g = fixed_multipolygon.ogr osm_code = Code.objects.get( type=code_type_osm, code=osm_id, area__generation_high__lte=current_generation, area__generation_high__gte=current_generation) m = osm_code.area previous_geos_geometry = m.polygons.aggregate(Collect('polygon'))['polygon__collect'] previous_geos_geometry = shapely.wkb.loads(str(previous_geos_geometry.simplify(tolerance=0).ewkb)) new_geos_geometry = shapely.wkb.loads(str(g.geos.simplify(tolerance=0).ewkb)) if previous_geos_geometry.almost_equals(new_geos_geometry, decimal=7): verbose(' Boundary unchanged') else: verbose(' In the current generation, the boundary was different') poly = [g] if options['commit']: save_polygons({'dummy': (m, poly)})