def handle(self, **options): shape_file = options['shape'][0] sensor = options['sensor'][0] country = options['country'][0] column = options['column'][0] country_object = Country.objects.get(name=country) with fiona.open(shape_file) as source: print(source.crs) for feat in source: s1 = shape(feat['geometry']) if source.crs['init'] != 'epsg:4326': project = partial( pyproj.transform, pyproj.Proj(source.crs), pyproj.Proj(init='EPSG:4326')) s2 = transform(project, s1) else: s2 = s1 geom = GEOSGeometry(s2.wkt) if country_object.the_geom.intersects(geom): name = '%03d%03d' % (feat['properties']['PATH'], feat['properties']['ROW']) if name: try: o = Footprint(the_geom = geom, sensor=sensor, name=name) o.save() except IntegrityError: logger.error('Name %s already exists.' % name)
def parse_args(self, args): #program_name = os.path.basename(sys.argv[0]) parser = argparse.ArgumentParser() # description='', usage = '' #parser.add_argument("-w", "--worker", default=None, help="worker (i/n)") parser.add_argument("--name", type=str, default=None, help="base name for output") parser.add_argument("--center", type=str, default=None, help="center of target area") parser.add_argument("--area", type=str, default=None, help="target area polygon GeoJSON") #parser.add_argument("--radius", type=float, default=None, help="radius of target area") #parser.add_argument("--area", type=str, help="GeoJSON polygon of target area") #parser.add_argument("--tile", type=float, help="tile size in meters (0 for entire area)") args = parser.parse_args(args) center = args.center.split(",") self.center = (float(center[0]), float(center[1])) self.name = args.name self.area = shape(json.loads(args.area))
def calculate_neighbors(state_abbr, year): with open(f'{state_abbr.upper()}{sep}2017{sep}{state_abbr}_zoom7_0.geojson' ) as f: precincts = json.load(f) pshapes = {} for pfeature in precincts['features']: pshapes[pfeature['properties']['id']] = geo.shape(pfeature['geometry']) all_neighbors = {} # create a dictionary of precinct id keys and lists of ids of neighbors # will be saved in a separate file not with GeoJSON since this data is # not stored in the master DB it will have to be transmitted separately # so keeping it separate will be helpful for (outer_id, shape) in pshapes.items(): print(f"Finding neighbors for {outer_id}") for (inner_id, candidate) in pshapes.items(): # note that touches alone is not enough try: if (shape.touches(candidate) or shape.intersects(candidate) or shape.overlaps(candidate)): if outer_id in all_neighbors: all_neighbors[outer_id].append(inner_id) else: all_neighbors[outer_id] = [inner_id] except shapely.errors.TopologicalError as e: print(f'Error in finding neighbors for {outer_id}.') print(e) with open( f'{state_abbr.upper()}{sep}{year}{sep}{state_abbr}_p_neighbors.scsv', 'w') as results_file: for (pid, neighbor_list) in all_neighbors.items(): results_file.write(f'{pid}|{",".join(neighbor_list)}\n')
def calculate_border_precincts(state_abbr, year): with open(f'{state_abbr.upper()}{sep}2017{sep}{state_abbr}_zoom7_0.geojson' ) as f: precincts = json.load(f) pshapes = {} for pfeature in precincts['features']: try: precinct = Precinct( pfeature['properties']['id'], pfeature['properties']['containing_district_id'], geo.shape(pfeature['geometry'])) pshapes[pfeature['properties']['id']] = precinct except KeyError as e: id = pfeature['properties'][ 'id'] if 'properties' in pfeature and 'id' in pfeature[ 'properties'] else '?' print(f"Error on precinct: {id} -> {e}") all_neighbors = {} # dictionary relating district ids to lists of border precinct ids border_precincts = set(map(lambda p: p.district_id, pshapes.values())) border_precincts = {k: [] for k in border_precincts} # all neighbors is a dictionary that maps precinct objects to lists # of objects that are their neighbors for (outer_id, precinct) in pshapes.items(): print(f"Finding neighbors for {outer_id}") for (inner_id, candidate_precinct) in pshapes.items(): try: # note that touches alone is not enough if (precinct.shape.touches(candidate_precinct.shape) or precinct.shape.intersects(candidate_precinct.shape) or precinct.shape.overlaps(candidate_precinct.shape)): if precinct in all_neighbors: all_neighbors[precinct].append(candidate_precinct) else: all_neighbors[precinct] = [candidate_precinct] except shapely.errors.TopologicalError as e: print(f'Error in finding neighbors for {outer_id}.') print(e) # run through all the neighbors of each precinct and if any of them # are in a district that is not the same as the precinct we call that # a border precinct for (precinct, neighbors) in all_neighbors.items(): if any(i.district_id != precinct.district_id for i in neighbors): # if this is true then this precinct `precinct` is a border # precinct within its district border_precincts[precinct.district_id].append(precinct) with open( f'{state_abbr.upper()}{sep}{year}{sep}{state_abbr}_border_precincts.scsv', 'w') as f: for (did, precincts) in border_precincts.items(): f.write( f'{did}|{",".join(list(map(lambda p: p.id, precincts)))}\n')
def fromfile(cls, data, file_path): kwargs = super().fromfile(data, file_path) if 'geometry' not in data: raise ValueError('missing geometry.') try: kwargs['geometry'] = shape(data['geometry']) except: raise ValueError(_('Invalid GeoJSON.')) return kwargs
def storePolygons(jsonfile): with open(jsonfile) as data_file: js = json.load(data_file) list_of_polygons = {} for feature in js['features']: if feature['geometry']: polygon_id = feature['properties']['SA2_MAIN11'] list_of_polygons[polygon_id] = shape(feature['geometry']) return list_of_polygons
def spans_feature(self, rect, ref_shape): min_x, min_y, max_x, max_y = rect tile_ulp = (min_x, max_y) tile_dlp = (min_x, min_y) tile_drp = (max_x, min_y) tile_urp = (max_x, max_y) tile = Polygon([tile_ulp, tile_dlp, tile_drp, tile_urp]) #return not tile.intersection(geom).is_empty for feature in ref_shape: shp_geom = shape(feature['geometry']) poly_int =tile.intersection(shp_geom) if not poly_int.is_empty: return True return False
def count_polygons(shp): count=0 for feature in shp: geom=shape(feature['geometry']) print( feature['properties']) if geom.type == 'Polygon': count+=1 elif geom.type == 'MultiPolygon': for part in geom: count+=1 else: raise ValueError('Unhandled geometry type: ' + repr(geom.type)) print( count) return count
def _iter_source_(self): with fiona.open(self.path, 'r') as source: for feature in source: # ensure the feature is valid # https://github.com/Toblerity/Fiona/blob/master/examples/with-shapely.py geom = shape(feature['geometry']) try: if not geom.is_valid: clean = geom.buffer(0.0) geom = clean feature['geometry'] = mapping(geom) assert clean.is_valid assert (clean.geom_type == 'Polygon') except (AssertionError, AttributeError) as e: warn('{2}. Invalid geometry found with id={0} and properties: {1}'.format(feature['id'], feature['properties'], e)) feature['shapely'] = geom yield feature
def _iter_source_(self): with fiona.open(self.path, 'r') as source: for feature in source: # ensure the feature is valid # https://github.com/Toblerity/Fiona/blob/master/examples/with-shapely.py geom = shape(feature['geometry']) try: if not geom.is_valid: clean = geom.buffer(0.0) geom = clean feature['geometry'] = mapping(geom) assert clean.is_valid assert (clean.geom_type == 'Polygon') except (AssertionError, AttributeError) as e: warn('{2}. Invalid geometry found with id={0} and properties: {1}'.format(feature['id'], feature['properties'], e)) feature['shapely'] = geom yield feature
def handle(self, **options): path = options['path'][0] column = options['column'][0] model = options['model'][0] with fiona.open(path) as src: print json.dumps(src.schema, indent=4) print src.crs for feat in src: #print feat['geometry']['type'] s = shape(feat['geometry']) if feat['geometry']['type'] == 'Polygon': s = MultiPolygon([s]) print json.dumps(feat['geometry']) klass = locate('madmex.models.%s' % model) f = klass(name=feat['properties'][column], the_geom=GEOSGeometry(s.wkt)) f.save()
def get_2d_state_boundaries(self): geoms = [] build = True with fiona.open('/home/local/WX/ben.koziol/Dropbox/nesii/project/ocg/bin/shp/state_boundaries/state_boundaries.shp','r') as source: for ii,row in enumerate(source): if build: nrows = len(source) dtype = [] for k,v in source.schema['properties'].iteritems(): if v.startswith('str'): v = str('|S{0}'.format(v.split(':')[1])) else: v = getattr(np,v.split(':')[0]) dtype.append((str(k),v)) fill = np.empty(nrows,dtype=dtype) ref_names = fill.dtype.names build = False fill[ii] = tuple([row['properties'][n] for n in ref_names]) geoms.append(shape(row['geometry'])) geoms = np.atleast_2d(geoms) return(geoms,fill)
def _iter_source_(self): with fiona.open(self.path, "r") as source: for feature in source: ## ensure the feature is valid ## https://github.com/Toblerity/Fiona/blob/master/examples/with-shapely.py try: geom = shape(feature["geometry"]) if not geom.is_valid: clean = geom.buffer(0.0) geom = clean feature["geometry"] = mapping(geom) assert clean.is_valid assert clean.geom_type == "Polygon" except (AssertionError, AttributeError) as e: warn( "{2}. Invalid geometry found with id={0} and properties: {1}".format( feature["id"], feature["properties"], e ) ) feature["shapely"] = geom yield (feature)
def sensors(db, provider, request): import json import os.path from shapely.geometry.geo import shape from app.models import Collection, Item collection = Collection.create(name="sensors", is_public=True, provider_uuid=provider["uuid"]) my_path = os.path.abspath(os.path.dirname(__file__)) path = os.path.join(my_path, "../../data/sensors.json") with open(path) as f: data = json.load(f) for feature in data["features"]: Item.create( collection_uuid=collection.uuid, properties=feature["properties"], geometry=shape(feature["geometry"]).to_wkt(), ) Collection.session.commit() return collection.to_dict()
def get_2d_state_boundaries(self): geoms = [] build = True sc = GeomCabinet() path = sc.get_shp_path('state_boundaries') with fiona.open(path, 'r') as source: for ii, row in enumerate(source): if build: nrows = len(source) dtype = [] for k, v in source.schema['properties'].iteritems(): if v.startswith('str'): v = str('|S{0}'.format(v.split(':')[1])) else: v = getattr(np, v.split(':')[0]) dtype.append((str(k), v)) fill = np.empty(nrows, dtype=dtype) ref_names = fill.dtype.names build = False fill[ii] = tuple([row['properties'][n] for n in ref_names]) geoms.append(shape(row['geometry'])) geoms = np.atleast_2d(geoms) return geoms, fill
def get_2d_state_boundaries(self): geoms = [] build = True sc = GeomCabinet() path = sc.get_shp_path('state_boundaries') with fiona.open(path, 'r') as source: for ii, row in enumerate(source): if build: nrows = len(source) dtype = [] for k, v in source.schema['properties'].items(): if v.startswith('str'): v = str('|S{0}'.format(v.split(':')[1])) else: v = getattr(np, v.split(':')[0]) dtype.append((str(k), v)) fill = np.empty(nrows, dtype=dtype) ref_names = fill.dtype.names build = False fill[ii] = tuple([row['properties'][n] for n in ref_names]) geoms.append(shape(row['geometry'])) geoms = np.atleast_2d(geoms) return geoms, fill
def osm_init_params(root, pipeline, logger): """ Process OSM build parameters. """ if pipeline.data.get('ddd:osm:area', None): # Use provided shape area_shape = shape(json.loads(pipeline.data.get('ddd:osm:area'))) pipeline.data['ddd:osm:area:shape'] = area_shape pipeline.data['ddd:osm:area:radius'] = None if pipeline.data.get('ddd:osm:area:radius', None): # Convert radius to meters pipeline.data['ddd:osm:area:radius'] = parse_meters( pipeline.data['ddd:osm:area:radius']) pipeline.data['ddd:osm:area:shape'] = None if pipeline.data.get('ddd:osm:area:chunk_size', None): pipeline.data['ddd:osm:area:chunk_size'] = parse_meters( pipeline.data['ddd:osm:area:chunk_size']) xyztile = pipeline.data.get('ddd:osm:area:xyztile', None) if xyztile: if ('ddd:osm:area:radius' in pipeline.data or 'ddd:osm:area:center' in pipeline.data or 'ddd:osm:area:size' in pipeline.data): logger.error( "Option --xyztile cannot be used with --radius, --center or --size ." ) sys.exit(2) pipeline.data['ddd:osm:area:xyztile'] = parse_tile(xyztile) if pipeline.data.get('ddd:osm:area:center', None): center = pipeline.data['ddd:osm:area:center'].split(",") center = (float(center[0]), float(center[1])) pipeline.data['ddd:osm:area:center'] = center
def countries_intersecting(boundary: Geometry): bounds = shape(boundary) subset = gdf.loc[gdf.intersects(bounds)] return rows_to_countries(subset)
async def process_source(filename: str, session: ClientSession): try: async with aiofiles.open(filename, mode="r", encoding="utf-8") as f: contents = await f.read() source = json.loads(contents) # Exclude sources # Skip non wms layers if not source["properties"]["type"] == "wms": return # check if it is esri rest and not wms if "bboxSR" in source["properties"]["url"]: return if "available_projections" not in source["properties"]: return if "header" in source["properties"]["url"]: return if "geometry" not in source: return processed_sources.add(filename) category = source["properties"].get("category", None) if source["geometry"] is None: geom: MultiPolygon | Polygon = box(-180, -90, 180, 90) pt: Point = Point(7.44, 46.56) else: geom = shape(source["geometry"]) pt: Point = geom.representative_point() # type: ignore test_zoom_level = ZOOM_LEVEL if "min_zoom" in source["properties"]: test_zoom_level = max(source["properties"]["min_zoom"], test_zoom_level) if "max_zoom" in source["properties"]: test_zoom_level = min(source["properties"]["max_zoom"], test_zoom_level) old_url = source["properties"]["url"] old_projections = source["properties"]["available_projections"] # Get existing image hash original_img_messages: List[str] = [] original_image_result = await get_image( url=old_url, available_projections=old_projections, lon=pt.x, # type: ignore lat=pt.y, # type: ignore zoom=test_zoom_level, session=session, messages=original_img_messages, ) if not original_image_result.status == ImageHashStatus.SUCCESS or original_image_result.image_hash is None: # type: ignore ignored_sources[ filename] = "Not possible to download reference image" # We are finished if it was not possible to get the image return if max_count(str( original_image_result.image_hash)) == 16: # type: ignore if "category" in source["properties"] and "photo" in source[ "properties"]["category"]: msgs = "\n\t".join(original_img_messages) logging.warning( f"{filename}: has category {category} but image hash is {original_image_result.image_hash}:\n\t{msgs}" ) # type: ignore # These image hashes indicate that the downloaded image is not useful to determine # if the updated query returns the same image error_msgs = "\n\t".join(original_img_messages) logging.warning( f"{filename}: Image hash {original_image_result.image_hash} not useful ({category}): \n\t{error_msgs}" ) # type: ignore ignored_sources[ filename] = f"Image hash {original_image_result.image_hash} not useful ({category})" # type: ignore return # Update wms wms_messages: List[str] = [] result = await update_wms(old_url, session, wms_messages) if result is None: error_msgs = "\n\t".join(wms_messages) logging.info( f"{filename}: Not possible to update wms url:\n\t{error_msgs}") ignored_sources[filename] = "Not possible to update wms url" return new_url, new_projections = result del result # Servers that report a lot of projection may be configured wrongly # Check for CRS:84, EPSG:3857, EPSG:4326 and keep existing projections if still advertised if len(new_projections) > 15: filtered_projs: Set[str] = set() for proj in ["CRS:84", "EPSG:3857", "EPSG:4326"]: if proj in new_projections: filtered_projs.add(proj) for proj in old_projections: if proj in new_projections: filtered_projs.add(proj) new_projections = filtered_projs # Download image for updated url new_img_messages: List[str] = [] updated_image_result = await get_image( url=new_url, available_projections=new_projections, lon=pt.x, # type: ignore lat=pt.y, # type: ignore zoom=test_zoom_level, session=session, messages=new_img_messages, ) if not updated_image_result.status == ImageHashStatus.SUCCESS or updated_image_result.image_hash is None: # type: ignore error_msgs = "\n\t".join(new_img_messages) logging.warning( f"{filename}: Could not download image with updated url: {updated_image_result.status}\n\t{error_msgs}" ) ignored_sources[ filename] = "Could not download image with updated url" return # Only sources are updated where the new query returns the same image if not image_similar(original_image_result.image_hash, updated_image_result.image_hash, test_zoom_level): # type: ignore original_hash = original_image_result.image_hash # type: ignore new_hash = updated_image_result.image_hash # type: ignore hash_diff = original_hash - new_hash # type: ignore error_original_img_messages = "\n\t".join(original_img_messages) error_new_img_messages = "\n\t".join(new_img_messages) logging.info( f"{filename}: ImageHash not the same for: {filename}: {original_hash} - {new_hash}: {hash_diff}\n\t{error_original_img_messages} \n\t{error_new_img_messages}" ) ignored_sources[ filename] = f"ImageHash for reference image and image with updated url differs: {original_hash} - {new_hash}: {new_hash}" return # Test if selected projections work despite not being advertised for EPSG in {"EPSG:3857", "EPSG:4326"}: if EPSG not in new_projections: epsg_check_messages: List[str] = [] epsg_image_result = await get_image( url=new_url, available_projections=[EPSG], lon=pt.x, # type: ignore lat=pt.y, # type: ignore zoom=test_zoom_level, session=session, messages=epsg_check_messages, ) epsg_check_messages_str = "\n\t".join(epsg_check_messages) logging.info( f"{filename}: Test if projection {EPSG} works despite not advertised:\n\t{epsg_check_messages_str}" ) if epsg_image_result.status == ImageHashStatus.NETWORK_ERROR: if EPSG in old_projections and EPSG not in new_projections: new_projections.add(EPSG) added_projections[filename][ "Network error, but projection was previously included."].append( EPSG) elif epsg_image_result.status == ImageHashStatus.SUCCESS: epsg_image_hash = epsg_image_result.image_hash # type: ignore original_image_hash = original_image_result.image_hash # type: ignore # Relax similarity constraint to account for differences due to loss of quality due to re-projection hash_diff = original_image_result.image_hash - epsg_image_result.image_hash # type: ignore if image_similar(original_image_result.image_hash, epsg_image_result.image_hash, test_zoom_level): # type: ignore new_projections.add(EPSG) added_projections[filename][ "Projection returns similar image despite not advertised."].append( EPSG) logging.info( f"{filename}: Add {EPSG} despite not being advertised: {epsg_image_hash} - {original_image_hash}: {hash_diff}" ) elif epsg_image_hash is not None: logging.info( f"{filename}: Do not add {EPSG} Difference: {epsg_image_hash} - {original_image_hash}: {hash_diff}" ) else: logging.info( f"{filename}: Do not add {EPSG} No image returned." ) # Check if projections are supported by server not_supported_projections: Set[str] = set() image_hashes: Dict[str, Tuple[ImageResult, List[str]]] = {} for proj in new_projections: proj_messages: List[str] = [] epsg_image_result = await get_image( url=new_url, available_projections=[proj], lon=pt.x, # type: ignore lat=pt.y, # type: ignore zoom=test_zoom_level, session=session, messages=proj_messages, ) image_hashes[proj] = (epsg_image_result, proj_messages) msgs = "\n\t".join(proj_messages) logging.info( f"{filename} Projection check: {proj}: {epsg_image_result.status}:\n\t{msgs}" ) if epsg_image_result.status == ImageHashStatus.IMAGE_ERROR: not_supported_projections.add(proj) removed_projections[filename][ "Projection check: does not return an image"].append(proj) elif epsg_image_result.status == ImageHashStatus.NETWORK_ERROR: # If not successfully status do not add if not previously added if proj not in old_projections: removed_projections[filename][ "Projection check: network error and previously not included"].append( proj) not_supported_projections.add(proj) if len(not_supported_projections) > 0: removed = ",".join(not_supported_projections) logging.info( f"{filename}: remove projections that are advertised but do not return an image: {removed}" ) new_projections -= not_supported_projections # Check if EPSG:3857 and EPSG:4326 are similar if ("EPSG:3857" in image_hashes and "EPSG:4326" in image_hashes and image_hashes["EPSG:3857"][0].status == ImageHashStatus.SUCCESS and image_hashes["EPSG:4326"][0].status == ImageHashStatus.SUCCESS): img_hash_3857 = image_hashes["EPSG:3857"][ 0].image_hash # type: ignore img_hash_4326 = image_hashes["EPSG:4326"][ 0].image_hash # type: ignore diff_hash = img_hash_3857 - img_hash_4326 # type: ignore if not image_similar(img_hash_3857, img_hash_4326, test_zoom_level): msgs = "\n\t".join(image_hashes["EPSG:3857"][1] + image_hashes["EPSG:4326"][1]) logging.warning( f"{filename}: ({category}) ImageHash for EPSG:3857 and EPSG:4326 not similar: {img_hash_3857} - {img_hash_4326}: {diff_hash}:\n\t{msgs}" ) # Check projections again to filter out EPSG:3857 alias new_projections = eliutils.clean_projections(new_projections) # Check if only formatting has changed url_has_changed = not compare_urls(source["properties"]["url"], new_url) projections_have_changed = not compare_projs( source["properties"]["available_projections"], new_projections, ) if url_has_changed: source["properties"]["url"] = new_url if projections_have_changed: source["properties"]["available_projections"] = list( sorted( new_projections, key=lambda x: (x.split(":")[0], int(x.split(":")[1])), )) if url_has_changed or projections_have_changed: with open(filename, "w", encoding="utf-8") as out: json.dump(source, out, indent=4, sort_keys=False, ensure_ascii=False) out.write("\n") except Exception as e: logging.exception( f"{filename}: Error occurred while processing source: {e}")
_x, _y = pyproj.transform(proj_lv03, proj_lv95, x, y) bbox = (_x - D, _y - D, _x + D, _y + D) crs = CRS.from_epsg(2056) else: src = fiona.open('/home/rbuffat/buildings_av_osm_tlm.shp') bbox = (x - D, y - D, x + D, y + D) crs = CRS.from_epsg(21781) proj = pyproj.Proj("+init=EPSG:21781") geom = None for ss in src.filter(bbox=bbox): if ss['properties']['bid'] == int( row['bid'] ) and ss['properties']['btype'] == row['btype']: geom = shape(ss['geometry']) # get building area barea = geom.area hor_sis = [] # get hor solar irad lon, lat = pyproj.transform(proj_lv03, pyproj.Proj("+init=EPSG:4326"), x, y) d = datetime.date(2017, 1, 1) while d.year < 2018: # logging.info("{}_{}: {}".format(btype, bid, str(d))) year = d.year month = d.month day = d.day
def check_tms(source: Dict[str, Any], messages: List[Message]) -> None: """Check TMS source Parameters ---------- source : Dict[str, Any] The source messages : List[Message] The list to add messages to """ try: url = source["properties"]["url"] source_headers = get_http_headers(source) if source["geometry"] is None: geom = None else: geom = shape(source["geometry"]) # Validate URL try: _url = re.sub(r"switch:?([^}]*)", "switch", url).replace("{", "").replace("}", "") validators.url(_url) # type: ignore except validators.utils.ValidationFailure as e: messages.append( Message(level=MessageLevel.ERROR, message=f"URL validation error {e} / {url}")) # Check URL parameter parameters = {} # {z} instead of {zoom} if "{z}" in source["properties"]["url"]: messages.append( Message( level=MessageLevel.ERROR, message= f"Parameter {{z}} is used instead of {{zoom}} in tile url: {url}" )) return # We can't test sources that have an apikey, that is unknown to ELI if "{apikey}" in url: messages.append( Message( level=MessageLevel.WARNING, message= f"Not possible to check URL, apikey is required: {url}")) return # If URL contains a {switch:a,b,c} parameters, use the first for tests match = re.search(r"switch:?([^}]*)", url) if match is not None: switches = match.group(1).split(",") url = url.replace(match.group(0), "switch") parameters["switch"] = switches[0] # Check zoom levels min_zoom = 0 max_zoom = 22 if "min_zoom" in source["properties"]: min_zoom = int(source["properties"]["min_zoom"]) if "max_zoom" in source["properties"]: max_zoom = int(source["properties"]["max_zoom"]) # Check if we find a TileMap Resource to check for zoom levels # While there is a typical location for metadata, there is no requirement # that the metadata need to be located there. tms_url = tmshelper.TMSURL(url=url) tilemap_resource_url = tms_url.get_tilemap_resource_url() if tilemap_resource_url is not None: for tilemap_url in [ tilemap_resource_url, tilemap_resource_url + "/tilemapresource.xml", ]: try: r, xml = get_text_encoded(tilemap_url.format(**parameters), headers=headers) if r.status_code == 200 and xml is not None: try: tilemap_resource = tmshelper.TileMapResource(xml) except Exception: # Not all TMS server provide TileMap resources. continue if tilemap_resource.tile_map is None: continue # Check zoom levels against TileMapResource tilemap_minzoom, tilemap_maxzoom = tilemap_resource.get_min_max_zoom_level( ) if min_zoom == tilemap_minzoom: messages.append( Message( level=MessageLevel.WARNING, message= f"min_zoom level '{min_zoom}' not the same as specified in TileMap: '{tilemap_minzoom}': {tilemap_url}. " "Caution: this might be intentional as some server timeout for low zoom levels.", )) if not max_zoom == tilemap_maxzoom: messages.append( Message( level=MessageLevel.WARNING, message= f"max_zoom level '{max_zoom}' not the same as specified in TileMap: '{tilemap_maxzoom}': {tilemap_url}", )) # Check geometry within bbox if geom is not None and tilemap_resource.tile_map.bbox84 is not None: max_area_outside = max_area_outside_bbox( geom, tilemap_resource.tile_map.bbox84) # 5% is an arbitrary chosen value and should be adapted as needed if max_area_outside > 5.0: messages.append( Message( level=MessageLevel.ERROR, message= f"{round(max_area_outside, 2)}% of geometry is outside of the layers bounding box. Geometry should be checked", )) break except Exception as e: print(f"Error fetching TMS: {e}: {url}") pass # Test zoom levels by accessing tiles for a point within the geometry if geom is not None: centroid: Point = geom.representative_point() # type: ignore else: centroid = Point(6.1, 49.6) centroid_x: float = centroid.x # type: ignore centroid_y: float = centroid.y # type: ignore zoom_failures: List[Tuple[int, str, int, Optional[str]]] = [] zoom_success: List[int] = [] tested_zooms: Set[int] = set() def test_zoom(zoom: int) -> None: tested_zooms.add(zoom) tile: mercantile.Tile = mercantile.tile(centroid_x, centroid_y, zoom) # type: ignore tile_x: int = tile.x # type: ignore tile_y: int = tile.y # type: ignore query_url = url if "{-y}" in url: y = 2**zoom - 1 - tile_y query_url = query_url.replace("{-y}", str(y)) elif "{!y}" in url: y = 2**(zoom - 1) - 1 - tile_y query_url = query_url.replace("{!y}", str(y)) else: query_url = query_url.replace("{y}", str(tile_y)) parameters["x"] = tile_x parameters["zoom"] = zoom query_url = query_url.format(**parameters) url_is_good, http_code, mime = test_image(query_url, source_headers) if url_is_good: zoom_success.append(zoom) else: zoom_failures.append((zoom, query_url, http_code, mime)) # Test zoom levels for zoom in range(min_zoom, max_zoom + 1): test_zoom(zoom) tested_str = ",".join(list(map(str, sorted(tested_zooms)))) sorted_failures = sorted(zoom_failures, key=lambda x: x[0]) if len(zoom_failures) == 0 and len(zoom_success) > 0: messages.append( Message( level=MessageLevel.INFO, message=f"Zoom levels reachable. (Tested: {tested_str})")) elif len(zoom_failures) > 0 and len(zoom_success) > 0: not_found_str = ",".join( list(map(str, [level for level, _, _, _ in sorted_failures]))) messages.append( Message( level=MessageLevel.WARNING, message= f"Zoom level {not_found_str} not reachable. (Tested: {tested_str}) Tiles might not be present at tested location: {centroid_x},{centroid_y}", )) for level, url, http_code, mime_type in sorted_failures: messages.append( Message( level=MessageLevel.WARNING, message= f"URL for zoom level {level} returned HTTP Code {http_code}: {url} MIME type: {mime_type}", )) else: messages.append( Message( level=MessageLevel.ERROR, message= f"No zoom level reachable. (Tested: {tested_str}) Tiles might not be present at tested location: {centroid_x},{centroid_y}", )) for level, url, http_code, mime_type in sorted_failures: messages.append( Message( level=MessageLevel.WARNING, message= f"URL for zoom level {level} returned HTTP Code {http_code}: {url} MIME type: {mime_type}", )) except Exception as e: messages.append( Message( level=MessageLevel.ERROR, message=f"Failed testing TMS source: Exception: {e}", ))
def check_wms(source: Dict[str, Any], messages: List[Message]) -> None: """Check WMS source Parameters ---------- source : Dict[str, Any] The source messages : List[Message] The list to add messages to """ url = source["properties"]["url"] wms_url = wmshelper.WMSURL(url) source_headers = get_http_headers(source) params = ["{proj}", "{bbox}", "{width}", "{height}"] missingparams = [p for p in params if p not in url] if len(missingparams) > 0: messages.append( Message( level=MessageLevel.ERROR, message= f"The following values are missing in the URL: {','.join(missingparams)}", )) try: wms_url.is_valid_getmap_url() except validators.utils.ValidationFailure as e: messages.append( Message(level=MessageLevel.ERROR, message=f"URL validation error {e} for {url}")) # Check mandatory WMS GetMap parameters (Table 8, Section 7.3.2, WMS 1.3.0 specification) # Normalize parameter names to lower case wms_args = {key.lower(): value for key, value in wms_url.get_parameters()} # Check if it is actually a ESRI Rest url and not a WMS url is_esri = "request" not in wms_args # Check if required parameters are missing missing_request_parameters: Set[str] = set() if is_esri: required_parameters = [ "f", "bbox", "size", "imageSR", "bboxSR", "format" ] else: required_parameters = [ "version", "request", "layers", "bbox", "width", "height", "format", ] for request_parameter in required_parameters: if request_parameter.lower() not in wms_args: missing_request_parameters.add(request_parameter) if not is_esri: if "version" in wms_args and wms_args["version"] == "1.3.0": if "crs" not in wms_args: missing_request_parameters.add("crs") if "srs" in wms_args: messages.append( Message( level=MessageLevel.ERROR, message= f"WMS {wms_args['version']} URLs should not contain SRS parameter: {url}", )) elif "version" in wms_args and not wms_args["version"] == "1.3.0": if "srs" not in wms_args: missing_request_parameters.add("srs") if "crs" in wms_args: messages.append( Message( level=MessageLevel.ERROR, message= f"WMS {wms_args['version']} URLs should not contain CRS parameter: {url}", )) if len(missing_request_parameters) > 0: missing_request_parameters_str = ",".join(missing_request_parameters) messages.append( Message( level=MessageLevel.ERROR, message= f"Parameter '{missing_request_parameters_str}' is missing in URL: {url}.", )) return # Nothing more to do for ESRI Rest API if is_esri: return # Styles is mandatory according to the WMS specification, but some WMS servers seems not to care if "styles" not in wms_args: messages.append( Message( level=MessageLevel.WARNING, message= f"Parameter 'styles' is missing in url. 'STYLES=' can be used to request default style.: {url}", )) # We first send a service=WMS&request=GetCapabilities request to server # According to the WMS Specification Section 6.2 Version numbering and negotiation, the server should return # the GetCapabilities XML with the highest version the server supports. # If this fails, it is tried to explicitly specify a WMS version exceptions: List[str] = [] wms = None for wms_version in [None, "1.3.0", "1.1.1", "1.1.0", "1.0.0"]: if wms_version is None: wms_version_str = "-" else: wms_version_str = wms_version wms_getcapabilities_url = None try: wms_getcapabilities_url = wms_url.get_capabilities_url( wms_version=wms_version) _, xml = get_text_encoded(wms_getcapabilities_url, headers=source_headers) if xml is not None: wms = wmshelper.WMSCapabilities(xml) break except Exception as e: exceptions.append( f"WMS {wms_version_str}: Error: {e} {wms_getcapabilities_url}") continue # Check if it was possible to parse the WMS GetCapability response # If not, there is nothing left to check if wms is None: for msg in exceptions: messages.append(Message( level=MessageLevel.ERROR, message=msg, )) return # Log access constraints and fees metadata for access_constraint in wms.access_constraints: messages.append( Message( level=MessageLevel.INFO, message=f"AccessConstraints: {access_constraint}", )) for fee in wms.fees: messages.append( Message( level=MessageLevel.INFO, message=f"Fee: {fee}", )) if source["geometry"] is None: geom = None else: geom = shape(source["geometry"]) # type: ignore # Check layers if "layers" in wms_args: layers = wms_args["layers"].split(",") # Check if layers in WMS GetMap URL are advertised by WMS server. not_found_layers = [ layer_name for layer_name in layers if layer_name not in wms.layers ] if len(not_found_layers) > 0: messages.append( Message( level=MessageLevel.ERROR, message= f"Layers '{','.join(not_found_layers)}' not advertised by WMS GetCapabilities request (Some server do not advertise layers, but they are very rare).: {url}", )) # Check source geometry against layer bounding box # Regardless of its projection, each layer should advertise an approximated bounding box in lon/lat. # See WMS 1.3.0 Specification Section 7.2.4.6.6 EX_GeographicBoundingBox if geom is not None and geom.is_valid: # type: ignore bboxs = [ wms.layers[layer_name].bbox for layer_name in layers if layer_name in wms.layers and wms.layers[layer_name].bbox ] bboxs = [bbox for bbox in bboxs if bbox is not None] max_area_outside = max_area_outside_bbox(geom, bboxs) # type: ignore # 5% is an arbitrary chosen value and should be adapted as needed if max_area_outside > 5.0: messages.append( Message( level=MessageLevel.ERROR, message= f"{round(max_area_outside, 2)}% of geometry is outside of the layers bounding box. Geometry should be checked", )) # Check styles if "styles" in wms_args: style_parameter = wms_args["styles"] # default style needs not to be advertised by the server if not (style_parameter == "default" or style_parameter == "" or style_parameter == "," * len(layers)): styles = style_parameter.split(",") if not len(styles) == len(layers): messages.append( Message( level=MessageLevel.ERROR, message= f"Not the same number of styles and layers. {len(styles)} vs {len(layers)}", )) else: for layer_name, style_name in zip(layers, styles): if (len(style_name) > 0 and not style_name == "default" and layer_name in wms.layers and style_name not in wms.layers[layer_name].styles): messages.append( Message( level=MessageLevel.ERROR, message= f"Layer '{layer_name}' does not support style '{style_name}'", )) # Check CRS if "available_projections" not in source["properties"]: messages.append( Message( level=MessageLevel.ERROR, message= f"Sources of type wms must include the 'available_projections' element.", )) else: # A WMS server can include many CRS. Some of them are frequently used by editors. We require them to be included if they are supported by the WMS server. crs_should_included_if_available = { "EPSG:4326", "EPSG:3857", "CRS:84" } for layer_name in layers: if layer_name in wms.layers: # Check for CRS in available_projections that are not advertised by the WMS server not_supported_crs: Set[str] = set() available_projections: List[str] = source["properties"][ "available_projections"] for crs in available_projections: if crs.upper() not in wms.layers[layer_name].crs: not_supported_crs.add(crs) if len(not_supported_crs) > 0: supported_crs_str = ",".join( wms.layers[layer_name].crs) not_supported_crs_str = ",".join(not_supported_crs) messages.append( Message( level=MessageLevel.WARNING, message= f"Layer '{layer_name}': CRS '{not_supported_crs_str}' not in: {supported_crs_str}. Some server support CRS which are not advertised.", )) # Check for CRS supported by the WMS server but not in available_projections supported_but_not_included: Set[str] = set() for crs in crs_should_included_if_available: if crs not in available_projections and crs in wms.layers[ layer_name].crs: supported_but_not_included.add(crs) if len(supported_but_not_included) > 0: supported_but_not_included_str = ",".join( supported_but_not_included) messages.append( Message( level=MessageLevel.WARNING, message= f"Layer '{layer_name}': CRS '{supported_but_not_included_str}' not included in available_projections but supported by server.", )) # Check if server supports a newer WMS version as in url if wms_args["version"] < wms.version: messages.append( Message( level=MessageLevel.WARNING, message= f"Query requests WMS version '{wms_args['version']}', server supports '{wms.version}'", )) # Check image formats request_imagery_format = wms_args["format"] wms_advertised_formats_str = "', '".join(wms.formats) if request_imagery_format not in wms.formats: messages.append( Message( level=MessageLevel.ERROR, message= f"Format '{request_imagery_format}' not in '{wms_advertised_formats_str}': {url}.", )) # For photo sources it is recommended to use jpeg format, if it is available if "category" in source["properties"] and "photo" in source["properties"][ "category"]: if "jpeg" not in request_imagery_format and "jpeg" in wms.formats: messages.append( Message( level=MessageLevel.WARNING, message= f"Server supports JPEG, but '{request_imagery_format}' is used. " f"JPEG is typically preferred for photo sources, but might not be always " f"the best choice. (Server supports: '{wms_advertised_formats_str}')", ))
# jsonschema validate try: validator.validate(source, schema) except Exception as e: messages.append( Message( level=MessageLevel.ERROR, message=f"{filename} JSON validation error: {e}", )) logger.info(f"Type: {source['properties']['type']}") # Check geometry if "geometry" in source: geom = shape(source["geometry"]) # type: ignore # Check if geometry is a valid (e.g. no intersection etc.) if not geom.is_valid: # type: ignore try: reason = explain_validity(geom) # type: ignore messages.append( Message( level=MessageLevel.ERROR, message=f"{filename} invalid geometry: {reason}", )) valid_geom = make_valid(geom) # type: ignore valid_geom = eliutils.orient_geometry_rfc7946( valid_geom) # type: ignore valid_geom_json = json.dumps( mapping(valid_geom),
#Transfer Variables pow2_bounds = None with fiona.open(args.input_shp, 'r', 'ESRI Shapefile') as input_shp_fh: print(f"CRS: {input_shp_fh.crs}") print(f"Features #: {str(len(input_shp_fh))}") count=0 for data_feature in input_shp_fh: if count > FEATURE_COUNT_LIMIT: break pprint(data_feature['properties']) #get feature geometry df_geom = shape(data_feature['geometry']) print(f"Shapefile Geom bounds: {df_geom.bounds}") pow2_bounds = get_pow2_extents(get_bounds_1x1km(df_geom.bounds), TILE_SIZE) print(f"Bounds^2: {str(pow2_bounds)}") """ Gen QuadTree """ rootrect = list(pow2_bounds) rootnode = Node(None, rootrect) tree = QuadTree(rootnode, TILE_SIZE, df_geom) print(f"Leaves: {str(len(tree.leaves))}") out_shp_schema = {
def areNeighbors(geom1, geom2): geom1 = geo.shape(geom1) geom2 = geo.shape(geom2) if geom1.intersects(geom2): return True return False
def parse_args(self, args): #program_name = os.path.basename(sys.argv[0]) parser = argparse.ArgumentParser() # description='', usage = '' #parser.add_argument("-w", "--worker", default=None, help="worker (i/n)") parser.add_argument("-l", "--limit", type=int, default=None, help="tasks limit") parser.add_argument("--name", type=str, default=None, help="base name for output") parser.add_argument("--center", type=str, default=None, help="center of target area (lon, lat)") parser.add_argument("--area", type=str, default=None, help="target area polygon GeoJSON") parser.add_argument("--radius", type=float, default=None, help="radius of target area (m)") parser.add_argument("--size", type=float, default=None, help="tile size or 0 (m)") parser.add_argument("--xyztile", type=str, default=None, help="XYZ grid tile") args = parser.parse_args(args) self.limit = args.limit self.name = args.name if args.area: self.area = shape(json.loads(args.area)) self._radius = None else: self.area = None self._radius = args.radius self.chunk_size = args.size self.xyztile = None if (args.xyztile): if (args.radius or args.center or args.size): logger.error( "Option --xyztile cannot be used with --radius, --center or --size ." ) sys.exit(2) x, y, z = args.xyztile.split(",") self.xyztile = int(x), int(y), int(z) else: center = args.center.split(",") self.center = (float(center[0]), float(center[1]))
def load_geojson(self, files): features = [] for f in files: fs = geojson.load(open(f, 'r')) features.extend(fs['features']) seen = set() dedup = [] features_custom = [] for f in features: #oid = hash(str(f)) # f['properties']['osm_id'] # TODO: better way to distinguish custom features if 'id' not in f: features_custom.append(f) continue oid = f['id'] if oid not in seen: seen.add(oid) dedup.append(f) logger.info("Loaded %d features (%d unique)" % (len(features), len(dedup))) features = dedup # Project to local # TODO: Do this with self.project coordinates after creating shapes (TEST!) transformer = pyproj.Transformer.from_proj(self.osm_proj, self.ddd_proj) for f in features: if not f['geometry']['coordinates']: logger.info("Feature with no coordinates: %s", f) else: try: f['geometry']['coordinates'] = project_coordinates( f['geometry']['coordinates'], transformer) except Exception as e: logger.error("Cannot project coordinates for: %s", f) # Filter "supertile" filtered = [] for f in features: if not f['geometry']['coordinates']: continue try: geom = shape(f['geometry']) except Exception as e: logger.warn( "Could not load feature with invalid geometry (%s): %s", str(f.properties)[:240], e) continue #geom = make_valid(geom) #if self.area_filter.contains(geom.centroid): try: if self.area_filter.intersects(geom): filtered.append(f) except Exception as e: logger.debug( "Could not load feature (1/2) with invalid geometry (%s): %s", str(f.properties)[:240], e) # Attempt intersection first try: geom = geom.intersection(self.area_filter) if self.area_filter.intersects(geom): filtered.append(f) except Exception as e: logger.warn( "Could not load feature (2/2) with invalid geometry (%s): %s", f.properties, e) continue features = filtered logger.info("Using %d features after filtering to %s" % (len(features), self.area_filter.bounds)) self.features = features self.features_custom = features_custom
def plot_flow_directions_and_basin_boundaries(ax, s, sim_name_to_station_to_model_point, sim_name_to_manager=None): assert isinstance(ax, Axes) assert isinstance(s, Station) assert isinstance(sim_name_to_station_to_model_point, dict) mp_list = list(sim_name_to_station_to_model_point.items())[0][1][s] #selecting only one (the first model point) mp = mp_list[0] manager = list(sim_name_to_manager.items())[0][1] assert isinstance(manager, Crcm5ModelDataManager) flow_in_mask = manager.get_mask_for_cells_upstream(mp.ix, mp.jy) lons2d, lats2d = manager.lons2D, manager.lats2D i_upstream, j_upstream = np.where(flow_in_mask == 1) nx_total, ny_total = lons2d.shape imin, imax = np.min(i_upstream), np.max(i_upstream) jmin, jmax = np.min(j_upstream), np.max(j_upstream) margin = 8 imin = max(0, imin - margin) jmin = max(0, jmin - margin) imax = min(nx_total - 1, imax + margin) jmax = min(ny_total - 1, jmax + margin) sub_lons2d = lons2d[imin:imax + 1, jmin:jmax + 1] sub_lats2d = lats2d[imin:imax + 1, jmin:jmax + 1] sub_flow_directions = manager.flow_directions[imin:imax + 1, jmin:jmax + 1] sub_flow_in_mask = flow_in_mask[imin:imax + 1, jmin:jmax + 1] sub_i_upstream, sub_j_upstream = np.where(sub_flow_in_mask == 1) basemap = Crcm5ModelDataManager.get_rotpole_basemap_using_lons_lats( lons2d=sub_lons2d, lats2d=sub_lats2d, resolution="h" ) #plot all stations #stations = sim_name_to_station_to_model_point.items()[0][1].keys() x_list = [the_station.longitude for the_station in (s,)] y_list = [the_station.latitude for the_station in (s,)] basemap_big = Crcm5ModelDataManager.get_rotpole_basemap_using_lons_lats( lons2d=lons2d, lats2d=lats2d ) x_list, y_list = basemap_big(x_list, y_list) basemap_big.scatter(x_list, y_list, c="r", s=40, linewidths=0, ax=ax) basemap_big.drawcoastlines(ax=ax) basemap_big.drawrivers(ax=ax) ax.annotate(s.id, xy=basemap(s.longitude, s.latitude), xytext=(3, 3), textcoords='offset points', font_properties=FontProperties(weight="bold"), bbox=dict(facecolor='w', alpha=1), ha="left", va="bottom", zorder=2) x_big, y_big = basemap_big(manager.lons2D, manager.lats2D) ####zoom to the area of interest #axins = zoomed_inset_axes(ax, 3, loc=2) displayCoords = ax.transData.transform((x_big[imin, jmin], y_big[imin, jmin])) x_shift_fig, y_shift_fig = ax.figure.transFigure.inverted().transform(displayCoords) print("After transData", displayCoords) print("xshift and yshift", x_shift_fig, y_shift_fig) #ax.annotate("yuyu", xy = ( 0.733264985153, 0.477182994408), xycoords = "figure fraction" ) rect = [0.1, y_shift_fig + 0.1, 0.4, 0.4] axins = ax.figure.add_axes(rect) #assert isinstance(axins, Axes) x, y = basemap(sub_lons2d, sub_lats2d) x1d_start = x[sub_flow_in_mask == 1] y1d_start = y[sub_flow_in_mask == 1] fld1d = sub_flow_directions[sub_flow_in_mask == 1] from util import direction_and_value ishift, jshift = direction_and_value.flowdir_values_to_shift(fld1d) sub_i_upstream_next = sub_i_upstream + ishift sub_j_upstream_next = sub_j_upstream + jshift u = x[sub_i_upstream_next, sub_j_upstream_next] - x1d_start v = y[sub_i_upstream_next, sub_j_upstream_next] - y1d_start u2d = np.ma.masked_all_like(x) v2d = np.ma.masked_all_like(y) u2d[sub_i_upstream, sub_j_upstream] = u v2d[sub_i_upstream, sub_j_upstream] = v basemap.quiver(x, y, u2d, v2d, angles="xy", scale_units="xy", scale=1, ax=axins) x_list = [the_station.longitude for the_station in (s,)] y_list = [the_station.latitude for the_station in (s,)] x_list, y_list = basemap(x_list, y_list) basemap.scatter(x_list, y_list, c="r", s=40, linewidths=0) basemap.drawcoastlines(ax=axins) basemap.drawrivers(ax=axins) #read waterbase file, and plot only the polygons which contain at least one upstream point shp_path = "/skynet3_exec1/huziy/Netbeans Projects/Java/DDM/data/shape/waterbase/na_bas_ll_r500m/na_bas_ll_r500m.shp" c = fiona.open(shp_path) hits = c.filter(bbox=(sub_lons2d[0, 0], sub_lats2d[0, 0], sub_lons2d[-1, -1], sub_lats2d[-1, -1])) points = [Point(the_x, the_y) for the_x, the_y in zip(x1d_start, y1d_start)] selected_polygons = [] for feature in hits: new_coords = [] old_coords = feature["geometry"]["coordinates"] #transform to the map coordinates for ring in old_coords: x1 = [tup[0] for tup in ring] y1 = [tup[1] for tup in ring] x2, y2 = basemap(x1, y1) new_coords.append(list(zip(x2, y2))) feature["geometry"]["coordinates"] = new_coords poly = shape(feature["geometry"]) #print poly, type(poly) #print feature.keys() #print feature["properties"] prep_poly = prep.prep(poly) hits = list(filter(prep_poly.contains, points)) if len(hits) > 2: selected_polygons.append(feature["geometry"]) for p in selected_polygons: axins.add_patch(PolygonPatch(p, fc="none", ec="b", lw=1.5)) zoom_lines_color = "#6600FF" #set color of the frame for child in axins.get_children(): if isinstance(child, Spine): child.set_color(zoom_lines_color) child.set_linewidth(3) # draw a bbox of the region of the inset axes in the parent axes and # connecting lines between the bbox and the inset axes area mark_inset(ax, axins, loc1=1, loc2=3, fc="none", ec=zoom_lines_color, lw=3) #basemap.readshapefile(, "basin") pass
def handle(self, **options): shape_file = options['shape'] interpret = options['interpret'] predict = options['predict'] dataset = options['dataset'] year = options['year'] scheme = options['scheme'] filter = options['filter'] filename = basename(shape_file, False) if filter is not None and filter <= 1: filter = None with fiona.open(shape_file) as source: project = partial(pyproj.transform, pyproj.Proj(source.crs), pyproj.Proj(init='EPSG:4326')) if filter == None: object_list = [(TrainObject(the_geom=GEOSGeometry( transform(project, shape(feat['geometry'])).wkt), filename=filename, creation_year=year), feat['properties']) for feat in source] else: object_list = [ (TrainObject(the_geom=GEOSGeometry( transform(project, shape(feat['geometry'])).wkt), filename=filename, creation_year=year), feat['properties']) for ind, feat in enumerate(source) if ind % filter == 0 ] TrainObject.objects.bulk_create(list(map(lambda x: x[0], object_list))) train_classification_objects = [] tag_map = {} for tup in object_list: o = tup[0] p = tup[1] value = p[interpret] interpret_tag = tag_map.get(value) if not interpret_tag: interpret_tag, created = Tag.objects.get_or_create( numeric_code=value, scheme=scheme) tag_map[value] = interpret_tag if created: logger.info( 'New tag created with values: numeric_code=%s, scheme=%s' % (value, scheme)) predict_tag = None if predict is not None: value = p[predict] predict_tag = tag_map.get(value) if not predict_tag: try: predict_tag = Tag.objects.get(numeric_code=value, scheme=scheme) except Tag.DoesNotExist: predict_tag = Tag.objects.get(numeric_code=value, scheme=scheme) predict_tag.save() tag_map[value] = predict_tag train_classification_objects.append( TrainClassification(train_object=o, predict_tag=predict_tag, interpret_tag=interpret_tag, training_set=dataset)) TrainClassification.objects.bulk_create(train_classification_objects)