def merge_to_multi_polygon(feature_collection: str,
dissolve: bool) -> geojson.MultiPolygon:
"""
Merge all geometries to a single multipolygon
:param feature_collection: geojson feature collection str containing features
:param dissolve: flag for wther to to dissolve internal boundaries.
:return: geojson.MultiPolygon
"""
parsed_geojson = GridService._to_shapely_geometries(
json.dumps(feature_collection))
multi_polygon = GridService._convert_to_multipolygon(parsed_geojson)
if dissolve:
multi_polygon = GridService._dissolve(multi_polygon)
aoi_multi_polygon_geojson = geojson.loads(
json.dumps(mapping(multi_polygon)))
# validate the geometry
if type(aoi_multi_polygon_geojson) is not geojson.MultiPolygon:
raise InvalidGeoJson(
"Area Of Interest: geometry must be a MultiPolygon")
is_valid_geojson = geojson.is_valid(aoi_multi_polygon_geojson)
if is_valid_geojson["valid"] == "no":
raise InvalidGeoJson(
f"Area of Interest: Invalid MultiPolygon - {is_valid_geojson['message']}"
)
return aoi_multi_polygon_geojson
def create_feature_collection(self, csv_file):
"""
Parse a CSV to create a FeatureCollection object for all postcodes.
:param csv_file: csv file name
:return: FeatureCollection of all postcodes in file
"""
all_features = list()
coord_ref = crs.Named(
properties={'name': 'urn:ogc:def:crs:EPSG::27700'})
with open(path.join(self.cur_dir, csv_file), 'r') as f:
reader = csv.reader(f)
for row in reader:
postcode = str(row[0]).replace(' ', '')
easting = int(row[2])
northing = int(row[3])
point = Point(coordinates=(easting, northing))
feature = Feature(geometry=point,
properties={'postcode': postcode})
lol = is_valid(feature)
all_features.append(feature)
# print(feature)
# Set CRS to BNG
# feature_collection = FeatureCollection(all_features, crs=coord_ref)
return all_features
def _attach_tasks_to_project(draft_project: Project, tasks_geojson):
"""
Validates then iterates over the array of tasks and attach them to the draft project
:param draft_project: Draft project in scope
:param tasks_geojson: GeoJSON feature collection of mapping tasks
:raises InvalidGeoJson, InvalidData
"""
tasks = geojson.loads(json.dumps(tasks_geojson))
if type(tasks) is not geojson.FeatureCollection:
raise InvalidGeoJson(
"Tasks: Invalid GeoJson must be FeatureCollection")
is_valid_geojson = geojson.is_valid(tasks)
if is_valid_geojson["valid"] == "no":
raise InvalidGeoJson(
f"Tasks: Invalid FeatureCollection - {is_valid_geojson['message']}"
)
task_count = 1
for feature in tasks["features"]:
try:
task = Task.from_geojson_feature(task_count, feature)
except (InvalidData, InvalidGeoJson) as e:
raise e
draft_project.tasks.append(task)
task_count += 1
task_count -= 1 # Remove last increment before falling out loop
draft_project.total_tasks = task_count
def to_geojson(self, json_path=None, places=True, massifs=True, circuits=True, boulders=False):
features = []
if places:
features.extend([place for place in self.places if place])
if massifs:
features.extend([massif for massif in self.massifs if massif])
if circuits:
if boulders:
exported_circuits = [circuit for circuit in self.circuits if circuit]
else:
exported_circuits = [circuit.to_feature() for circuit in self.circuits if circuit]
features.extend(exported_circuits)
feature_collections = geojson.FeatureCollection(features)
if not geojson.is_valid(feature_collections):
raise ValueError('Non valid GeoJSON')
# Fixme: crs geojson.named API
kwargs = dict(indent=2, ensure_ascii=False, sort_keys=True)
if json_path is not None:
with open(json_path, 'w', encoding='utf8') as f:
geojson.dump(feature_collections, f, **kwargs)
else:
return geojson.dumps(feature_collections, **kwargs)
def extract(self, target, data):
s = kmltogeojson(data)
if s:
data = geojson.loads(s)
result = geojson.is_valid(data)
if result['valid'] == 'yes':
return [('geojson', data)]
def from_geojson_feature(cls, task_id, task_feature):
"""
Constructs and validates a task from a GeoJson feature object
:param task_id: Unique ID for the task
:param task_feature: A geoJSON feature object
:raises InvalidGeoJson, InvalidData
"""
if type(task_feature) is not geojson.Feature:
raise InvalidGeoJson('Task: Invalid GeoJson should be a feature')
task_geometry = task_feature.geometry
if type(task_geometry) is not geojson.MultiPolygon:
raise InvalidGeoJson('Task: Geometry must be a MultiPolygon')
is_valid_geojson = geojson.is_valid(task_geometry)
if is_valid_geojson['valid'] == 'no':
raise InvalidGeoJson(
f"Task: Invalid MultiPolygon - {is_valid_geojson['message']}")
task = cls()
try:
task.x = task_feature.properties['x']
task.y = task_feature.properties['y']
task.zoom = task_feature.properties['zoom']
task.splittable = task_feature.properties['splittable']
except KeyError as e:
raise InvalidData(f'Task: Expected property not found: {str(e)}')
task.id = task_id
task_geojson = geojson.dumps(task_geometry)
task.geometry = ST_SetSRID(ST_GeomFromGeoJSON(task_geojson), 4326)
return task
def test_write_stops_geojson(self):
in_memory_file = io.StringIO()
exports.write_stops_geojson(self.gtfs, in_memory_file)
in_memory_file.seek(0)
self.assertTrue(geojson.is_valid(in_memory_file.read(-1)))
in_memory_file.seek(0)
gjson = geojson.loads(in_memory_file.read(-1))
gjson_properties = gjson['features'][0]['properties']
self.assertIn("name", gjson_properties.keys())
self.assertIn("stop_I", gjson_properties.keys())
def test_footprints_s2(products):
footprints = SentinelAPI.to_geojson(products)
for footprint in footprints['features']:
validation = geojson.is_valid(footprint['geometry'])
assert validation['valid'] == 'yes', validation['message']
with open('tests/expected_search_footprints_s2.geojson') as geojson_file:
expected_footprints = geojson.loads(geojson_file.read())
# to compare unordered lists (JSON objects) they need to be sorted or changed to sets
assert set(footprints) == set(expected_footprints)
def test_footprints_s2(products):
footprints = SentinelAPI.to_geojson(products)
for footprint in footprints['features']:
validation = geojson.is_valid(footprint['geometry'])
assert validation['valid'] == 'yes', validation['message']
with open('tests/expected_search_footprints_s2.geojson') as geojson_file:
expected_footprints = geojson.loads(geojson_file.read())
# to compare unordered lists (JSON objects) they need to be sorted or changed to sets
assert set(footprints) == set(expected_footprints)
def extract(self, target, data):
t = data.read()
try:
data = geojson.loads(t)
except:
# possibly malformed json?
start = min(t.find('{'), t.find('['))
data = geojson.loads(t[start:])
result = geojson.is_valid(data)
if result['valid'] == 'yes':
return [('geojson', data)]
def post():
''' This is the main POST method that takes in a request from Geodesign Hub and processes the input data '''
# check if the request headers are specified and the data sent are JSON
if request.headers['Content-Type'] == 'application/json':
pass
else:
# If the media type is not JSON, return a HTTP 415 message
msg = {"message":"Unsupported Media Type"}
return Response(json.dumps(msg), status=415, mimetype='application/json')
# Load the JSON data from Geodesign Hub
req = json.loads(request.data)
# check that
try:
# check that the data coming in from Geodesign Hub has all the keys
jobid = req['jobid']
callback = req['callback']
geometries = req['geometry']
jobtype = req['type']
projectname = req['projectname']
except KeyError as e:
msg = json.dumps({"message":"Four parameters are required: jobid, jobtype, callback, geometry and Geodesign Hub projectname. One or more of these were not found in your JSON request."})
return Response(msg, status=400, mimetype='application/json')
# Once the data has been verified, parse it.
gjData = geojson.loads(json.dumps(geometries))
# validate geojson
validation = geojson.is_valid(gjData)
# if geojson is is valid
if validation['valid'] == 'yes':
# Implement your impacts code here. The code below takes all the features and assigns a random impacts to them.
impactsAssignmentList = ['orange', 'orange2', 'yellow','purple', 'purple2']
allFeats = []
for feature in gjData['features']:
feature['properties']['areatype'] = random.choice(impactsAssignmentList)
allFeats.append(feature)
# Once all features have been parsed, build a feature collection
fc = {"type":"FeatureCollection", "features":allFeats}
# Setup the API.
myAPIHelper = GeodesignHub.GeodesignHubClient(url = config.apisettings['serviceurl'], token=config.apisettings['apitoken'])
# Send the impact geometries back to the project
upload = myAPIHelper.post_gdservice_JSON(geometry=fc, jobid=jobid)
# if status code is 202 / Accepted then all Ok
if upload.status_code == 202:
pass
else:
# If invalid geoJSON is submitted, return a 400 message.
msg = json.dumps({"message":"Invalid GeoJSON submitted."})
return Response(msg, status=400, mimetype='application/json')
op = json.dumps ({"message":"OK"})
return Response(op, status=200, mimetype='application/json')
def validate_geojson(geojson):
"""
Validate string is GeoJson.
:param geojson: GeoJson string
:return: bool
"""
try:
if is_valid(loads(geojson))['valid'].lower() == 'yes':
return True
else:
return False
except:
return False
def validate_geojson(geojson_path):
""" Validate a GeoJSON file """
## Validate the GeoJSON file
with open(geojson_path, 'r') as geojson_file:
geojson_dump = geojson_file.read()
features = geojson.loads(geojson_dump)
validation = geojson.is_valid(features)
print "Is the geojson file valid? ", validation["valid"]
if validation["message"]:
print "Info: ", validation["message"]
return validation["valid"]
def _index_spatial_property(self, node, key, value, label):
sidx_key = u"{}_{}".format(label, key)
if sidx_key in self.sidx:
geo_value = geojson.loads(value)
is_valid_geojson = geojson.is_valid(geo_value)['valid'] != 'no'
if is_valid_geojson:
# Add node to index
index = self.sidx[sidx_key]["index"]
index_key = self.sidx[sidx_key]["key"]
wkt_value = shape(geo_value).wkt
node[index_key] = wkt_value
index.add(SPATIAL_INDEX_KEY, SPATIAL_INDEX_VALUE, node)
# Add node to layer
self.spatial.addNodeToLayer(layer=index.name, node=node.url)
def test_write_sections_geojson(self):
in_memory_file = io.StringIO()
exports.write_sections_geojson(self.gtfs, in_memory_file)
in_memory_file.seek(0)
self.assertTrue(geojson.is_valid(in_memory_file.read(-1)))
in_memory_file.seek(0)
gjson = geojson.loads(in_memory_file.read(-1))
gjson_properties = gjson['features'][0]['properties']
self.assertIn("from_stop_I", gjson_properties.keys())
self.assertIn("to_stop_I", gjson_properties.keys())
self.assertIn("n_vehicles", gjson_properties.keys())
self.assertIn("duration_avg", gjson_properties.keys())
self.assertIn("route_I_counts", gjson_properties.keys())
self.assertIn("route_type", gjson_properties.keys())
def _index_spatial_property(self, node, key, value, label):
sidx_key = u"{}_{}".format(label, key)
if sidx_key in self.sidx:
geo_value = geojson.loads(value)
is_valid_geojson = geojson.is_valid(geo_value)['valid'] != 'no'
if is_valid_geojson:
# Add node to index
index = self.sidx[sidx_key]["index"]
index_key = self.sidx[sidx_key]["key"]
wkt_value = shape(geo_value).wkt
node[index_key] = wkt_value
index.add(SPATIAL_INDEX_KEY, SPATIAL_INDEX_VALUE, node)
# Add node to layer
self.spatial.addNodeToLayer(layer=index.name, node=node.url)
def _convert_to_geojson(network):
nodes = network.get('nodes')
links = network.get('links')
projection = network.get('projection', '')
feature_list = []
_build_feature_list_links(feature_list, links, projection)
_build_feature_list_nodes(feature_list, nodes, projection)
feature_collection = geojson.FeatureCollection(feature_list)
print(geojson.is_valid(feature_collection))
return feature_collection
def test_footprints_s1():
api = SentinelAPI(**_api_auth)
products = api.query(geojson_to_wkt(read_geojson('tests/map.geojson')),
datetime(2014, 10, 10),
datetime(2014, 12, 31),
producttype="GRD")
footprints = api.to_geojson(products)
for footprint in footprints['features']:
validation = geojson.is_valid(footprint['geometry'])
assert validation['valid'] == 'yes', validation['message']
with open('tests/expected_search_footprints_s1.geojson') as geojson_file:
expected_footprints = geojson.loads(geojson_file.read())
# to compare unordered lists (JSON objects) they need to be sorted or changed to sets
assert set(footprints) == set(expected_footprints)
def test_footprints_s1():
api = SentinelAPI(**_api_auth)
products = api.query(
geojson_to_wkt(read_geojson('tests/map.geojson')),
datetime(2014, 10, 10), datetime(2014, 12, 31), producttype="GRD"
)
footprints = api.to_geojson(products)
for footprint in footprints['features']:
validation = geojson.is_valid(footprint['geometry'])
assert validation['valid'] == 'yes', validation['message']
with open('tests/expected_search_footprints_s1.geojson') as geojson_file:
expected_footprints = geojson.loads(geojson_file.read())
# to compare unordered lists (JSON objects) they need to be sorted or changed to sets
assert set(footprints) == set(expected_footprints)
def from_dict(cls, area_poly: dict):
""" Create a new Priority Area from dictionary """
pa_geojson = geojson.loads(json.dumps(area_poly))
if type(pa_geojson) is not geojson.Polygon:
raise InvalidGeoJson("Priority Areas must be supplied as Polygons")
is_valid_geojson = geojson.is_valid(pa_geojson)
if is_valid_geojson["valid"] == "no":
raise InvalidGeoJson(
f"Priority Area: Invalid Polygon - {is_valid_geojson['message']}"
)
pa = cls()
valid_geojson = geojson.dumps(pa_geojson)
pa.geometry = ST_SetSRID(ST_GeomFromGeoJSON(valid_geojson), 4326)
return pa
def check(self):
result = CheckResult(geoservice_id=self.service.id,
geoservice_name=self.service.name,
geoservice_type=self.service.type)
startTime = datetime.datetime.utcnow()
try:
response = requests.get(self.service.url, timeout=self.timeout)
result.http_code = response.status_code
# content-type не проверяется, вместо этого проверяем код ответа
# и если он равен 200, то выполняем валидацию содержимого ответа на geojson
if response.status_code == 200:
geojson_obj = geojson.loads(response.text)
validation = geojson.is_valid(geojson_obj)
if validation['valid'] == 'yes':
result.data = response.text
result.cumulative_status = CumulativeStatus.WORKS
else:
result.cumulative_status = CumulativeStatus.PROBLEMATIC
result.error_text = validation['message']
result.error_type = CheckStatusErrorType.INVALID_RESPONSE
else:
result.cumulative_status = CumulativeStatus.PROBLEMATIC
result.error_text = 'Non 200 http code'
result.http_response = response.text
result.error_type = CheckStatusErrorType.INVALID_RESPONSE
# если requests вернул код ошибки веб-сервера
except HTTPError as error:
result.cumulative_status = CumulativeStatus.FAILED
result.error_text = unicode(error)
# исключение по таймауту 10 секунд
except Timeout as error:
result.cumulative_status = CumulativeStatus.FAILED
result.error_type = CheckStatusErrorType.TIMEOUT_ERROR
except Exception as error:
result.cumulative_status = CumulativeStatus.FAILED
result.error_text = unicode(error)
duration_time = datetime.datetime.utcnow() - startTime
result.check_duration = duration_time.total_seconds()
return result
def csv_to_json(self):
"""
Main run method
"""
csv_list = self.get_csv()
if csv_list:
for csv_f in csv_list:
print csv_f
fc = self.create_feature_collection(csv_f)
validation = is_valid(fc)
if validation['valid'] == 'yes':
self.encode_to_raw_json(fc, csv_f)
else:
print "Failed to create GeoJson for file: " + csv_f + " - Error: " + str(validation['message'])
print "Finished!"
else:
print "No CSV's to process"
def upload_docs_to_ES(docs,index,doc_type,id_field="",geopoint=False):
#input: list of JSON documents, an index name, document type, ID field, and name of an (OPTIONAL) geopoint field.
#uploads each feature element to ElasticSearch
#es = Elasticsearch(ES_url)
es = Elasticsearch(['http://' + ES_username + ':' + ES_password + '@' + ES_url + ':9200/'])
try:
es.indices.create(index)
except:
#do not try to recreate the index
pass
#if the data has a location field, set the geo_point mapping
if geopoint:
mapping = {doc_type:{'properties':{geopoint:{'type':'geo_point','store':'yes'}}}}
es.indices.put_mapping(index=index, doc_type=doc_type, body=mapping)
actions = []
#build the list of ElasticSearch uploads for bulk command
for doc in docs:
action = {
'_index': index,
'_type': doc_type,
}
#check if the document is a geojson document
validation = geojson.is_valid(doc)
if validation['valid'].lower() == 'yes':
#add the point to the document properties
doc['properties'][geopoint] = list(geojson.utils.coords(doc))[0]
#load the document properties into ES
action['_source'] = doc['properties']
#get id from geojson properties document
if len(id_field) != 0:
action['_id'] = doc['properties'][id_field]
else:
#assign id for typical json document
if len(id_field) != 0:
action['_id'] = doc[id_field]
action['_source'] = doc
actions.append(action)
helpers.bulk(es, actions)
def ExporttoJson(geometry_collection):
feature_lst = []
for geometry in geometry_collection:
json_geom = geojson.loads(geometry.ExportToJson())
feature = geojson.Feature(geometry=json_geom)
feature_lst.append(feature)
feature_coll = geojson.FeatureCollection(feature_lst)
validation = geojson.is_valid(feature_coll)
if validation['valid'] is 'yes':
# pprint(geojson.dumps(feature_json, sort_keys=False))
return geojson.dumps(feature_coll, sort_keys=True)
else:
raise TypeError('This is not a valid geojson string.')
def test_ways(self, testapp):
""" Tests that after OSM data is imported, valid GeoJSON can be returned """
nodes, ways = osm.map_data_to_nodes_ways(mock_map_data)
osm.update_db_ways(ways)
rv = testapp.get('/api/0.1/ways')
rv_str = rv.data.decode('utf-8')
assert rv.status_code == 200
assert 'FeatureCollection' in rv_str
assert 'Feature' in rv_str
assert 'LineString' in rv_str
way_geo = geojson.loads(rv_str)
assert way_geo['type'] == 'FeatureCollection'
assert way_geo['features'][0]['type'] == 'Feature'
assert way_geo['features'][0]['geometry']['type'] == 'LineString'
assert 'id' in way_geo['features'][0]['properties']['osm_data']
assert 'yes' == geojson.is_valid(way_geo)['valid']
def from_geojson_feature(cls, task_id, task_feature):
"""
Constructs and validates a task from a GeoJson feature object
:param task_id: Unique ID for the task
:param task_feature: A geoJSON feature object
:raises InvalidGeoJson, InvalidData
"""
if type(task_feature) is not geojson.Feature:
raise InvalidGeoJson("Task: Invalid GeoJson should be a feature")
task_geometry = task_feature.geometry
if type(task_geometry) is not geojson.MultiPolygon:
raise InvalidGeoJson("Task: Geometry must be a MultiPolygon")
is_valid_geojson = geojson.is_valid(task_geometry)
if is_valid_geojson["valid"] == "no":
raise InvalidGeoJson(
f"Task: Invalid MultiPolygon - {is_valid_geojson['message']}"
)
task = cls()
try:
task.x = task_feature.properties["x"]
task.y = task_feature.properties["y"]
task.zoom = task_feature.properties["zoom"]
task.is_square = task_feature.properties["isSquare"]
except KeyError as e:
raise InvalidData(f"Task: Expected property not found: {str(e)}")
if "extra_properties" in task_feature.properties:
task.extra_properties = json.dumps(
task_feature.properties["extra_properties"]
)
task.id = task_id
task_geojson = geojson.dumps(task_geometry)
task.geometry = ST_SetSRID(ST_GeomFromGeoJSON(task_geojson), 4326)
return task
def create_feature_collection(self, csv_file):
"""
Parse a CSV to create a FeatureCollection object for all postcodes.
:param csv_file: csv file name
:return: FeatureCollection of all postcodes in file
"""
all_features = list()
coord_ref = crs.Named(properties={'name': 'urn:ogc:def:crs:EPSG::27700'})
with open(path.join(self.cur_dir, csv_file), 'r') as f:
reader = csv.reader(f)
for row in reader:
postcode = str(row[0]).replace(' ', '')
easting = int(row[2])
northing = int(row[3])
point = Point(coordinates=(easting, northing))
feature = Feature(geometry=point, properties={'postcode': postcode})
lol = is_valid(feature)
all_features.append(feature)
# print(feature)
# Set CRS to BNG
# feature_collection = FeatureCollection(all_features, crs=coord_ref)
return all_features
def build_polygon(coords):
polygon = geojson.Polygon(coords)
check = geojson.is_valid(polygon)
if check['valid'] == 'yes':
return polygon
return tup
with open("gate_coords_66.txt") as coords:
flag = 0
for line in coords:
sask_coords = []
if flag == 0:
flag = 1
continue
words = line.split()
# print(line)
for x in range(8,12):
# print(words[x-6],words[x])
sask_coords.append((float(words[x]),float(words[x-4])))
cs.append((words[x-4], words[x]))
sask_coords.append((float(words[8]),float(words[4])))
sask_coords = [transform(x) for x in sask_coords]
sask_polys.append(geojson.Feature(geometry=geojson.Polygon([sask_coords]),id=str(words[0].zfill(2))+str(words[1].zfill(2))))
featureCollection = geojson.FeatureCollection(sask_polys)
validation = geojson.is_valid(featureCollection)
with open("coords.txt",'w') as outfile:
for c in cs:
outfile.write(c[1]+ ',' + c[0]+"\n")
with open("cly_geojsondata.json",'w') as outfile:
geojson.dump(featureCollection,outfile)
def upload_docs_to_ES(docs,
index,
doc_type,
id_field=False,
geopoint=False,
geoshape=False,
delete_index=False):
#input: list of JSON documents, an index name, document type, ID field, and name of an (OPTIONAL) geopoint field.
#uploads each feature element to ElasticSearch
#es = Elasticsearch(ES_url)
es = Elasticsearch([
'http://' + ES_username + ':' + ES_password + '@' + ES_url + ':9200/'
])
if delete_index:
try:
es.indices.delete(index=index, ignore=400)
except:
pass
try:
es.indices.create(index)
except:
#do not try to recreate the index
pass
#if the data has a location field, set the geo_point mapping
if geopoint:
try:
mapping = {
doc_type: {
'properties': {
geopoint: {
'type': 'geo_point',
'store': 'yes'
}
}
}
}
es.indices.put_mapping(index=index,
doc_type=doc_type,
body=mapping)
except:
pass
if geoshape:
try:
mapping = {
doc_type: {
'properties': {
geoshape: {
'type': 'geo_shape',
'tree': 'quadtree',
'precision': '1m'
}
}
}
}
es.indices.put_mapping(index=index,
doc_type=doc_type,
body=mapping)
except:
pass
actions = []
#build the list of ElasticSearch uploads for bulk command
index = 0
for doc in docs:
action = {
'_index': index,
'_type': doc_type,
}
#check if the document is a geojson document
validation = geojson.is_valid(doc)
if validation['valid'].lower() == 'yes':
#add the point/shape to the document properties
if geopoint:
doc['properties'][geopoint] = list(
geojson.utils.coords(doc))[0]
if geoshape:
doc['properties'][geoshape] = doc['geometry']
doc['geometry']['type'] = doc['geometry']['type'].lower(
) #convert type to lowercase for leaflet
#get id from geojson properties document
if id_field: action['_id'] = doc['properties'][id_field]
#load the document properties into ES
action['_source'] = doc['properties']
else:
#assign id for typical json document
if id_field: action['_id'] = doc[id_field]
action['_source'] = doc
actions.append(action)
index += 1
#upload 10k records at a time
if index == 10000:
try:
helpers.bulk(es, actions)
print 'Sucessfully uploaded %s records!' % str(len(actions))
actions = []
index = 0
except Exception as e:
print '#### ERROR:s'
pprint(e)
#upload remaining 10k records
try:
helpers.bulk(es, actions)
print 'Sucessfully uploaded %s records!' % str(len(actions))
except Exception as e:
print '#### ERROR:'
pprint(e)
def upload_individual_docs_to_ES(docs,
index,
doc_type,
id_field=False,
geopoint=False,
geoshape=False,
delete_index=False):
#input: list of JSON documents, an index name, document type, ID field, and name of an (OPTIONAL) geopoint field.
#uploads each feature element to ElasticSearch
#es = Elasticsearch(ES_url)
es = Elasticsearch([
'http://' + ES_username + ':' + ES_password + '@' + ES_url + ':9200/'
])
if delete_index:
try:
es.indices.delete(index=index, ignore=400)
except:
pass
try:
es.indices.create(index)
except:
#do not try to recreate the index
pass
#if the data has a location field, set the geo_point mapping
if geopoint:
try:
mapping = {
doc_type: {
'properties': {
geopoint: {
'type': 'geo_point',
'store': 'yes'
}
}
}
}
es.indices.put_mapping(index=index,
doc_type=doc_type,
body=mapping)
except:
pass
if geoshape:
try:
mapping = {
doc_type: {
'properties': {
geoshape: {
'type': 'geo_shape',
'tree': 'quadtree',
'precision': '1m'
}
}
}
}
es.indices.put_mapping(index=index,
doc_type=doc_type,
body=mapping)
except:
pass
#iterate through and upload individual documents
succeeded = 0
failed = 0
for doc in docs:
#check if the document is a geojson document
validation = geojson.is_valid(doc)
if validation['valid'].lower() == 'yes':
#add the point/shape to the document properties
if geopoint:
doc['properties'][geopoint] = list(
geojson.utils.coords(doc))[0]
if geoshape:
doc['properties'][geoshape] = doc['geometry']
doc['geometry']['type'] = doc['geometry']['type'].lower(
) #convert type to lowercase for leaflet
#load the document properties into ES
to_upload = deepcopy(doc['properties'])
else:
to_upload = deepcopy(doc)
if id_field: _id = to_upload[id_field]
#upload the document
try:
res = es.index(index=index,
doc_type=doc_type,
id=_id,
body=to_upload)
succeeded += 1
except Exception as e:
print "ERROR: " % str(e)
failed += 1
print "Finished uploading documents. %s succeeded. %s failed." % (
succeeded, failed)
def __init__(self, feature):
validation = geojson.is_valid(feature)
assert validation['valid']=='yes', validation['message']
super(Geom, self).__init__()
self.feature = feature
def renderMapView(self, mbtoken):
# generate a working pandas data frame using the fields we need
df = self.getWorkingPandasDataFrame()
keyFields = self.getKeyFields()
# geomType can be either 0: (Multi)Point, 1: (Multi)LineString, 2: (Multi)Polygon
geomType = 0
bins = []
if len(keyFields)>0:
if len(keyFields)==1:
geomType = -1 #unknown as of yet
else:
lonFieldIdx = 0
latFieldIdx = 1
if keyFields[0] == self.getLatField():
lonFieldIdx = 1
latFieldIdx = 0
min = [df[keyFields[lonFieldIdx]].min(), df[keyFields[latFieldIdx]].min()]
max = [df[keyFields[lonFieldIdx]].max(), df[keyFields[latFieldIdx]].max()]
self.options["mapBounds"] = json.dumps([min,max], default=defaultJSONEncoding)
valueFields = self.getValueFields()
#check if we have a preserveCols
preserveCols = self.options.get("preserveCols", None)
preserveCols = [a for a in preserveCols.split(",") if a not in keyFields and a not in valueFields] if preserveCols is not None else []
# Transform the data into GeoJSON for use in the Mapbox client API
allProps = valueFields + preserveCols + self.getExtraFields()
features = []
for rowidx, row in df.iterrows():
feature = {'type':'Feature',
'properties':{},
'geometry':{'type':'Point',
'coordinates':[]}}
if geomType == 0:
feature['geometry']['coordinates'] = [row[keyFields[lonFieldIdx]], row[keyFields[latFieldIdx]]]
else:
geomIdx = df.columns.get_loc(keyFields[0])+1
feature['geometry'] = json.loads(row[geomIdx])
for fld in allProps:
feature['properties'][fld] = row[fld]
features.append(feature)
if len(features)>0:
pygeojson = {'type':'FeatureCollection', 'features':features}
self.options["mapData"] = json.dumps(pygeojson,default=defaultJSONEncoding)
# Now let's figure out whether we have Line or Polygon data, if it wasn't already found to be Point
if geomType != 1:
if features[0]['geometry']['type'].endswith('LineString'):
geomType = 1
elif features[0]['geometry']['type'].endswith('Polygon'):
geomType = 2
else:
geomType = -1
#### build up the map style
# basic color
paint = {}
if geomType == 1:
paint['line-color'] = '#ff0000'
paint['line-width'] = 2
if self.options.get("coloropacity"):
paint['line-opacity'] = float(self.options.get("coloropacity")) / 100
elif geomType == 2:
paint['fill-color'] = '#ff0000'
paint['fill-opacity'] = 0.8
if self.options.get("coloropacity"):
paint['fill-opacity'] = float(self.options.get("coloropacity")) / 100
else:
paint['circle-radius'] = 12
paint['circle-color'] = '#ff0000'
paint['circle-opacity'] = 0.25
if self.options.get("coloropacity"):
paint['circle-opacity'] = float(self.options.get("coloropacity")) / 100
if (self.options.get("kind") and self.options.get("kind").find("cluster") >= 0):
paint['circle-opacity'] = 1.0
if len(valueFields) > 0:
mapValueField = valueFields[0]
self.options["mapValueField"] = mapValueField
if not self.options.get("kind"):
self.options["kind"] = "choropleth-cluster"
# if there's a numeric value field and type is not 'simple', paint the data as a choropleth map
if self.options.get("kind") and self.options.get("kind").find("simple") < 0 and len(valueFields) > 0:
# get value from the "Number of Bins" slider
numBins = int(self.options.get("numbins", 5))
# custom index, uses "Custom Base Color" hex value in options menu.
customBaseColor = self.options.get("custombasecolor", "#ff0000")
# custom index, uses "Secondary Custom Base Color" hex value in options menu.
secondaryCustomBaseColor = self.options.get("custombasecolorsecondary", "#ff0000")
# color options
bincolors = []
bincolors.append(self._getColorList('#ffffcc', '#253494', numBins)) #yellow to blue
bincolors.append(self._getMonochromeLight('#de2d26', numBins)) # reds monochrome light
bincolors.append(self._getMonochromeDark('#f7f7f7', numBins)) # grayscale monochrome dark
bincolors.append(self._getColorList('#e66101', '#5e3c99', numBins)) # orange to purple
bincolors.append(self._getColorList('#3E9E7A', '#0F091D', numBins)) # green to purple
bincolors.append(self._getMonochromeLight(customBaseColor, numBins)) # custom monochrome light (saturation)
bincolors.append(self._getMonochromeDark(customBaseColor, numBins)) # custom monochrome dark (value)
bincolors.append(self._getColorList(customBaseColor, secondaryCustomBaseColor, numBins)) # custom range (value)
bincolorsIdx = 0
if self.options.get("colorrampname"):
if self.options.get("colorrampname") == "Light to Dark Red":
bincolorsIdx = 1
if self.options.get("colorrampname") == "Grayscale":
bincolorsIdx = 2
if self.options.get("colorrampname") == "Orange to Purple":
bincolorsIdx = 3
if self.options.get("colorrampname") == "Green to Purple":
bincolorsIdx = 4
if self.options.get("colorrampname") == "Custom Monochrome Light":
bincolorsIdx = 5
if self.options.get("colorrampname") == "Custom Monochrome Dark":
bincolorsIdx = 6
if self.options.get("colorrampname") == "Custom Color Range":
bincolorsIdx = 7
# only use list of quantiles if it matches the number of bins
if self.options.get("quantiles") and len(self.options.get("quantiles").split(",")) == numBins:
quantileFloats = [float(x) for x in self.options.get("quantiles").split(",")]
self.debug("Using quantileFloats: %s" % quantileFloats)
for i in range(numBins):
bins.append((df[valueFields[0]].quantile(quantileFloats[i]),bincolors[bincolorsIdx][i%len(bincolors[bincolorsIdx])]))
else:
# default, equal-size bins based on numBins (if cannot find quantiles array in options)
self.debug("Using equal-size bins based on numBins: %s" % numBins)
for i in range(numBins):
bins.append((df[valueFields[0]].quantile(float(i)/(numBins-1.0)),bincolors[bincolorsIdx][i%len(bincolors[bincolorsIdx])]))
if geomType == 1:
# paint['line-opacity'] = 0.65
paint['line-color'] = {"property":mapValueField}
paint['line-color']['stops'] = []
for bin in bins:
paint['line-color']['stops'].append([bin[0], bin[1]])
elif geomType == 2:
paint['fill-color'] = {"property":mapValueField}
paint['fill-color']['stops'] = []
for bin in bins:
paint['fill-color']['stops'].append([bin[0], bin[1]])
else:
# paint['circle-opacity'] = 0.65
paint['circle-color'] = {"property":mapValueField}
paint['circle-color']['stops'] = []
for bin in bins:
paint['circle-color']['stops'].append([bin[0], bin[1]])
paint['circle-radius'] = 12
self.options["mapStyle"] = json.dumps(paint,default=defaultJSONEncoding)
w = self.getPreferredOutputWidth()
h = self.getPreferredOutputHeight()
# handle custom layers
userlayers = []
l = (ShellAccess,ShellAccess)
papp = self.options.get("nostore_pixieapp")
if papp is not None and ShellAccess[papp] is not None:
l = (ShellAccess[papp], dir(ShellAccess[papp]))
for key in [a for a in l[1] if not callable(getattr(l[0], a)) and not a.startswith("_")]:
v = getattr(l[0],key)
if isinstance(v, dict) and "maptype" in v and v["maptype"].lower() == "mapbox" and "source" in v and "type" in v["source"] and v["source"]["type"] == "geojson" and "id" in v and "data" in v["source"]:
gj = geojson.loads(json.dumps(v["source"]["data"]))
isvalid = True
if hasattr(geojson, "is_valid"): # then we're using old version of geojson module
isvalid = geojson.is_valid(gj)["valid"] == "yes"
self.debug("IN hasattr(geojson,is_valid). Validity is "+str(isvalid))
else: # we're using a newer version of geojson module
isvalid = gj.is_valid
if isvalid:
userlayers.append(v)
else:
self.debug("Invalid GeoJSON: {0}".format(str(v["source"]["data"])))
self.debug("userlayers length: "+str(len(userlayers)))
# end handle custom layers
uniqueid = str(uuid.uuid4())[:8]
return self.renderTemplate("mapView.html", bins=bins, userlayers=userlayers, prefwidth=w, prefheight=h, randomid=uniqueid)
def pre_validate(self, form):
if self.data:
result = geojson.is_valid(self.data)
if result['valid'] == 'no':
raise validators.ValidationError(result['message'])
return True
def renderMapView(self, mbtoken):
df = self.getWorkingPandasDataFrame()
keyFields = self.getKeyFields()
# geomType can be either 0: (Multi)Point, 1: (Multi)LineString, 2: (Multi)Polygon
geomType = 0
bins = []
if len(keyFields) > 0:
if len(keyFields) == 1:
geomType = -1 # unknown as of yet
else:
lonFieldIdx = 0
latFieldIdx = 1
if keyFields[0] == self.getLatField():
lonFieldIdx = 1
latFieldIdx = 0
min = [
df[keyFields[lonFieldIdx]].min(),
df[keyFields[latFieldIdx]].min()
]
max = [
df[keyFields[lonFieldIdx]].max(),
df[keyFields[latFieldIdx]].max()
]
self.options["mapBounds"] = json.dumps(
[min, max], default=defaultJSONEncoding)
valueFields = self.getValueFields()
# check if we have a preserveCols
preserveCols = self.options.get("preserveCols", None)
preserveCols = [
a for a in preserveCols.split(",")
if a not in keyFields and a not in valueFields
] if preserveCols is not None else []
valueFieldIdxs = []
allProps = valueFields + preserveCols
for j, valueField in enumerate(allProps):
valueFieldIdxs.append(df.columns.get_loc(valueField))
# Transform the data into GeoJSON for use in the Mapbox client API
features = []
for row in df.itertuples():
feature = {
'type': 'Feature',
'properties': {},
'geometry': {
'type': 'Point',
'coordinates': []
}
}
if geomType == 0:
feature['geometry']['coordinates'] = [
row[lonFieldIdx + 1], row[latFieldIdx + 1]
]
else:
geomIdx = df.columns.get_loc(keyFields[0]) + 1
feature['geometry'] = json.loads(row[geomIdx])
for idx, valueFieldIdx in enumerate(valueFieldIdxs):
feature['properties'][allProps[idx]] = row[valueFieldIdx + 1]
features.append(feature)
if len(features) > 0:
pygeojson = {'type': 'FeatureCollection', 'features': features}
self.options["mapData"] = json.dumps(pygeojson,
default=defaultJSONEncoding)
# Now let's figure out whether we have Line or Polygon data, if it wasn't already found to be Point
if geomType != 1:
if features[0]['geometry']['type'].endswith('LineString'):
geomType = 1
elif features[0]['geometry']['type'].endswith('Polygon'):
geomType = 2
else:
geomType = -1
#### build up the map style
# basic color
paint = {}
if geomType == 1:
paint['line-color'] = '#ff0000'
paint['line-width'] = 2
if self.options.get("coloropacity"):
paint['line-opacity'] = float(
self.options.get("coloropacity")) / 100
elif geomType == 2:
paint['fill-color'] = '#ff0000'
paint['fill-opacity'] = 0.8
if self.options.get("coloropacity"):
paint['fill-opacity'] = float(
self.options.get("coloropacity")) / 100
else:
paint['circle-radius'] = 12
paint['circle-color'] = '#ff0000'
paint['circle-opacity'] = 0.25
if self.options.get("coloropacity"):
paint['circle-opacity'] = float(
self.options.get("coloropacity")) / 100
if (self.options.get("kind")
and self.options.get("kind").find("cluster") >= 0):
paint['circle-opacity'] = 1.0
if len(valueFields) > 0:
mapValueField = valueFields[0]
self.options["mapValueField"] = mapValueField
if not self.options.get("kind"):
self.options["kind"] = "choropleth-cluster"
# if there's a numeric value field and type is not 'simple', paint the data as a choropleth map
if self.options.get("kind") and self.options.get("kind").find(
"simple") < 0 and len(valueFields) > 0:
# color options
bincolors = []
bincolors.append(
['#ffffcc', '#a1dab4', '#41b6c4', '#2c7fb8',
'#253494']) # yellow to blue
bincolors.append(
['#fee5d9', '#fcae91', '#fb6a4a', '#de2d26',
'#a50f15']) # reds
bincolors.append(
['#f7f7f7', '#cccccc', '#969696', '#636363',
'#252525']) # grayscale
bincolors.append(
['#e66101', '#fdb863', '#f7f7f7', '#b2abd2',
'#5e3c99']) # orange to purple (diverging values)
bincolorsIdx = 0
if self.options.get("colorrampname"):
if self.options.get(
"colorrampname") == "Light to Dark Red":
bincolorsIdx = 1
if self.options.get("colorrampname") == "Grayscale":
bincolorsIdx = 2
if self.options.get("colorrampname") == "Orange to Purple":
bincolorsIdx = 3
minval = df[valueFields[0]].min()
maxval = df[valueFields[0]].max()
bins.append((minval, bincolors[bincolorsIdx][0]))
bins.append((df[valueFields[0]].quantile(0.25),
bincolors[bincolorsIdx][1]))
bins.append((df[valueFields[0]].quantile(0.5),
bincolors[bincolorsIdx][2]))
bins.append((df[valueFields[0]].quantile(0.75),
bincolors[bincolorsIdx][3]))
bins.append((maxval, bincolors[bincolorsIdx][4]))
if geomType == 1:
# paint['line-opacity'] = 0.65
paint['line-color'] = {"property": mapValueField}
paint['line-color']['stops'] = []
for bin in bins:
paint['line-color']['stops'].append([bin[0], bin[1]])
elif geomType == 2:
paint['fill-color'] = {"property": mapValueField}
paint['fill-color']['stops'] = []
for bin in bins:
paint['fill-color']['stops'].append([bin[0], bin[1]])
else:
# paint['circle-opacity'] = 0.65
paint['circle-color'] = {"property": mapValueField}
paint['circle-color']['stops'] = []
for bin in bins:
paint['circle-color']['stops'].append([bin[0], bin[1]])
paint['circle-radius'] = 12
self.options["mapStyle"] = json.dumps(paint,
default=defaultJSONEncoding)
w = self.getPreferredOutputWidth()
h = self.getPreferredOutputHeight()
# handle custom layers
userlayers = []
l = (ShellAccess, ShellAccess)
papp = self.options.get("nostore_pixieapp")
if papp is not None and ShellAccess[papp] is not None:
l = (ShellAccess[papp], dir(ShellAccess[papp]))
for key in [
a for a in l[1]
if not callable(getattr(l[0], a)) and not a.startswith("_")
]:
v = getattr(l[0], key)
if isinstance(v, dict) and "maptype" in v and v["maptype"].lower(
) == "mapbox" and "source" in v and "type" in v["source"] and v[
"source"][
"type"] == "geojson" and "id" in v and "data" in v[
"source"]:
gj = geojson.loads(json.dumps(v["source"]["data"]))
isvalid = geojson.is_valid(gj)
if isvalid["valid"] == "yes":
userlayers.append(v)
# self.debug("GOT VALID GEOJSON!!!!")
else:
self.debug("Invalid GeoJSON: {0}".format(
str(v["source"]["data"])))
self.debug("userlayers length: " + str(len(userlayers)))
# end handle custom layers
uniqueid = str(uuid.uuid4())[:8]
return self.renderTemplate("mapView.html",
bins=bins,
userlayers=userlayers,
prefwidth=w,
prefheight=h,
randomid=uniqueid)
def is_geojson_valid(geojson_file):
validation = geojson.is_valid(geojson_file)
return validation['valid']
def is_geometry_valid(geometry):
geometry_str = ujson.dumps(geometry)
valid = geojson.is_valid(geojson.loads(geometry_str))
return 'valid' in valid and (valid['valid'] == 'yes' or valid['valid'] == '')
scenario_df = pd.read_csv(os.path.join(scenario_path, 'scenario.csv'), sep=';')
components_df = pd.read_csv(os.path.join(scenario_path, 'components.csv'),
sep=';')
timeseries_df = pd.read_csv(os.path.join(scenario_path, 'timeseries.csv'),
sep=';')
hubs_df = pd.read_csv(os.path.join(scenario_path, 'hubs.csv'), sep=';')
features = create_features(components_df)
features.extend(create_features(hubs_df))
feature_collection = gj.FeatureCollection(features)
scenario_dict = dict(scenario_df.iloc[0])
# json package does not serialize numpy.int64 therefore
for key, value in scenario_dict.copy().items():
if isinstance(value, numpy.int64):
scenario_dict[key] = int(scenario_dict[key])
# probably better to use https://pypi.python.org/pypi/geojson/#custom-classes
feature_collection['scenario'] = scenario_dict
validation = gj.is_valid(feature_collection)
print("Feature collection is valid: " + validation['valid'])
if validation['valid'] != 'yes':
print(validation['message'])
raise Exception()
with open(output_filename, 'w') as output_file:
gj.dump(feature_collection, output_file, sort_keys=True)
print("File saved in " + os.path.join(os.getcwd(), output_filename))
def spatial_to_meta(self, extra, data_dict, metadata_dict):
"""
Helper to get GeoJSON from extras->spatial into a metadata_dict for the given
extra item
"""
# check for valid GeoJSON to prevent ckan
# from rejecting the whole dataset
try:
# fix invalid Polygon-Features
# (because - yes - people have problems reading the spec)
fixed_spatial_source = extra['value'].replace(
'polygon',
'Polygon'
)
spatial = geojson.loads(fixed_spatial_source)
spatial_validation = geojson.is_valid(spatial)
if spatial_validation['valid'] == 'yes':
# additional check: does the interior share more
# than 1 point with exterior? --> invalid
if len(spatial.coordinates) > 1:
# check all internal polygons
for internal_polygon in spatial.coordinates[1:]:
shared_coordinates_counter = 0
# iterate external coordinates,
# see if >1 matches internal polygon
for coord_external in spatial.coordinates[0]:
if coord_external in internal_polygon:
shared_coordinates_counter += 1
if shared_coordinates_counter > 1:
# skip spatial coordinates
raise ValueError('More than one shared coordinate!')
# extract string to JSON
metadata_dict['boundingbox'] = json.loads(
fixed_spatial_source
)
# calculate area covered by the the shape
spatial_area = self.calculate_geojson_area(spatial)
# area must at least be >0. We are using 1/X to rank the results
if spatial_area < 0:
spatial_area = 1
metadata_dict['spatial_area'] = spatial_area
# calculate center of the shape
spatial_center_x, spatial_center_y = self.calculate_geojson_center(
spatial
)
metadata_dict['spatial_center'] = {
"lat": spatial_center_y,
"lon": spatial_center_x
}
else:
raise ValueError(spatial_validation['message'])
except Exception as ex:
info_message = "invalid GeoJSON in extras->spatial "
info_message += "at dataset: " + data_dict['name']
info_message += ", Exception: "
info_message += type(ex).__name__
info_message += ", "
info_message += str(ex.args)
logger.info(info_message)
# Need to follow the right hand rule UR, UL, LL, LR, UR
thisBox = [[mapGrid[R][C+1],
mapGrid[R][C],
mapGrid[R + 1][C],
mapGrid[R + 1][C + 1],
mapGrid[R][C + 1]]]
mapDictionary[repr(rowValues[R])+repr(colValues[C])] = thisBox
print(len(mapDictionary))
sorted(mapDictionary.keys())
geoJsonFeatures = []
for k, v in mapDictionary.items():
if str(k + ".dwg") in FD.printList:
thisPolygon = geojson.Polygon(v)
thisFeature = geojson.Feature(geometry=thisPolygon, id=k, properties={"id": k, "link": str(directory + "\\" + k + ".dwg")})
geoJsonFeatures.append(thisFeature)
geoJsonComplete = geojson.FeatureCollection(geoJsonFeatures)
validation = geojson.is_valid(geoJsonComplete)
print(validation['valid'])
print(validation['message'])
print("Writing File")
f = open("GeoJsonOutput", "w")
geojson.dump(geoJsonComplete, f)
f.close()
)
@app.route("/encode", methods=["POST"])
def geojson_encode():
try:
g = request.form["geojson"]
f = geojson.loads(g)
except Exception, e:
error = "failed to load geojson, because %s" % e
logging.error(error)
return jsonify(ok=0, error=error)
# Does the input pass the smell check?
validation = geojson.is_valid(f)
if validation["valid"] == "no":
error = "GeoJSON doesn't smell right: %s" % validation["message"]
logging.error(error)
return jsonify(ok=0, error=error)
e = mapzen.whosonfirst.geojson.encoder(precision=None)
try:
fh = StringIO.StringIO()
e.encode_feature(f, fh)
except Exception, e:
error = "failed to encode geojson, because %s" % e
logging.error(error)
return jsonify(ok=0, error=error)
feat = Feature(geometry=Point([xseq[xi], yseq[yi]]), properties={"xyid": str(xi) + " " + str(yi)})
else:
coordlist = rectpolyctl(xseq[xi], xseq[xi + 1], yseq[yi], yseq[yi + 1])
feat = Feature(
geometry=Polygon(
[
coordlist
# [
# (xseq[xi], yseq[yi]),
# (xseq[xi], yseq[yi+1]),
# (xseq[xi+1], yseq[yi+1]),
# (xseq[xi+1], yseq[yi]),
# (xseq[xi], yseq[yi])
# ]
]
),
properties={"xyid": str(xi) + " " + str(yi)},
)
farr.append(feat)
if args.crsname is not None:
fc = FeatureCollection(farr, crs={"type": "name", "properties": {"name": args.crsname}})
else:
fc = FeatureCollection(farr)
v = is_valid(fc)
if v["valid"] != "yes":
sys.stdout.write(str(v))
print(fc)
def upload_individual_docs_to_ES_cURL(docs,
index,
doc_type,
id_field=False,
geopoint=False,
geoshape=False,
delete_index=False):
#input: list of JSON documents, an index name, document type, ID field, and name of an (OPTIONAL) geopoint field.
#uploads each feature element to ElasticSearch using subprocess and cURL
#es = Elasticsearch(ES_url)
es = 'http://%s:%s@%s:9200' % (ES_username, ES_password, ES_url)
if delete_index:
try:
p = subprocess.Popen(['curl', '-XDELETE', '%s/%s' % (es, index)])
#subprocess.call('curl -XDELETE %s/%s' % (es,index))
out, err = p.communicate()
if err: print '\n' + err
except Exception as e:
print "Error deleting index:"
print e
print "\n\nPast Delete code\n\n"
try:
#create the index, set the replicas so uploads will not err out
settings = {"settings": {"number_of_replicas": 1}}
p = subprocess.Popen([
'curl', '-XPUT',
'%s/%s' % (es, index), '-d',
json.dumps(settings)
])
out, err = p.communicate()
if err: print '\n' + err
print "\n\nPast Create code\n\n"
#build the mapping document
mapping = {}
mapping['properties'] = {}
#if the data has a location field, set the geo_point mapping
if geopoint:
mapping['properties'][geopoint] = {
'type': 'geo_point',
'store': 'yes'
}
if geoshape:
mapping['properties'][geoshape] = {
'type': 'geo_shape',
'tree': 'quadtree',
'precision': '1m'
}
#use cURL to put the mapping
p = subprocess.Popen([
'curl',
'%s/%s/_mapping/%s' % (es, index, doc_type), '-d',
'%s' % json.dumps(mapping)
],
stderr=subprocess.PIPE)
out, err = p.communicate()
if err: print '\n' + err
print "\n\nPast Mapping code"
except Exception as e:
#do not try to recreate the index
print "Error creating index:"
print e
#double-check that the correct number of replicas are being used
p = subprocess.Popen([
'curl', '-XPUT',
'%s/%s/_settings' % (es, index), '-d', '{"number_of_replicas": 1}'
])
out, err = p.communicate()
if err: print '\n' + err
#iterate through and upload individual documents
succeeded = 0
failed = 0
idx = 0
for doc in docs:
idx += 1
#check if the document is a geojson document
validation = geojson.is_valid(doc)
if validation['valid'].lower() == 'yes':
#add the point/shape to the document properties
if geopoint:
doc['properties'][geopoint] = list(
geojson.utils.coords(doc))[0]
if geoshape:
doc['properties'][geoshape] = doc['geometry']
doc['geometry']['type'] = doc['geometry']['type'].lower(
) #convert type to lowercase for leaflet
#load the document properties into ES
to_upload = deepcopy(doc['properties'])
else:
to_upload = deepcopy(doc)
if id_field:
_id = to_upload[id_field]
else:
_id = idx
#up_doc = json.dumps(to_upload)
#print up_doc
#upload the document
p = subprocess.Popen([
'curl', '-XPUT',
'%s/%s/%s/%s' % (
es,
index,
doc_type,
_id,
), '-d',
'%s' % json.dumps(to_upload)
],
stderr=subprocess.PIPE)
out, err = p.communicate()
if not err:
succeeded += 1
else:
failed += 1
print "Finished uploading documents. %s succeeded. %s failed." % (
succeeded, failed)
def bulk_upload_docs_to_ES_cURL(docs, **kwargs):
#input: list of JSON documents, an index name, document type, ID field, and name of an (OPTIONAL) geopoint field.
#uploads each feature element to ElasticSearch using subprocess and cURL
#es = Elasticsearch(ES_url)
#define keyword inputs
index = kwargs.get('index', 'index_tmp')
doc_type = kwargs.get('doc_type', 'doc_tmp')
id_field = kwargs.get('id_field', False)
geopoint = kwargs.get('geopoint', False)
geoshape = kwargs.get('geoshape', False)
delete_index = kwargs.get('delete_index', False)
es = 'http://%s:%s@%s:9200' % (ES_username, ES_password, ES_url)
if delete_index:
try:
p = subprocess.Popen(['curl', '-XDELETE', '%s/%s' % (es, index)])
#subprocess.call('curl -XDELETE %s/%s' % (es,index))
out, err = p.communicate()
if err: print '\n' + err + '\n\n'
except Exception as e:
print "Error deleting index:"
print e
try:
#create the index, set the replicas so uploads will not err out
settings = {"settings": {"number_of_replicas": 1}}
p = subprocess.Popen([
'curl', '-XPUT',
'%s/%s' % (es, index), '-d',
json.dumps(settings)
])
out, err = p.communicate()
if err: print '\n' + err + '\n\n'
#build the mapping document
mapping = {}
mapping['properties'] = {}
#if the data has a location field, set the geo_point mapping
if geopoint:
mapping['properties'][geopoint] = {
'type': 'geo_point',
'store': 'yes'
}
if geoshape:
mapping['properties'][geoshape] = {
'type': 'geo_shape',
'tree': 'quadtree',
'precision': '1m'
}
#use cURL to put the mapping
p = subprocess.Popen([
'curl',
'%s/%s/_mapping/%s' % (es, index, doc_type), '-d',
'%s' % json.dumps(mapping)
],
stderr=subprocess.PIPE)
out, err = p.communicate()
if err: print '\n' + err + '\n\n'
except Exception as e:
#do not try to recreate the index
print "Error creating index:"
print e
#double-check that the correct number of replicas are being used
p = subprocess.Popen([
'curl', '-XPUT',
'%s/%s/_settings' % (es, index), '-d', '{"number_of_replicas": 1}'
])
out, err = p.communicate()
if err: print '\n' + err + '\n\n'
#iterate through and upload individual documents
actions = []
idx = 0
bulk = 0
for doc in docs:
bulk += 1
idx += 1
#check if the document is a geojson document
validation = geojson.is_valid(doc)
if validation['valid'].lower() == 'yes':
#add the point/shape to the document properties
if geopoint:
doc['properties'][geopoint] = list(
geojson.utils.coords(doc))[0]
if geoshape:
doc['properties'][geoshape] = doc['geometry']
doc['geometry']['type'] = doc['geometry']['type'].lower(
) #convert type to lowercase for leaflet
#load the document properties into ES
to_upload = deepcopy(doc['properties'])
else:
to_upload = deepcopy(doc)
if id_field:
_id = to_upload[id_field]
else:
_id = idx
actions.append(
'{ "create" : {"_id" : "%s", "_type" : "%s", "_index" : "%s"} }\n'
% (_id, doc_type, index))
actions.append('%s\n' % json.dumps(to_upload))
#upload 10k records at a time
if bulk >= 10000:
with open('bulk.txt', 'w') as bulk_file:
bulk_file.writelines(
actions
) #write the actions to a file to be read by the bulk cURL command
#upload the remaining records
p = subprocess.Popen([
'curl', '-XPOST',
'%s/_bulk' % es, '--data-binary', '@bulk.txt'
],
stderr=subprocess.PIPE)
#p = subprocess.Popen(['curl','-XPOST','%s/_bulk' % es,'-d','%s' % ('').join(actions)],stderr=subprocess.PIPE)
out, err = p.communicate()
if err:
print err + '\n\n'
bulk = 0
actions = []
#upload the remaining records
with open('bulk.txt', 'w') as bulk_file:
bulk_file.writelines(
actions
) #write the actions to a file to be read by the bulk cURL command
p = subprocess.Popen(
['curl', '-XPOST',
'%s/_bulk' % es, '--data-binary', '@bulk.txt'],
stderr=subprocess.PIPE)
#p = subprocess.Popen(['curl','-XPOST','%s/_bulk' % es,'-d','%s' % ('').join(actions)],stderr=subprocess.PIPE)
out, err = p.communicate()
if err:
print err
os.remove('bulk.txt')
def clean(self):
cleaned_data = super(ItemForm, self).clean()
not_false_values = [
bool(unicode(v).strip()) for v in cleaned_data.values()
]
if DELETION_FIELD_NAME in cleaned_data:
self.delete = cleaned_data.pop(DELETION_FIELD_NAME)
if ITEM_FIELD_NAME in cleaned_data:
self.item_id = cleaned_data.pop(ITEM_FIELD_NAME)
if (len(cleaned_data) <= 0 or not any(not_false_values)):
msg = _("At least one field must be filled")
for field_key, field_value in cleaned_data.items():
if not field_value.strip():
self._errors[field_key] = self.error_class([msg])
del cleaned_data[field_key]
# Extra check for choices fields
choices_properties = self.itemtype.properties.filter(datatype="c")
for choice_property in choices_properties:
choice_dict = dict(choice_property.get_choices())
key = choice_property.key
if key in cleaned_data:
value = choice_dict[cleaned_data[key]].strip()
if value == NULL_OPTION:
# cleaned_data[key] = None
cleaned_data.pop(key)
else:
cleaned_data[key] = value
elif choice_property.required:
msg = _("This field is required and "
"must have some value selected.")
self._errors[key] = self.error_class([msg])
else:
cleaned_data[key] = u""
# Extra check for spatial fields
spatial_properties = self.itemtype.properties.filter(
datatype__in=["p", "l", "m"])
for spatial_property in spatial_properties:
key = spatial_property.key
if key in cleaned_data and cleaned_data[key]:
try:
field = geojson.loads(cleaned_data[key])
except ValueError:
msg = _("This field is required to "
"be in a valid JSON format.")
self._errors[key] = self.error_class([msg])
else:
validity = geojson.is_valid(field)
is_not_valid = validity['valid'] == 'no'
if is_not_valid and spatial_property.datatype == u'p':
msg = _("This field is required to "
"be a valid GeoJSON point.")
elif is_not_valid and spatial_property.datatype == u'l':
msg = _("This field is required to "
"be a valid GeoJSON path.")
elif is_not_valid and spatial_property.datatype == u'm':
msg = _("This field is required to "
"be a valid GeoJSON area.")
elif is_not_valid:
msg = _("This field is required to "
"be a valid GeoJSON.")
if is_not_valid:
self._errors[key] = self.error_class([msg])
else:
cleaned_data[key] = u""
return cleaned_data
def keyholemarkup2x(file,output='df'):
"""
Takes Keyhole Markup Language Zipped (KMZ) or KML file as input. The
output is a pandas dataframe, geopandas geodataframe, csv, geojson, or
shapefile.
All core functionality from:
http://programmingadvent.blogspot.com/2013/06/kmzkml-file-parsing-with-python.html
Parameters
----------
file : {string}
The string path to your KMZ or .
output : {string}
Defines the type of output. Valid selections include:
- shapefile - 'shp', 'shapefile', or 'ESRI Shapefile'
Returns
-------
self : object
"""
r = re.compile(r'(?<=\.)km+[lz]?',re.I)
try:
extension = r.search(file).group(0) #(re.findall(r'(?<=\.)[\w]+',file))[-1]
except IOError as e:
logging.error("I/O error {0}".format(e))
if (extension.lower()=='kml') is True:
buffer = file
elif (extension.lower()=='kmz') is True:
kmz = ZipFile(file, 'r')
vmatch = np.vectorize(lambda x:bool(r.search(x)))
A = np.array(kmz.namelist())
sel = vmatch(A)
buffer = kmz.open(A[sel][0],'r')
else:
raise ValueError('Incorrect file format entered. Please provide the '
'path to a valid KML or KMZ file.')
parser = xml.sax.make_parser()
handler = PlacemarkHandler()
parser.setContentHandler(handler)
parser.parse(buffer)
try:
kmz.close()
except:
pass
df = pd.DataFrame(handler.mapping).T
names = list(map(lambda x: x.lower(),df.columns))
if 'description' in names:
extradata = df.apply(PlacemarkHandler.htmlizer,axis=1)
df = df.join(extradata)
output = output.lower()
if output=='df' or output=='dataframe' or output == None:
result = df
elif output=='csv':
out_filename = file[:-3] + "csv"
df.to_csv(out_filename,encoding='utf-8',sep="\t")
result = ("Successfully converted {0} to CSV and output to"
" disk at {1}".format(file,out_filename))
elif output=='gpd' or output == 'gdf' or output=='geoframe' or output == 'geodataframe':
try:
import shapely
from shapely.geometry import Polygon,LineString,Point
except ImportError as e:
raise ImportError('This operation requires shapely. {0}'.format(e))
try:
import fiona
except ImportError as e:
raise ImportError('This operation requires fiona. {0}'.format(e))
try:
import geopandas as gpd
except ImportError as e:
raise ImportError('This operation requires geopandas. {0}'.format(e))
geos = gpd.GeoDataFrame(df.apply(PlacemarkHandler.spatializer,axis=1))
result = gpd.GeoDataFrame(pd.concat([df,geos],axis=1))
elif output=='geojson' or output=='json':
try:
import shapely
from shapely.geometry import Polygon,LineString,Point
except ImportError as e:
raise ImportError('This operation requires shapely. {0}'.format(e))
try:
import fiona
except ImportError as e:
raise ImportError('This operation requires fiona. {0}'.format(e))
try:
import geopandas as gpd
except ImportError as e:
raise ImportError('This operation requires geopandas. {0}'.format(e))
try:
import geojson
except ImportError as e:
raise ImportError('This operation requires geojson. {0}'.format(e))
geos = gpd.GeoDataFrame(df.apply(PlacemarkHandler.spatializer,axis=1))
gdf = gpd.GeoDataFrame(pd.concat([df,geos],axis=1))
out_filename = file[:-3] + "geojson"
gdf.to_file(out_filename,driver='GeoJSON')
validation = geojson.is_valid(geojson.load(open(out_filename)))['valid']
if validation == 'yes':
result = ("Successfully converted {0} to GeoJSON and output to"
" disk at {1}".format(file,out_filename))
else:
raise ValueError('The geojson conversion did not create a '
'valid geojson object. Try to clean your '
'data or try another file.')
elif output=='shapefile' or output=='shp' or output =='esri shapefile':
try:
import shapely
from shapely.geometry import Polygon,LineString,Point
except ImportError as e:
raise ImportError('This operation requires shapely. {0}'.format(e))
try:
import fiona
except ImportError as e:
raise ImportError('This operation requires fiona. {0}'.format(e))
try:
import geopandas as gpd
except ImportError as e:
raise ImportError('This operation requires geopandas. {0}'.format(e))
try:
import shapefile
except ImportError as e:
raise ImportError('This operation requires pyshp. {0}'.format(e))
geos = gpd.GeoDataFrame(df.apply(PlacemarkHandler.spatializer,axis=1))
gdf = gpd.GeoDataFrame(pd.concat([df,geos],axis=1))
out_filename = file[:-3] + "shp"
print(gdf)
gdf.to_file(out_filename,driver='ESRI Shapefile')
#sf = shapefile.Reader(out_filename)
import shapefile
sf = shapefile.Reader(out_filename)
print(sf)
print('hello')
try:
print(sf.shapes())
len(sf.shapes())>0
except:
print('huh')
validation = "no"
else:
if len(sf.shapes())>0:
validation = "yes"
else:
validation = "no"
validation = "yes"
if validation == 'yes':
result = ("Successfully converted {0} to Shapefile and output to"
" disk at {1}".format(file,out_filename))
else:
raise ValueError('The Shapefile conversion did not create a '
'valid shapefile object. Try to clean your '
'data or try another file.')
else:
raise ValueError('The conversion returned no data; check if'
' you entered a correct output file type. '
'Valid output types are geojson, shapefile,'
' csv, geodataframe, and/or pandas dataframe.')
return result
def clean(self):
cleaned_data = super(ItemForm, self).clean()
not_false_values = [bool(unicode(v).strip())
for v in cleaned_data.values()]
if DELETION_FIELD_NAME in cleaned_data:
self.delete = cleaned_data.pop(DELETION_FIELD_NAME)
if ITEM_FIELD_NAME in cleaned_data:
self.item_id = cleaned_data.pop(ITEM_FIELD_NAME)
if (len(cleaned_data) <= 0 or
not any(not_false_values)):
msg = _("At least one field must be filled")
for field_key, field_value in cleaned_data.items():
if not field_value.strip():
self._errors[field_key] = self.error_class([msg])
del cleaned_data[field_key]
# Extra check for choices fields
choices_properties = self.itemtype.properties.filter(datatype="c")
for choice_property in choices_properties:
choice_dict = dict(choice_property.get_choices())
key = choice_property.key
if key in cleaned_data:
value = choice_dict[cleaned_data[key]].strip()
if value == NULL_OPTION:
# cleaned_data[key] = None
cleaned_data.pop(key)
else:
cleaned_data[key] = value
elif choice_property.required:
msg = _("This field is required and "
"must have some value selected.")
self._errors[key] = self.error_class([msg])
else:
cleaned_data[key] = u""
# Extra check for spatial fields
spatial_properties = self.itemtype.properties.filter(
datatype__in=["p", "l", "m"])
for spatial_property in spatial_properties:
key = spatial_property.key
if key in cleaned_data and cleaned_data[key]:
try:
field = geojson.loads(cleaned_data[key])
except ValueError:
msg = _("This field is required to "
"be in a valid JSON format.")
self._errors[key] = self.error_class([msg])
else:
validity = geojson.is_valid(field)
is_not_valid = validity['valid'] == 'no'
if is_not_valid and spatial_property.datatype == u'p':
msg = _("This field is required to "
"be a valid GeoJSON point.")
elif is_not_valid and spatial_property.datatype == u'l':
msg = _("This field is required to "
"be a valid GeoJSON path.")
elif is_not_valid and spatial_property.datatype == u'm':
msg = _("This field is required to "
"be a valid GeoJSON area.")
elif is_not_valid:
msg = _("This field is required to "
"be a valid GeoJSON.")
if is_not_valid:
self._errors[key] = self.error_class([msg])
else:
cleaned_data[key] = u""
return cleaned_data
