def get_bbox_for(insee):
"""
Returns left/right/bottom/top (min and max latitude / longitude) of the
bounding box around the INSEE code
"""
relation = get_municipality_relations(insee[:-3], insee, False)
city = pygeoj.new()
city.add_feature(geometry=get_geometry_for(relation[0]))
city.update_bbox()
return city.bbox
def dggs_enable_line(thisfile, myOutput_location_and_name, resolution):
''' takes a geojson file and adds AusPix DGGS cell information at the resolution called for
maintains any properties data for out but at the end
AusPix data in the first columns - then the properties data
'''
# calc cell area in m2
resArea = (rdggs.cell_area(resolution, plane=False))
# metadata
print('number of features', len(thisfile)) # the number of features
print('bbox of entire file', thisfile.bbox) # the bounding box region of the entire file
print('area of each cell at this resolution', resArea)
print('crs', thisfile.crs) # the coordinate reference system
print('attributes', thisfile.all_attributes) # retrieves the combined set of all feature attributes
print('common attributes',
thisfile.common_attributes) # retrieves only those field attributes that are common to all features
print()
# make an output file of DGGS centroid points with the at atttibute properties
newfile = pygeoj.new() # default projection is WGS84
#work through the features (polygons) one by one and ask for DGGS cells
for feature in thisfile: #each feature is an individual line
print('feature attributes ', feature.properties) # the feature attributes - want to keep for output
coords = feature.geometry.coordinates # xy's along line
print('in_coords= ', (coords))
densified_coords = densify_my_line(coords, resolution)
print('out coords = ', (densified_coords))
for item in densified_coords:
doneDGGScells = []
for coords in item: # for this road or sreat line look at all the points that describe it
#print('thispt', coords)
thisDGGS = rdggs.cell_from_point(resolution, coords, plane=False) # false = on the elipsoidal curve
if thisDGGS not in doneDGGScells: # == new one
doneDGGScells.append(thisDGGS) # save as a done cell
verts = thisDGGS.vertices(plane=False) # find the cell corners = vertices from the engine
#print('v', verts[0])
verts.append(verts[0]) #add the first point to the end to make a closed poly
#print('verts', verts)
my_prop = feature.properties
my_Cell = {"AusPIX_DGGS": str(thisDGGS), "CellArea_M2": resArea}
#include the AusPIX cell information in attributes
these_attributes = dict(list(my_Cell.items()) + list(my_prop.items()))
#print('these attributes = ', these_attributes)
newfile.add_feature(properties=these_attributes, geometry={"type": "Polygon", "coordinates": [verts]})
#save the ouput geojson file
newfile.save(myOutput_location_and_name + '.geojson') # saves into the folder where you have the script - edit to change
def merge_geojson(cls, folder, file_name, files):
geojson = pygeoj.new()
for path in files:
f = pygeoj.load(path)
for feature in f:
geojson.add_feature(obj=feature)
full_path = os.path.join(folder, file_name + ".json")
geojson.add_all_bboxes()
geojson.update_bbox()
geojson.save(full_path)
return full_path
def update_geojson_heatmap(self):
output = pygeoj.new()
for idx, dsp in self.df_disposals.iterrows():
if not np.isnan(dsp.rent):
dsp.fillna(0., inplace=True)
pnt = geojson.Point((dsp.lng, dsp.lat))
ppt = {
'size_max': dsp.size_max,
'size_min': dsp.size_min,
'rent': dsp.rent,
'rates': dsp.rates
}
print(ppt)
output.add_feature(properties=ppt, geometry=pnt)
output.add_unique_id()
output.save("web_data\point_heat.geojson")
def dggs_cells_for_points(geojson, resolution):
# make an output file of DGGS centroid points with the at atttibute properties
newfile = pygeoj.new() # default projection is WGS84
resArea = (rdggs.cell_area(resolution, plane=False))
#work through the features (polygons) one by one and ask for DGGS cells
for feature in testfile:
print('feature attributes ', feature.properties
) # the feature attributes - want to keep for output
coords = feature.geometry.coordinates # xy
print('geom', coords)
#this_dggs_cell = rdggs.cell_from_point(resolution, coords, plane=False) # false = on the elipsoidal curve
this_dggs_cell = latlong_to_DGGS(coords, resolution)
print('found cell = ', this_dggs_cell)
my_prop = feature.properties
my_Cell = {
"AusPIX_DGGS": str(this_dggs_cell),
"LongiWGS84": coords[0],
"LatiWGS84": coords[1],
"CellArea_M2": resArea
}
#include the AusPIX cell information in attributes
these_attributes = dict(list(my_Cell.items()) + list(my_prop.items()))
#print('these attributes = ', these_attributes)
newfile.add_feature(properties=these_attributes,
geometry={
"type": "Point",
"coordinates": coords
})
#save the ouput geojson file
newfile.save(
"test_points.geojson"
) # saves into the folder where you have the script - edit to change
def write_geojson_file(self, folder, file_name, shp_geometry, shp_field):
if (len(shp_field) == 0):
values = [str(uuid.uuid4()) for i in range(len(shp_geometry))]
shp_field = [Field(values, 'UUID', 'C', 40, 0)]
full_path = os.path.join(folder, file_name + ".json")
geojson = pygeoj.new()
if (self.field_legth_checking(shp_field, shp_geometry)
and self.create_valid_folder(folder)):
for i, geometry in enumerate(shp_geometry):
if geometry.name == "point":
self.__write_point(i, geojson, geometry, shp_field)
elif geometry.name == "multipoint":
self.__write_multi_point(i, geojson, geometry, shp_field)
elif geometry.name == "curve":
self.__write_linestring(i, geojson, geometry, shp_field)
elif geometry.name == "multicurve":
self.__write_multi_linestring(i, geojson, geometry,
shp_field)
elif geometry.name == "polygon":
self.__write_polygon(i, geojson, geometry, shp_field)
elif geometry.name == "multipolygon":
self.__write_multi_polygon(i, geojson, geometry, shp_field)
geojson.add_all_bboxes()
geojson.update_bbox()
geojson.save(full_path)
return full_path
else:
return False
resArea = (rdggs.cell_area(resolution, plane=False))
# for item in cells_inbb:
# print(item)
#metadata
print('crs', testfile.crs) # the coordinate reference system
print('attributes', testfile.all_attributes
) # retrieves the combined set of all feature attributes
print(
'common attributes', testfile.common_attributes
) # retrieves only those field attributes that are common to all features
print()
# make an output file of DGGS centroid points with the at atttibute properties
newfile = pygeoj.new() # default projection is WGS84
#work through the features (polygons) one by one and ask for DGGS cells
for feature in testfile:
print('first xxx ', feature.properties
) # the feature attributes - want to keep for output
print()
#print('bbox', feature.geometry.bbox) # the bounding box of the feature
fea_bbox = feature.geometry.bbox
print(feature.geometry.coordinates)
this_poly_cells = cells_in_poly(
fea_bbox, feature.geometry.coordinates, resolution
) # returns the cells in the poly and lat long of centroid
print('num cells in this poly =', len(this_poly_cells))
import pygeoj as gj
testfile = gj.new()
testfile.add_feature(geometry=gj.Geometry(type="Point", coordinates=[(11,11)]),
properties=dict(hello=1, world=2))
testfile.add_feature(geometry=gj.Geometry(type="Point", coordinates=[(12,12)]),
properties=dict(hello=1, world=2))
for feat in testfile:
feat.properties["new"] = "stuff"
feat.geometry.coordinates = (99,99)
#feat.properties = dict(new="shit")
#feat.geometry = gj.Geometry(type="Point", coordinates=(7777,7777))
print("hmm", feat.geometry, feat.properties, feat.__geo_interface__)
for feat in testfile:
print(feat, feat.geometry, feat.properties, feat.__geo_interface__, feat.validate())
StateInitials = FirmLicenseSheet['B'][1:]
inmemorydata = []
listtuple = []
for state in FirmLicenseSheet.rows:
listtuple = []
for j in state:
listtuple.append(j.value)
inmemorydata.append(listtuple)
states_url = 'M:/Grow People/Licenses/ZZZ-Scripts for Map/states20m.json'
states = pygeoj.load(filepath=states_url)
attributes = states.all_attributes
outfile = pygeoj.new()
outfile.define_crs(type="link",
link="http://spatialreference.org/ref/epsg/4326/esriwkt/",
link_type="esriwkt")
for state in states:
geometry = state.geometry
for item in state.properties.iteritems():
if item[0] == u'NAME':
StateName = item[1]
if item[0] == u'STUSPS':
StateInitials = item[1]
for FirmLicenseState in inmemorydata:
if FirmLicenseState[1] == StateInitials:
try:
to = strftime('%m/%d/%Y', FirmLicenseState[4])
def main(argv):
fin = sys.argv[1] #the path of Weather/Forcing folder
output = sys.argv[2] # the path of output folder
# print file
# print output
#create directory C:\\Users\\George\\Desktop\\Weather
with open(fin + "\\bySite\X3045Y525.csv") as f:
reader = csv.reader(f)
data = [r for r in reader]
# print len(data)
# print len(data[0][0].split())
for i in range(1, len(data)):
year = output + "\\" + data[i][0].split()[0][0:4]
if not os.path.exists(year):
os.makedirs(year)
month = year + "\\" + data[i][0].split()[0][4:6]
if not os.path.exists(month):
os.makedirs(month)
day = month + "\\" + data[i][0].split()[0][6:8]
if not os.path.exists(day):
os.makedirs(day)
with open(fin + "\\bySite\X3045Y525.csv") as f:
reader = csv.reader(f)
data = [r for r in reader]
time = []
vars = data[0][0].split()
for i in range(len(data)):
time.append(data[i][0].split()[0])
# print time
# print vars
with open(fin + "\\FLDAS_grids.csv") as f:
read = csv.reader(f)
polygon = [r for r in read]
for m in range(1, 6208):
for n in range(1, 26):
newfile = gj.new()
newfile.define_crs(type="name",
name="urn:ogc:def:crs:EPSG::3857")
for i in range(1, len(polygon)):
CSV = "X" + polygon[i][6][:-3] + polygon[i][6][
-2:] + "Y" + polygon[i][5][:-3] + polygon[i][5][
-2:] + ".csv"
coor = [[(float(polygon[i][1]), float(polygon[i][3])),
(float(polygon[i][2]), float(polygon[i][3])),
(float(polygon[i][2]), float(polygon[i][4])),
(float(polygon[i][1]), float(polygon[i][4])),
(float(polygon[i][1]), float(polygon[i][3]))]]
filename = fin + "\\bySite\\" + CSV
with open(filename) as file:
data = csv.reader(file)
values = [r for r in data]
newfile.add_feature(properties={
"value":
float(values[m][0].split()[n])
},
geometry={
"type": "Polygon",
"coordinates": coor
})
#C:\\Users\\George\\Desktop\\output
geopath = output + "\\" + time[m][0:4] + "\\" + time[m][
4:6] + "\\" + time[m][6:8] + "\\" + vars[n] + ".geojson"
newfile.save(geopath)
import certifi
import json
from rdflib.namespace import Namespace
from rdflib import Graph, BNode, RDF, RDFS, URIRef, Literal, XSD
import pygeoj
from geojson import Feature, Point
import coloredlogs, logging
geoj = pygeoj.new()
NAMESPACES = {
'schema': Namespace('http://schema.org/'),
'dcterms': Namespace('http://purl.org/dc/terms/')
}
g = Graph()
g.parse("./data/poit.rdf")
qres = g.query("""
PREFIX sdo: <http://schema.org/>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
PREFIX dc: <http://purl.org/dc/terms/>
SELECT DISTINCT ?newsitemlabel ?label ?lon ?lat ?newsitemdate ?url
WHERE {
?item rdfs:label ?label .
?item sdo:geo ?geo .
?geo sdo:latitude ?lat .
?geo sdo:longitude ?lon .
?newsitem sdo:about ?item .
except FileNotFoundError:
print('>> ERROR: file not found or incorrect file format')
quit()
# parse tau parameter and set default value if necessary
tau_default = 0.05
try:
tau = float(sys.argv[2])
except IndexError:
tau = tau_default
except ValueError:
tau = tau_default
file_output = file_name[:-8] + '_convex' + file_name[-8:]
# initialize new geojson file
new_file = pygeoj.new()
print('>> running ACD on: %s' % file_name)
print('>> output saved to: %s' % file_output)
print('>> concavity tolerance: %s' % str(tau))
# iterate through the features of the file and apply the algorithm
for feature in data:
# iterate through the polygons of each feature
for polygon in feature.geometry.coordinates:
# extract a reference length for this polygon
bounding_box = feature.geometry.bbox
length = min(abs(bounding_box[2] - bounding_box[0]),
abs(bounding_box[3] - bounding_box[1]))
help='The output path for the json catalog file',
default=None)
parser.add_argument('--dest_geojson', dest="geojson_filename",
help='The output path for the geojson file',
default=None)
parser.add_argument('--tiles_size', dest='tiles_size', type=int, default=256,
help=('Specify the tiles size in pixels '
'(assumes both x and y are the same)'))
args = parser.parse_args()
######################## Creating the geojson file
geojson = pygeoj.new()
geojson.define_crs(type="name", name="urn:ogc:def:crs:OGC:1.3:CRS84")
g = globalmaptiles.GlobalMercator()
for dirname, subdirList, filename_list in os.walk(args.tiles_path):
for filename in filename_list:
if filename.endswith('.png'):
rel_path = '.' + os.sep + os.path.join(dirname, filename)
tms_z, tms_x, tms_y = rel_path.rstrip('.png').split(os.sep)[-3:]
tms_z, tms_x, tms_y = map(int, (tms_z, tms_x, tms_y))
minLat, minLon, maxLat, maxLon = g.TileLatLonBounds(tms_x, tms_y, tms_z)
# minLat, minLon = lower left corner
# maxLat, maxLon = upper right corner
tile_upper_left_corner = minLon, maxLat
tile_upper_right_corner = maxLon, maxLat
from copy import deepcopy
print("Please enter the name of input file: ", end="")
inp = input()
levels = []
data = pygeoj.load(inp)
for i in data:
if 'level' in i.properties:
a = i.properties['level'].split(";")
for j in a:
if j not in levels:
levels.append(j)
levels.sort()
print("levels are:", levels)
for l in levels:
export = pygeoj.new()
for i in data:
if 'level' in i.properties:
if l in i.properties["level"].split(";"):
export.add_feature(deepcopy(i))
for j in export:
j.properties["level"] = l
export.save(inp.split(".")[0] + "_level_" + l + ".geojson")
print("exported level:", l)
print("levels:", levels, "were saved to separate files named",
inp.split(".")[0] + "_level_x")
export = pygeoj.new()
for i in data:
def to_file(fields, rows, geometries, filepath, encoding="utf8"):
def encode(value):
if isinstance(value, (float,int)):
# nrs are kept as nrs
return value
elif isinstance(value, unicode):
# unicode is custom encoded into bytestring
return value.encode(encoding)
else:
# brute force anything else to string representation
return bytes(value)
# shapefile
if filepath.endswith(".shp"):
shapewriter = pyshp.Writer()
# set fields with correct fieldtype
for fieldindex,fieldname in enumerate(fields):
for row in rows:
value = row[fieldindex]
if value != "":
try:
# make nr fieldtype if content can be made into nr
float(value)
fieldtype = "N"
fieldlen = 16
decimals = 8
except:
# but turn to text if any of the cells cannot be made to float bc they are txt
fieldtype = "C"
fieldlen = 250
decimals = 0
break
else:
# empty value, so just keep assuming nr type
fieldtype = "N"
fieldlen = 16
decimals = 8
# clean fieldname
fieldname = fieldname.replace(" ","_")[:10]
# write field
shapewriter.field(fieldname.encode(encoding), fieldtype,fieldlen, decimals)
# convert geojson to shape
def geoj2shape(geoj):
# create empty pyshp shape
shape = pyshp._Shape()
# set shapetype
geojtype = geoj["type"]
if geojtype == "Null":
pyshptype = pyshp.NULL
elif geojtype == "Point":
pyshptype = pyshp.POINT
elif geojtype == "LineString":
pyshptype = pyshp.POLYLINE
elif geojtype == "Polygon":
pyshptype = pyshp.POLYGON
elif geojtype == "MultiPoint":
pyshptype = pyshp.MULTIPOINT
elif geojtype == "MultiLineString":
pyshptype = pyshp.POLYLINE
elif geojtype == "MultiPolygon":
pyshptype = pyshp.POLYGON
shape.shapeType = pyshptype
# set points and parts
if geojtype == "Point":
shape.points = [ geoj["coordinates"] ]
shape.parts = [0]
elif geojtype in ("MultiPoint","LineString"):
shape.points = geoj["coordinates"]
shape.parts = [0]
elif geojtype in ("Polygon"):
points = []
parts = []
index = 0
for ext_or_hole in geoj["coordinates"]:
points.extend(ext_or_hole)
parts.append(index)
index += len(ext_or_hole)
shape.points = points
shape.parts = parts
elif geojtype in ("MultiLineString"):
points = []
parts = []
index = 0
for linestring in geoj["coordinates"]:
points.extend(linestring)
parts.append(index)
index += len(linestring)
shape.points = points
shape.parts = parts
elif geojtype in ("MultiPolygon"):
points = []
parts = []
index = 0
for polygon in geoj["coordinates"]:
for ext_or_hole in polygon:
points.extend(ext_or_hole)
parts.append(index)
index += len(ext_or_hole)
shape.points = points
shape.parts = parts
return shape
# iterate through original shapes
for row,geom in itertools.izip(rows, geometries):
shape = geoj2shape(geom)
shapewriter._shapes.append(shape)
shapewriter.record(*[encode(value) for value in row])
# save
shapewriter.save(filepath)
#Geojson File
elif filepath.endswith((".geojson",".json")):
geojwriter = pygeoj.new()
for row,geom in itertools.izip(rows,geometries):
# encode row values
row = (encode(value) for value in row)
rowdict = dict(zip(fields, row))
# add feature
geojwriter.add_feature(properties=rowdict,geometry=geom)
# save
geojwriter.save(filepath)
else:
raise Exception("Could not save the vector data to the given filepath: the filetype extension is either missing or not supported")
def to_file(fields, rows, geometries, filepath, encoding='utf-8'):
def encode(value):
if isinstance(value, (float, int)):
# Keep numbers
return value
elif isinstance(value, unicode):
# Encode unicode
return value.encode(encoding)
else:
# Brute force the rest
return bytes(value)
if filepath.lower().endswith('.shp'):
# pyshp does not read geojson 'geometries' directly, so create an
# empty pyshp._Shape() and load it with point and part data
shapewriter = pyshp.Writer()
# Set fields with correct field type
# Sweep each column and try to coerce to number - fall back to text
for fieldindex, fieldname in enumerate(fields):
for row in rows:
value = row[fieldindex]
if value != "":
try:
# Try to parse as a number EAFTP
float(value)
fieldtype = 'N' # Number
fieldlen = 16
decimals = 8
except:
fieldtype = 'C' # characters / text
fieldlen = 250
decimals = 0
# Empty - assume number
else:
fieldtype = 'N'
fieldlen = 16
decimals = 8
# Clean up the field names for shapefile format (no spaces, <=10 chrs)
fieldname = fieldname.replace(' ', '_')[:10]
# Write field (fieldname, type, len, decimals)
shapewriter.field(fieldname.encode(encoding), fieldtype, fieldlen,
decimals)
geoj2shape_format = {
'Null': pyshp.NULL,
'Point': pyshp.POINT,
'LineString': pyshp.POLYLINE,
'Polygon': pyshp.POLYGON,
'MultiPoint': pyshp.MULTIPOINT,
'MultiLineString': pyshp.POLYLINE,
'MultiPolygon': pyshp.POLYGON
}
# Convert geojson to shape
def geoj2shape(geoj):
shape = pyshp._Shape()
geojtype = geoj['type']
shape.shapeType = geoj2shape_format[geojtype]
# Set points and parts
# Points is a list of points, parts is the index of the start of each unique part
if geojtype == 'Point':
# Points don't have parts - just a list of coords
shape.points = [geoj['cordinates']]
shape.parts = [0]
elif geojtype in ('MultiPoint', 'LineString'):
# Either a set of unrelated points or a single line feature - no feature parts
shape.points = geoj['coordinates']
shape.parts = [0]
elif geojtype == 'Polygon':
# Polygons can have exterior rings and interior holes - parts of a single feature
points = []
parts = []
index = 0
for ext_or_hole in geoj['coordinates']:
# Add the point list
points.extend(ext_or_hole)
# Track where each part starts in the point list
parts.append(index)
index += len(ext_or_hole)
shape.points = points
shape.parts = parts
elif geojtype == 'MultiLineString':
# Multiline string is a line with parts
points = []
parts = []
index = 0
for linestring in geoj['coordinates']:
points.extend(linestring)
parts.append(index)
index += len(linestring)
shape.points = points
shape.parts = parts
elif geojtype == 'MultiPolygon':
# Multipolygon is a multi-part polygon - multiple parts, each potentially with their own parts
points = []
parts = []
index = 0
for Polygon in geoj['coordinates']:
for ext_or_hole in polygon:
points.extend(ext_or_hole)
parts.append(index)
index += len(ext_or_hole)
shape.points = points
shape.parts = parts
return shape
for row, geom in itertools.izip(rows, geometries):
shape = geoj2shape(geom)
shapewriter._shapes.append(shape)
shapewriter.record(*[encode(value) for value in row])
shapewriter.save(filepath)
elif filepath.lower().endswith(('json', 'geojson')):
geojwriter = pygeoj.new()
for row, geom in itertools.izip(rows, geometries):
row = (encode(value) for value in row)
rowdict = dict(zip(fields, rows))
geojwriter.add_feature(properties=rowdict, geometry=geom)
geojwriter.save(filepath)
else:
raise Exception(
"Could not save vector data to the given filepath: "
"the filetype extension is either missing or not supported")
def to_file(fields,
rows,
geometries,
filepath,
encoding="utf8",
maxprecision=12,
**kwargs):
def encode(value):
if isinstance(value, int):
# ints are kept as ints
return value
elif isinstance(value, float):
if value.is_integer():
return int(value)
elif math.isnan(value):
return None
else:
# floats are rounded
return round(value, maxprecision)
elif isinstance(value, str):
# unicode is custom encoded into bytestring
return value.encode(encoding)
elif value is None:
return value
else:
# brute force anything else to string representation
return bytes(value)
def detect_fieldtypes(fields, rows):
# TODO: allow other data types such as dates etc...
# set fields with correct fieldtype
fieldtypes = []
for fieldindex, fieldname in enumerate(fields):
fieldlen = 1
decimals = 0
fieldtype = "N" # assume number until proven otherwise
for row in rows:
value = row[fieldindex]
if is_missing(value):
# empty value, so just keep assuming same type
pass
else:
try:
# make nr fieldtype if content can be made into nr
# convert to nr or throw exception if text
value = float(value)
# rare special case where text is 'nan', is a valid input to float, so raise exception to treat as text
if math.isnan(value):
raise ValueError()
# TODO: also how to handle inf? math.isinf(). Treat as text or nr?
if math.isinf(value):
raise NotImplementedError(
"Saving infinity values not yet implemented")
# detect nr type
if value.is_integer():
_strnr = bytes(value)
else:
# get max decimals, capped to max precision
_strnr = format(value,
".%sf" % maxprecision).rstrip("0")
decimals = max(
(len(_strnr.split(".")[1]), decimals))
fieldlen = max((len(_strnr), fieldlen))
except ValueError:
# but turn to text if any of the cells cannot be made to float bc they are txt
fieldtype = "C"
value = value if isinstance(value,
str) else bytes(value)
fieldlen = max((len(value), fieldlen))
if fieldtype == "N" and decimals == 0:
fieldlen -= 2 # bc above we measure lengths for ints as if they were floats, ie with an additional ".0"
func = lambda v: "" if is_missing(v) else int(float(v))
elif fieldtype == "N" and decimals:
func = lambda v: "" if is_missing(v) else float(v)
elif fieldtype == "C":
func = lambda v: v #encoding are handled later
else:
raise Exception(
"Unexpected bug: Detected field should be always N or C")
fieldtypes.append((fieldtype, func, fieldlen, decimals))
return fieldtypes
# shapefile
if filepath.endswith(".shp"):
shapewriter = pyshp.Writer(filepath, encoding='utf8')
fieldtypes = detect_fieldtypes(fields, rows)
# set fields with correct fieldtype
for fieldname, (fieldtype, func, fieldlen,
decimals) in zip(fields, fieldtypes):
fieldname = fieldname.replace(" ", "_")[:10]
shapewriter.field(fieldname, fieldtype, fieldlen, decimals)
# iterate through original shapes
for row, geoj in zip(rows, geometries):
shapewriter.shape(geoj)
row = [
func(value)
for (typ, func, length, deci), value in zip(fieldtypes, row)
]
shapewriter.record(*row)
# save
shapewriter.close()
# geojson file
elif filepath.endswith((".geojson", ".json")):
geojwriter = pygeoj.new()
fieldtypes = detect_fieldtypes(fields, rows)
for row, geom in zip(rows, geometries):
# encode row values
row = (func(value) for (typ, func, length,
deci), value in zip(fieldtypes, row))
row = (encode(value) for value in row)
rowdict = dict(zip(fields, row))
# create and add feature
geojwriter.add_feature(properties=rowdict, geometry=geom)
# save
geojwriter.save(filepath, encoding=encoding)
# normal table file without geometry
elif filepath.endswith((".txt", ".csv")):
import csv
# TODO: Add option of saving geoms as strings in separate fields
with open(filepath, "wb") as fileobj:
csvopts = dict()
csvopts["delimiter"] = kwargs.get(
"delimiter",
";") # tab is best for automatically opening in excel...
writer = csv.writer(fileobj, **csvopts)
writer.writerow([f.encode(encoding) for f in fields])
for row, geometry in zip(rows, geometries):
writer.writerow([encode(val) for val in row])
elif filepath.endswith(".xls"):
import xlwt
with open(filepath, "wb") as fileobj:
wb = xlwt.Workbook(
encoding=encoding) # module takes care of encoding for us
sheet = wb.add_sheet("Data")
# fields
for c, f in enumerate(fields):
sheet.write(0, c, f)
# rows
for r, (row, geometry) in enumerate(zip(rows, geometries)):
for c, val in enumerate(row):
# TODO: run val through encode() func, must spit out dates as well
sheet.write(r + 1, c, val)
# save
wb.save(filepath)
else:
raise Exception(
"Could not save the vector data to the given filepath: the filetype extension is either missing or not supported"
)
def to_file(fields, rows, geometries, filepath, encoding="utf8", maxprecision=12, **kwargs):
def encode(value):
if isinstance(value, int):
# ints are kept as ints
return value
elif isinstance(value, float):
if value.is_integer():
return int(value)
else:
# floats are rounded
return round(value, maxprecision)
elif isinstance(value, unicode):
# unicode is custom encoded into bytestring
return value.encode(encoding)
elif value is None:
return value
else:
# brute force anything else to string representation
return bytes(value)
def detect_fieldtypes(fields, rows):
# TODO: allow other data types such as dates etc...
# set fields with correct fieldtype
fieldtypes = []
for fieldindex,fieldname in enumerate(fields):
fieldlen = 1
decimals = 0
fieldtype = "N" # assume number until proven otherwise
for row in rows:
value = row[fieldindex]
if value not in (None,""):
try:
# make nr fieldtype if content can be made into nr
value = float(value)
if value.is_integer():
_strnr = bytes(value)
else:
# get max decimals, capped to max precision
_strnr = format(value, ".%sf"%maxprecision).rstrip("0")
decimals = max(( len(_strnr.split(".")[1]), decimals ))
fieldlen = max(( len(_strnr), fieldlen ))
except ValueError:
# but turn to text if any of the cells cannot be made to float bc they are txt
fieldtype = "C"
value = value if isinstance(value, unicode) else bytes(value)
fieldlen = max(( len(value), fieldlen ))
else:
# empty value, so just keep assuming same type
pass
if fieldtype == "N" and decimals == 0:
fieldlen -= 2 # bc above we measure lengths for ints as if they were floats, ie with an additional ".0"
func = lambda v: "" if v in (None,"") else int(v)
elif fieldtype == "N" and decimals:
func = lambda v: "" if v in (None,"") else float(v)
elif fieldtype == "C":
func = lambda v: v #encoding are handled later
else:
raise Exception("Unexpected bug: Detected field should be always N or C")
fieldtypes.append( (fieldtype,func,fieldlen,decimals) )
return fieldtypes
# shapefile
if filepath.endswith(".shp"):
shapewriter = pyshp.Writer()
fieldtypes = detect_fieldtypes(fields,rows)
# set fields with correct fieldtype
for fieldname,(fieldtype,func,fieldlen,decimals) in itertools.izip(fields, fieldtypes):
fieldname = fieldname.replace(" ","_")[:10]
shapewriter.field(fieldname.encode(encoding), fieldtype, fieldlen, decimals)
# convert geojson to shape
def geoj2shape(geoj):
# create empty pyshp shape
shape = pyshp._Shape()
# set shapetype
geojtype = geoj["type"]
if geojtype == "Null":
pyshptype = pyshp.NULL
elif geojtype == "Point":
pyshptype = pyshp.POINT
elif geojtype == "LineString":
pyshptype = pyshp.POLYLINE
elif geojtype == "Polygon":
pyshptype = pyshp.POLYGON
elif geojtype == "MultiPoint":
pyshptype = pyshp.MULTIPOINT
elif geojtype == "MultiLineString":
pyshptype = pyshp.POLYLINE
elif geojtype == "MultiPolygon":
pyshptype = pyshp.POLYGON
shape.shapeType = pyshptype
# set points and parts
if geojtype == "Point":
shape.points = [ geoj["coordinates"] ]
shape.parts = [0]
elif geojtype in ("MultiPoint","LineString"):
shape.points = geoj["coordinates"]
shape.parts = [0]
elif geojtype in ("Polygon"):
points = []
parts = []
index = 0
for ext_or_hole in geoj["coordinates"]:
points.extend(ext_or_hole)
parts.append(index)
index += len(ext_or_hole)
shape.points = points
shape.parts = parts
elif geojtype in ("MultiLineString"):
points = []
parts = []
index = 0
for linestring in geoj["coordinates"]:
points.extend(linestring)
parts.append(index)
index += len(linestring)
shape.points = points
shape.parts = parts
elif geojtype in ("MultiPolygon"):
points = []
parts = []
index = 0
for polygon in geoj["coordinates"]:
for ext_or_hole in polygon:
points.extend(ext_or_hole)
parts.append(index)
index += len(ext_or_hole)
shape.points = points
shape.parts = parts
return shape
# iterate through original shapes
for row,geom in itertools.izip(rows, geometries):
shape = geoj2shape(geom)
shapewriter._shapes.append(shape)
row = [encode(func(value)) for (typ,func,length,deci),value in zip(fieldtypes,row)]
shapewriter.record(*row)
# save
shapewriter.save(filepath)
# geojson file
elif filepath.endswith((".geojson",".json")):
geojwriter = pygeoj.new()
fieldtypes = detect_fieldtypes(fields,rows)
for row,geom in itertools.izip(rows,geometries):
# encode row values
row = (func(value) for (typ,func,length,deci),value in zip(fieldtypes,row))
row = (encode(value) for value in row)
rowdict = dict(zip(fields, row))
# create and add feature
geojwriter.add_feature(properties=rowdict,
geometry=geom)
# save
geojwriter.save(filepath, encoding=encoding)
# normal table file without geometry
elif filepath.endswith((".txt",".csv")):
import csv
# TODO: Add option of saving geoms as strings in separate fields
with open(filepath, "wb") as fileobj:
csvopts = dict()
csvopts["delimiter"] = kwargs.get("delimiter", ";") # tab is best for automatically opening in excel...
writer = csv.writer(fileobj, **csvopts)
writer.writerow([f.encode(encoding) for f in fields])
for row,geometry in itertools.izip(rows, geometries):
writer.writerow([val.encode(encoding) if isinstance(val, unicode) else val for val in row])
else:
raise Exception("Could not save the vector data to the given filepath: the filetype extension is either missing or not supported")
parser.add_argument('--dest_geojson',
dest="geojson_filename",
help='The output path for the geojson file',
default=None)
parser.add_argument('--tiles_size',
dest='tiles_size',
type=int,
default=256,
help=('Specify the tiles size in pixels '
'(assumes both x and y are the same)'))
args = parser.parse_args()
######################## Creating the geojson file
geojson = pygeoj.new()
geojson.define_crs(type="name", name="urn:ogc:def:crs:OGC:1.3:CRS84")
g = globalmaptiles.GlobalMercator()
for dirname, subdirList, filename_list in os.walk(args.tiles_path):
for filename in filename_list:
if filename.endswith('.png'):
rel_path = '.' + os.sep + os.path.join(dirname, filename)
tms_z, tms_x, tms_y = rel_path.rstrip('.png').split(
os.sep)[-3:]
tms_z, tms_x, tms_y = map(int, (tms_z, tms_x, tms_y))
minLat, minLon, maxLat, maxLon = g.TileLatLonBounds(
tms_x, tms_y, tms_z)
# minLat, minLon = lower left corner
# maxLat, maxLon = upper right corner
#### Unused fields - we will add these in later
## rowDict['date'] = date
## rowDict['location'] = location
## rowDict['bathLines'] = bathLines
## rowDict['bathInterv'] = bathInterv
## rowDict['instCallNo'] = instCallNo
## rowDict['digHold'] = digHold
## rowDict['thumbUrl'] = thumbUrl
## rowDict['iiifUrl'] = iiifUrl
## rowDict['downloadUrl'] = downloadUrl
## rowDict['websiteUrl'] = websiteUrl
return rowDict
# create a new geojson for the OIM Data
almaGeo = pygeoj.new()
for row in almaData:
rowDict = processRow(row)
outputWriter.writerow(rowDict)
# skip any rows that do not have coordinates
if rowDict['west'] is not None or rowDict['east'] is not None or rowDict['north'] is not None or rowDict['south'] is not None:
# Add the values from the dictionary into the geojson feature properties and coordinates
almaGeo.add_feature(properties={"label": rowDict['label'], "west": rowDict['west'], "east": rowDict['east'], "north": rowDict['north'],
"south": rowDict['south'], "scale": rowDict['scale'], "title": rowDict['title'],
"edition": rowDict['edition'], "available": rowDict['available'], "physHold": rowDict['physHold'],
"primeMer": rowDict['primeMer'], "projection": rowDict['projection'], "publisher": rowDict['publisher'],
"datePub": rowDict['datePub'], "color": rowDict['color'], "recId": rowDict['recId'], "note": rowDict['note']},
geometry={"type":"Polygon", "coordinates":[[(rowDict['west'],rowDict['north']),(rowDict['west'],rowDict['south']),(rowDict['east'],
rowDict['south']), (rowDict['east'], rowDict['north']), (rowDict['west'], rowDict['north'])]]} )
outputCSV.close()
#######################################
# DATA COLLECTION
#######################################
#Check if the folder "images" already exists; if not, create it
'''
if os.path.exists('images/'):
pass
else:
os.makedirs("images/")
'''
#Start reading the csv file
with open(gform_file, 'r', newline='') as gform_file:
file_output = pygeoj.new()
readgform = csv.reader(gform_file, delimiter='\t')
prog_number = itertools.count()
json_values = []
# Skip the first row, containing the header
next(readgform, None)
rows = [r for r in readgform]
for row in rows:
gform_nome = row[0].capitalize()
gform_cognome = row[1].capitalize()
gform_luogonascita = row[2]
gform_datanascita = row[3]
gform_intervista = row[4]
gform_qualifica = row[5]
gform_lat = row[6]