def mergeDBF(receiverFile, resultFile, mergedFile, ID):
DBFin1 = dbflib.open(receiverFile)
DBFin2 = dbflib.open(resultFile)
DBFOut = dbflib.create(mergedFile)
print DBFin1.record_count()-DBFin2.record_count(), ' points are missed.'
# add field
for n in xrange(DBFin1.field_count()):
fi = DBFin1.field_info(n)
DBFOut.add_field(fi[1], fi[0], fi[2], fi[3])
for n in xrange(DBFin2.field_count()):
fi = DBFin2.field_info(n)
DBFOut.add_field(fi[1], fi[0], fi[2], fi[3])
# copy attributes
for r in xrange(DBFin1.record_count()):
print 'merging ', r, ' record'
for f in xrange(DBFin1.field_count()):
v = DBFin1.read_attribute(r, f)
DBFOut.write_attribute(r, f, v)
IDentify = DBFin1.read_record(r)[ID]
for r2 in xrange(DBFin2.record_count()):
if IDentify==DBFin2.read_record(r2)[ID]:
break
for f in xrange(DBFin2.field_count()):
v = DBFin2.read_attribute(r2, f)
DBFOut.write_attribute(r, f+DBFin1.field_count(), v)
DBFin1.close()
DBFin2.close()
DBFOut.close()
def add_dbf_records(file):
# add some records to file
dbf = dbflib.open(file, "r+b")
# Records can be added as a dictionary...
dbf.write_record(0, {'NAME': "Weatherwax", "INT":1, "FLOAT":3.1415926535})
# ... or as a sequence
dbf.write_record(1, ("Ogg", 2, -1000.1234))
def shpGetField(shpFile, fieldName=None):
"""
Return an array containing the specified field (if it is contained
within the corresponding dbf file). If no field name is specified,
return all fields.
"""
try:
dbffh = dbflib.open(shpFile)
except IOError:
logger.warn("Cannot open %s.dbf"%shpFile)
raise
nrecs = dbffh.record_count()
nfields = dbffh.field_count()
data = {}
for f in xrange(nfields):
fname = dbffh.field_info(f)[1]
data[fname]=[]
for rec in xrange(nrecs):
recdata = dbffh.read_record(rec)
for key in recdata.keys():
data[key].append(recdata[key])
if fieldName:
if data.has_key(fieldName):
returnData = data[fieldName]
else:
logger.warn("%s.dbf does not contain a field called %s - returning all data" \
% (shpFile,fieldName))
returnData = data
else:
returnData = data
return returnData
def add_dbf_records(file):
# add some records to file
dbf = dbflib.open(file, "r+b")
# Records can be added as a dictionary...
dbf.write_record(0, {'NAME': "Weatherwax", "INT":1, "FLOAT":3.1415926535})
# ... or as a sequence
dbf.write_record(1, ("Ogg", 2, -1000.1234))
def get_data(fn):
''' Load a DBF file into a pandas DF '''
rows = []
dbf = dbflib.open(fn)
for i in range(dbf.record_count()):
rows.append(dbf.read_record(i))
return pd.DataFrame(rows)
def get_data(fn):
''' Load a DBF file into a pandas DF '''
rows = []
dbf = dbflib.open(fn)
for i in range(dbf.record_count()):
rows.append( dbf.read_record(i) )
return pd.DataFrame(rows)
def main(shapefile, picklefile):
if picklefile:
[npts, x, y, z, zraw, xil, yil, grid, missing] = cPickle.load(open(picklefile,'rb'))
points = np.vstack((x,y)).T
# Load administrative area
shp = ShapeFile(shapefile)
dbf = dbflib.open(shapefile)
coastline = []
# Process every shape from the ShapeFile
print "Processing shapes ..."
for npoly in range(shp.info()[0]):
shp_object = shp.read_object(npoly)
shp_dict = dbf.read_record(npoly)
verts = shp_object.vertices()
if "NAME_1" in shp_dict:
name = "%s" % (shp_dict["NAME_1"])
else:
name = "Unknown"
print "Processing %s" % (name)
# Extract city polygon vertices (ring per ring)
for ring in verts:
vx = []
vy = []
for point in ring:
vx.append(point[0])
vy.append(point[1])
# Only process big enough rings
if len(vx) > 256: # big enough
poly_verts = zip(vx,vy)
if picklefile:
# Compute intersections with the city
intersection = points_inside_poly(points, poly_verts)
npts = sum(1 for x in points_inside_poly(points, poly_verts) if x)
else:
npts = 1 # Add this polygon
# Add the ring to the coastine if measures inside
if npts > 0:
polygon = Polygon(poly_verts)
if not polygon.is_empty and polygon.is_valid:
print "- Add polygon (%d)" % (len(vx))
coastline.append(polygon)
else:
print "- Skip polygon (%d)" % (len(vx))
print "Union of %d polygons" % len(coastline)
coast = cascaded_union(coastline)
cPickle.dump(coast,open('coastline.pickle','wb'),-1)
print "Done."
def __init__(self,
shpfile = "worldmap/TM_WORLD_BORDERS-0.3.shp",
dbffile = "worldmap/TM_WORLD_BORDERS-0.3.dbf"):
shp = ShapeFile(shpfile)
dbf = dbflib.open(dbffile)
self.countries = {}
for i in range(shp.info()[0]):
c = dbf.read_record(i)['ISO2']
poly = shp.read_object(i)
self.countries[c] = poly.vertices()
def shpGetVertices(shpFile, keyName=None):
"""
Returns a dictionary of arrays containing coordinate pairs
representing vertices contained in the shapefile.
Dictionary keys can either be a field contained within the
corresponding dbf file (through the optional kwarg keyName), or if
no key name is provided, the object id number (i.e. the record
number) is used.
WARNING:
If any records share the same value for the chosen key, then only
one record will be retained in the returned values.
As yet untested on MULTIPOINT shapefiles (polylines, polygons).
Input: shpFile - path to a shapefile, excluding the extension
(shapelib/dbflib automatically append the correct extension)
keyName - (optional) name of a field in the dbf file that
acts as a key for the dictionary of returned
vertices
Output: dictionary keyed by object id number or optionally a field
name, with values being arrays of vertices of the
corresponding shape object.
Example: vertices = shpGetVertices(shpFile,keyName)
"""
try:
shpfh = shapelib.open(shpFile)
except IOError:
logger.warn("Cannot open %s"%shpFile)
raise
try:
dbffh = dbflib.open(shpFile)
except IOError:
logger.warn("Cannot open %s.dbf"%shpFile)
raise
vertices = {}
nshapes = shpfh.info()[0]
nfields = dbffh.field_count()
for oid in xrange(nshapes):
shpdata = shpfh.read_object(oid)
dbfdata = dbffh.read_record(oid)
nparts = len(shpdata.vertices())
v = []
for p in xrange(nparts):
v.append(shpdata.vertices()[p])
if keyName and dbfdata.has_key(keyName):
vertices[dbfdata[keyName]] = v
else:
vertices[oid] = v
shpfh.close()
dbffh.close()
return vertices
def load(self):
shapes = shapelib.open(self.name)
if len(self._fields) != 0:
data = dbflib.open(self.name)
list = SpatialPointList()
for i in range(shapes.info()[0]):
records = Dictionary()
for key, gen in self._fields:
records.update([(gen, float(data.read_record(i)[key]))])
x = float(shapes.read_object(i).vertices()[0][0])
y = float(shapes.read_object(i).vertices()[0][1])
list.append(SpatialPoint(x, y, records, self))
return list
def packReceiverToPKL(receiverShapeFile, IDFieldName):
shp = shapelib.open(receiverShapeFile)
dbf = dbflib.open(receiverShapeFile)
receiverObjects = []
for r in xrange(shp.info()[0]):
shpObj = shp.read_object(r)
vtx = shpObj.vertices()[0]
identify = dbf.read_record(r)[IDFieldName]
rObj = Receivers(vtx, identify)
receiverObjects.append(rObj)
shp.close()
dbf.close()
return receiverObjects
def packReceiverToPKL(receiverShapeFile, IDFieldName):
shp = shapelib.open(receiverShapeFile)
dbf = dbflib.open(receiverShapeFile)
receiverObjects = []
for r in xrange(shp.info()[0]):
shpObj = shp.read_object(r)
vtx = shpObj.vertices()[0]
identify = dbf.read_record(r)[IDFieldName]
rObj = Receivers(vtx, identify)
receiverObjects.append(rObj)
shp.close()
dbf.close()
return receiverObjects
def load(self):
shapes = shapelib.open(self.name)
if len(self._fields) != 0:
data = dbflib.open(self.name)
list = SpatialPointList()
for i in range(shapes.info()[0]):
records = Dictionary()
for key, gen in self._fields:
records.update([(gen, float(data.read_record(i)[key]))])
x = float(shapes.read_object(i).vertices()[0][0])
y = float(shapes.read_object(i).vertices()[0][1])
list.append(SpatialPoint(x, y, records, self))
return list
def read_shapefile(fp, heal=True):
"""
Reads an ESRI Shapeifle and converts it to Shapely Geometry objects. The DBF is also read and each returned geometry
has its DBF attributes attached.
Examples:
points = read_shapefile("airports.shp")
p0 = points[0]
p0.geom_type
>>'Point'
p0['Airport_code']
>>'YYZ'
Args:
fp (str or unicode): The path to the shapefile
heal (bool): If true, this function will attempt to auto-heal any invalid the geometry using buffer(0).
Returns:
List of ShapelyGeometry objects
"""
basepath, _ = path.splitext(fp)
shp_path = basepath + ".shp"
dbf_path = basepath + ".dbf"
shapefile = shp.open(shp_path)
try:
shp_length, shp_type, x_extents, y_extents = shapefile.info()
tablefile = dbf.open(dbf_path)
except:
shapefile.close()
raise
try:
geoms = []
assert shp_length == tablefile.record_count()
for fid in xrange(shp_length):
record = shapefile.read_object(fid)
geom = _make_geom(record)
if not geom.is_valid and heal:
geom = geom.buffer(0)
attrs = tablefile.read_record(fid)
geoms.append(ShapelyGeometry(geom, attrs))
return geoms
finally:
shapefile.close()
tablefile.close()
def packReceiverToPKL(receiverShapeFile, IDFieldName):
shp = shapelib.open(receiverShapeFile)
dbf = dbflib.open(receiverShapeFile)
receiverObjects = []
for r in xrange(shp.info()[0]):
print "receiver: ", r
shpObj = shp.read_object(r)
vtx = shpObj.vertices()[0]
absoluteHeight = antw_dtm.read_vtx(vtx)
identify = dbf.read_record(r)[IDFieldName]
rObj = Receivers(vtx, identify, absoluteHeight)
receiverObjects.append(rObj)
shp.close()
dbf.close()
return receiverObjects
def pointsFromSHP (pointsFilename, datafield=[]):
shapes = shapelib.open(pointsFilename)
if len(datafield) != 0:
data = dbflib.open (pointsFilename)
list = SpatialPointList ()
for i in range (shapes.info ()[0]):
records = {}
pair = []
for entry in datafield:
pair.append ((entry, data.read_record (i)[entry]))
records.update (pair)
x = shapes.read_object (i).vertices ()[0][0]
y = shapes.read_object (i).vertices ()[0][1]
list.append (SpatialPoint (x, y, records))
return list
def pointsFromSHP(pointsFilename, datafield=[]):
shapes = shapelib.open(pointsFilename)
if len(datafield) != 0:
data = dbflib.open(pointsFilename)
list = SpatialPointList()
for i in range(shapes.info()[0]):
records = {}
pair = []
for entry in datafield:
pair.append((entry, data.read_record(i)[entry]))
records.update(pair)
x = shapes.read_object(i).vertices()[0][0]
y = shapes.read_object(i).vertices()[0][1]
list.append(SpatialPoint(x, y, records))
return list
def load(self):
shapes = shapelib.open(self.name)
data = dbflib.open(self.name)
polys = PolygonDictionary()
for i in range(shapes.info()[0]):
shp = shapes.read_object(i)
d = data.read_record(i)[self._primary]
if self._subset != None:
for item in self._subset:
if item == d:
p = SpatialPolygon(shp.vertices(), [], self)
polys.update({d: p})
else:
p = SpatialPolygon(shp.vertices(), [], self)
polys.update({d: p})
return polys
def load(self):
shapes = shapelib.open(self.name)
data = dbflib.open(self.name)
polys = PolygonDictionary()
for i in range(shapes.info()[0]):
shp = shapes.read_object(i)
d = data.read_record(i)[self._primary]
if self._subset != None:
for item in self._subset:
if item == d:
p = SpatialPolygon(shp.vertices(), [], self)
polys.update({d: p})
else:
p = SpatialPolygon(shp.vertices(), [], self)
polys.update({d: p})
return polys
def packSourceToPKL(sourceShapeFile):
shp = shapelib.open(sourceShapeFile)
dbf = dbflib.open(sourceShapeFile)
specField = ['L_63', 'L_125', 'L_250', 'L_500', 'L_1000', 'L_2000', 'L_4000', 'L_8000']
sourceObjects = []
for r in xrange(shp.info()[0]):
shpObj = shp.read_object(r)
p_source = shpObj.vertices()[0] # (x, y)
rec = dbf.read_record(r)
specD = [rec[f] for f in specField]
spec = [specD]
sObj = Sources(p_source, spec)
sourceObjects.append(sObj)
shp.close()
dbf.close()
return sourceObjects # list of soruce object. values defined in "Sources" can be queried
def polysFromSHP(polysFilename, datafield, polySubset):
R.importLibrary('sp')
shapes = shapelib.open(polysFilename)
data = dbflib.open(polysFilename)
polyList = []
for i in range(shapes.info()[0]):
shp = shapes.read_object(i)
d = data.read_record(i)[datafield]
if len(polySubset) != 0:
for item in polySubset:
if item == d:
p = SpatialPolygon(i, shp.vertices(), d)
polyList.append(p)
else:
p = SpatialPolygon(i, shp.vertices(), d)
polyList.append(p)
return polyList
def mapcountries(countries): #Map list of countries?
from mpl_toolkits.basemap import Basemap
import matplotlib.pyplot as plt
fig = plt.figure(figsize=(11.7, 8.3))
plt.subplots_adjust()
m = Basemap(projection='hammer', resolution=None, lon_0=0)
m.drawcountries(linewidth=0.5)
m.drawcoastlines(linewidth=0.5)
from shapelib import ShapeFile
import dbflib
from matplotlib.collections import LineCollection
from matplotlib import cm
#for ctry in countries:
shp = ShapeFile(r"borders/world_adm1")
dbf = dbflib.open(r"borders/world_adm1")
for npoly in range(shp.info()[0]):
shpsegs = []
shpinfo = []
shp_object = shp.read_obj(npoly)
verts = shp_object.vertices()
rings = len(verts)
for ring in range(rings):
lons, lats = zip(*verts[ring])
if max(lons) > 721. or min(lons) < -721. or max(lats) > 91. or min(
lats) < -91.:
raise ValueError('Lats/Lons out of range')
x, y = m(lons, lats)
shpsegs.append(zip(x, y))
if ring == 0:
shapedict = dbf.read_record(npoly)
name = shapedict["NAME_1"]
shapedict['RINGNUM'] = ring + 1
shapedict['SHAPENUM'] = npoly + 1
shpinfo.append(shapedict)
print(name)
lines = LineCollection(shpsegs, antialiaseds=(1, ))
lines.set_facecolors(cm.jet(0.5))
lines.set_edgecolors('k')
lines.set_linewidth(0.3)
ax.add_collection(lines)
plt.show()
def plotclustering(label, zipcodes, nameee):
# print len(zipcodes)
label = np.array(label)
zipcodes = np.array(zipcodes)
cluster1 = zipcodes[label == 1]
cluster2 = zipcodes[label == 2]
cluster0 = zipcodes[label == 0]
cluster3 = zipcodes[label == 3]
fig = plt.figure(figsize =(15, 15))
ax = plt.subplot(111)
lllat = 40.473; urlat = 40.93; lllon = -74.27; urlon = -73.69 # define the boundary of the map
m = Basemap(ax=ax, projection = 'stere', lon_0 = (urlon + lllon)/2, lat_0 = (urlat +lllat)/2, llcrnrlon = lllon, llcrnrlat = lllat, urcrnrlon = urlon, urcrnrlat = urlat, resolution= 'l')# create the basemap
# m.drawcoastlines()
m.drawcountries()
shp = ShapeFile('c:/Users/gang/Desktop/nyczipregion')
dbf = dbflib.open('c:/Users/gang/Desktop/nyczipregion')
for npoly in range(shp.info()[0]):
shpsegs = []
shp_object = shp.read_object(npoly)
verts = shp_object.vertices()
rings = len(verts)
for ring in range(rings):
lons , lats = zip(*verts[ring])
x, y = m(lons, lats)
shpsegs.append(zip(x, y))
if ring ==0:
shapedict = dbf.read_record(npoly)
name = shapedict['Zcta5ce00']
lines = LineCollection(shpsegs, antialiaseds=(1,))
if name in cluster0:
lines.set_facecolors('b')
if name in cluster1:
lines.set_facecolors('g')
if name in cluster2:
lines.set_facecolors('r')
if name in cluster3:
lines.set_facecolors('y')
lines.set_alpha(1)
lines.set_edgecolors('k')
ax.add_collection(lines)
plt.title('Box Clustering Based On Taxi Trips')
plt.savefig(nameee+'_box_trip_Clustering.png', dpi=300)
plt.show()
def polysFromSHP (polysFilename, datafield, polySubset):
R.importLibrary ('sp')
shapes = shapelib.open(polysFilename)
data = dbflib.open (polysFilename)
polyList = []
for i in range (shapes.info ()[0]):
shp = shapes.read_object (i)
d = data.read_record (i)[datafield]
if len(polySubset) != 0:
for item in polySubset:
if item == d:
p = SpatialPolygon (i, shp.vertices (), d)
polyList.append(p)
else:
p = SpatialPolygon (i, shp.vertices (), d)
polyList.append(p)
return polyList
def readPolygonFile(iFilename, ID_field):
print 'reading simple polygon file', iFilename
relativeHeihgtField = 'REL_HEIGHT'
shp = shapelib.ShapeFile(iFilename)
dbf = dbflib.open(iFilename)
polygons = []
for i in xrange(shp.info()[0]):
if len(shp.read_object(i).vertices())>0:
vs = shp.read_object(i).vertices()[0]
vs = [(v[0] , v[1] ) for v in vs[1:]]
vs.reverse()
polygon = SimplePolygon2D(vs) # polygon.vertices return the vertices of the
polygon.relativeHeight = dbf.read_record(i)[relativeHeihgtField]
polygon.pid = dbf.read_record(i)[ID_field]
polygons.append(polygon)
shp.close()
dbf.close()
return polygons
def packSourceToPKL(sourceShapeFile):
shp = shapelib.open(sourceShapeFile)
dbf = dbflib.open(sourceShapeFile)
specField = [
'L_63', 'L_125', 'L_250', 'L_500', 'L_1000', 'L_2000', 'L_4000',
'L_8000'
]
sourceObjects = []
for r in xrange(shp.info()[0]):
shpObj = shp.read_object(r)
p_source = shpObj.vertices()[0] # (x, y)
rec = dbf.read_record(r)
specD = [rec[f] for f in specField]
spec = [specD]
sObj = Sources(p_source, spec)
sourceObjects.append(sObj)
shp.close()
dbf.close()
return sourceObjects # list of soruce object. values defined in "Sources" can be queried
def serialize_tm_world(shapefile_fn, output_fn):
print 'serialize ' + shapefile_fn + '.shp'
tm_world = world()
shp = ShapeFile(shapefile_fn + '.shp')
dbf = dbflib.open(shapefile_fn + '.dbf')
nb_records = shp.info()[0]
for i in xrange(nb_records):
shp_record = dbf.read_record(i)
shp_object = shp.read_object(i)
# countries
name = shp_record['NAME']
shp_record['SHAPE'] = shp_object.vertices()
tm_world.countries[name] = shp_record
# regions
region = shp_record['REGION']
tm_world.regions[region].append(name)
# subregions
subregion = shp_record['SUBREGION']
tm_world.subregions[subregion].append(name)
pickle.dump(tm_world, gzip.open(output_fn,'w'))
def _redrawMap(self, receiverFileName, epsilon, delta, timeHour, day):
receiverLevels = self._updateMap(epsilon, delta, timeHour)
DBF = dbflib.open(receiverFileName)
RECN = DBF.record_count()
if not os.path.exists(str(day)):
os.mkdir(str(day))
newDBF = dbflib.create(
str(day) + '\\' + receiverFileName + '_update_' + str(timeHour) +
'u')
# copy the old field and record
for f in xrange(DBF.field_count()):
fi = DBF.field_info(f)
newDBF.add_field(fi[1], fi[0], fi[2], fi[3])
for r in xrange(RECN):
rec = DBF.read_record(r)
newDBF.write_record(r, rec)
# add new field and attribute
newDBF.add_field('DBA_D', dbflib.FTDouble, 9, 1)
for r in xrange(RECN):
newDBF.write_attribute(r, f + 1, receiverLevels[r])
DBF.close()
newDBF.close()
def clear_small_polygons(self, shapeFile, newShapeFile, lenLimit):
shp = shapelib.ShapeFile(shapeFile)
dbf = dbflib.open(shapeFile)
newSHP = shapelib.create(newShapeFile, self.shpType)
newDBF = dbflib.create(newShapeFile)
for f in range(dbf.field_count()):
fi = dbf.field_info(f)
newDBF.add_field(fi[1], fi[0], fi[2], fi[3])
bb = 0
for b in range(shp.info()[0]):
sobj = shp.read_object(b)
rec = dbf.read_record(b)
if self.length_of_polygon(sobj.vertices()) > lenLimit:
shpobj = shapelib.SHPObject(self.shpType, bb,
[sobj.vertices()[0]])
newSHP.write_object(bb, shpobj)
newDBF.write_record(bb, rec)
bb += 1
shp.close()
dbf.close()
newSHP.close()
newDBF.close()
def map_communities_and_commutes(G):
G_mod = nx.read_gml("/home/sscepano/D4D res/allstuff/User movements graphs/commuting patterns/1/total_commuting_G_scaled_weights_11COM_713_7115.gml")
col = [str]*256
for i in range(256):
col[i] = 'w'
for node in G_mod.nodes_iter(data=True):
#print node[1]['label']
col_gephi = node[1]['graphics']['fill']
while (len(col_gephi) < 7):
col_gephi += '0'
subpref_gephi = int(float(node[1]['label']))
print subpref_gephi, col_gephi
col[subpref_gephi] = col_gephi
#print col
plt.clf()
plt.subplots_adjust(left=0.05,right=0.95,top=0.90,bottom=0.05,wspace=0.15,hspace=0.05)
ax = plt.subplot(111)
m = Basemap(llcrnrlon=-9, \
llcrnrlat=3.8, \
urcrnrlon=-1.5, \
urcrnrlat = 11, \
resolution = 'h', \
projection = 'tmerc', \
lat_0 = 7.38, \
lon_0 = -5.30)
# read the shapefile archive
s = m.readshapefile('/home/sscepano/DATA SET7S/D4D/SubPrefecture/GEOM_SOUS_PREFECTURE', 'subpref')
from shapelib import ShapeFile
import dbflib
from matplotlib.collections import LineCollection
from matplotlib import cm
shp = ShapeFile(r'/home/sscepano/DATA SET7S/D4D/SubPrefecture/GEOM_SOUS_PREFECTURE')
dbf = dbflib.open(r'/home/sscepano/DATA SET7S/D4D/SubPrefecture/GEOM_SOUS_PREFECTURE')
for npoly in range(shp.info()[0]):
shpsegs = []
shpinfo = []
shp_object = shp.read_object(npoly)
verts = shp_object.vertices()
rings = len(verts)
for ring in range(rings):
lons, lats = zip(*verts[ring])
x, y = m(lons, lats)
shpsegs.append(zip(x,y))
if ring == 0:
shapedict = dbf.read_record(npoly)
#print shapedict
name = shapedict["ID_DEPART"]
subpref_id = shapedict["ID_SP"]
# add information about ring number to dictionary.
shapedict['RINGNUM'] = ring+1
shapedict['SHAPENUM'] = npoly+1
shpinfo.append(shapedict)
#print subpref_id
#print name
lines = LineCollection(shpsegs,antialiaseds=(1,))
lines.set_facecolors(col[subpref_id])
lines.set_edgecolors('gray')
lines.set_linewidth(0.3)
ax.add_collection(lines)
m.drawcoastlines()
plt.show()
# # data to plot on the map
# lons = [int]*256
# lats = [int]*256
#
# # read in coordinates fo subprefs
# file_name2 = "/home/sscepano/DATA SET7S/D4D/SUBPREF_POS_LONLAT.TSV"
# f2 = open(file_name2, 'r')
#
# # read subpref coordinates
# subpref_coord = {}
# for line in f2:
# subpref_id, lon, lat = line.split('\t')
# lon = float(lon)
# lat = float(lat)
# subpref_id = int(subpref_id)
# subpref_coord.keys().append(subpref_id)
# subpref_coord[subpref_id] = (lon, lat)
#
# f2.close()
#
# # if wanna plot number of users whose this is home subpref
# for subpref in range(1,256):
# lons[subpref] = subpref_coord[subpref][0]
# lats[subpref] = subpref_coord[subpref][1]
#
#
# if G.has_node(-1):
# G.remove_node(-1)
#
# max7s = 1
# min7s = 1
# for u,v,d in G.edges(data=True):
# if d['weight'] > max7s:
# max7s = d['weight']
# if d['weight'] < min7s:
# min7s = d['weight']
#
# max7s = float(max7s)
# print max7s
# print min7s
#
# scaled_weight = defaultdict(int)
# for i in range(256):
# scaled_weight[i] = defaultdict(int)
#
# for u,v,d in G.edges(data=True):
# node1 = G.nodes(data=True)[u][1]['label']
# node2 = G.nodes(data=True)[v][1]['label']
# print u, node1
# print v, node2
# print d
# scaled_weight[node1][node2] = (d['weight'] - min7s) / (max7s - min7s)
#
## for u,v,d in G.edges(data=True):
## print u,v,d
## node1 = G.nodes(data=True)[u][1]['label']
## node2 = G.nodes(data=True)[v][1]['label']
## print node1, G_mod.nodes(data=True)[u][1]['label']
## print node2, G_mod.nodes(data=True)[v][1]['label']
#
#
# for u, v, d in G.edges(data=True):
# node1 = G.nodes(data=True)[u][1]['label']
# node2 = G.nodes(data=True)[v][1]['label']
# print node1
# print node2
# lo = []
# la = []
# print u
# print v
# lo.append(lons[node1])
# lo.append(lons[node2])
# la.append(lats[node1])
# la.append(lats[node2])
# #m.drawgreatcircle(lons[u],lats[u], lons[v],lats[v])
# x, y = m(lo, la)
# #linewidth7s = d['weight']
# #linewidth7s = d['weight'] / max7s
# #lons, lats = n.meshgrid(lo,la)
# linewidth7s = scaled_weight[node1][node2] * 7 + 0.2
# m.plot(x,y, 'b', linewidth = linewidth7s)
# #wave = 0.75*(np.sin(2.*lats)**8*np.cos(4.*lons))
# #mean = 0.5*np.cos(2.*lats)*((np.sin(2.*lats))**2 + 2.)
# #m.contour(x,y,linewidth=linewidth7s)
# #m.quiver(x,y,lons, lats, latlon=True)
## if linewidth7s > 1:
## print linewidth7s
#
#
# figure_name = "/home/sscepano/D4D res/allstuff/User movements graphs/commuting patterns/1/maps/mod_classes_SCALED_11COM_713_7115.png"
# print(figure_name)
# plt.savefig(figure_name, format = "png",dpi=1000)
return
def plot_gspan_res(G, subpref_id, color_val):
fig = plt.figure(subpref_id)
#Custom adjust of the subplots
plt.subplots_adjust(left=0.05,right=0.95,top=0.90,bottom=0.05,wspace=0.15,hspace=0.05)
ax = plt.subplot(111)
m = Basemap(llcrnrlon=-9, \
llcrnrlat=3.8, \
urcrnrlon=-1.5, \
urcrnrlat = 11, \
resolution = 'h', \
projection = 'tmerc', \
lat_0 = 7.38, \
lon_0 = -5.30)
# read the shapefile archive
s = m.readshapefile('/home/sscepano/DATA SET7S/D4D/SubPrefecture/GEOM_SOUS_PREFECTURE', 'subpref')
m.drawcoastlines()
shp = ShapeFile(r'/home/sscepano/DATA SET7S/D4D/SubPrefecture/GEOM_SOUS_PREFECTURE')
dbf = dbflib.open(r'/home/sscepano/DATA SET7S/D4D/SubPrefecture/GEOM_SOUS_PREFECTURE')
msg = "Out of bounds"
color_col = []
for npoly in range(shp.info()[0]):
shpsegs = []
shpinfo = []
shp_object = shp.read_object(npoly)
verts = shp_object.vertices()
rings = len(verts)
for ring in range(rings):
lons, lats = zip(*verts[ring])
x, y = m(lons, lats)
shpsegs.append(zip(x,y))
if ring == 0:
shapedict = dbf.read_record(npoly)
#print shapedict
name = shapedict["ID_DEPART"]
subpref_id2 = shapedict["ID_SP"]
#color_col
# add information about ring number to dictionary.
shapedict['RINGNUM'] = ring+1
shapedict['SHAPENUM'] = npoly+1
shpinfo.append(shapedict)
lines = LineCollection(shpsegs,antialiaseds=(1,))
if subpref_id == subpref_id2:
lines.set_facecolors('g')
lines.set_edgecolors('k')
lines.set_linewidth(0.3)
ax.add_collection(lines)
# data to plot on the map
lons = [int]*256
lats = [int]*256
num = []
# read in coordinates fo subprefs
file_name2 = "/home/sscepano/DATA SET7S/D4D/SUBPREF_POS_LONLAT.TSV"
f2 = open(file_name2, 'r')
# read subpref coordinates
subpref_coord = {}
for line in f2:
subpref_id, lon, lat = line.split('\t')
lon = float(lon)
lat = float(lat)
subpref_id = int(subpref_id)
subpref_coord.keys().append(subpref_id)
subpref_coord[subpref_id] = (lon, lat)
f2.close()
# if wanna plot number of users whose this is home subpref
for subpref in range(1,256):
lons[subpref] = subpref_coord[subpref][0]
lats[subpref] = subpref_coord[subpref][1]
for u, v, d in G.edges(data=True):
lo = []
la = []
lo.append(lons[u])
lo.append(lons[v])
la.append(lats[u])
la.append(lats[v])
x, y = m(lo, la)
m.plot(x,y, color = color_val)
return plt
COAPS_SW = [(-84.12, 31.52, '90140', 'Albany'),
(-84.18, 31.79, '91500', 'Camilla'),
(-84.77, 31.17, '92153', 'Colquitt'),
(-84.77, 31.77, '92450', 'Cuthbert')]
if 1:
file = "USGS_sGA.dbf"
dbf = dbflib.create(file)
dbf.add_field("LON", dbflib.FTDouble, 10, 4)
dbf.add_field("LAT", dbflib.FTDouble, 10, 4)
dbf.add_field("CODE", dbflib.FTString, 8, 0)
dbf.add_field("NAME", dbflib.FTString, 40, 0)
dbf.close()
#
dbf = dbflib.open(file, "r+b")
for (i, record) in enumerate(USGS_SE + USGS_SW):
dbf.write_record(i, record)
dbf.close()
#
filename = "USGS_sGA.shp"
outfile = shapelib.create(filename, shapelib.SHPT_POINT)
for (i, record) in enumerate(USGS_SE + USGS_SW):
obj = shapelib.SHPObject(shapelib.SHPT_POINT, i,
[[(record[0], record[1])]])
outfile.write_object(-1, obj)
outfile.close()
if 1:
file = "COAPS_sGA.dbf"
dbf = dbflib.create(file)
urcrnrlon=-1.5, \
urcrnrlat = 11, \
resolution = 'h', \
projection = 'tmerc', \
lat_0 = 7.38, \
lon_0 = -5.30)
m.drawcoastlines()
from shapelib import ShapeFile
import dbflib
from matplotlib.collections import LineCollection
from matplotlib import cm
shp = ShapeFile(r'/home/sscepano/DATA SET7S/D4D/SubPrefecture/GEOM_SOUS_PREFECTURE')
dbf = dbflib.open(r'/home/sscepano/DATA SET7S/D4D/SubPrefecture/GEOM_SOUS_PREFECTURE')
msg = "Out of bounds"
color_col = []
for npoly in range(shp.info()[0]):
shpsegs = []
shpinfo = []
shp_object = shp.read_object(npoly)
verts = shp_object.vertices()
rings = len(verts)
for ring in range(rings):
lons, lats = zip(*verts[ring])
x, y = m(lons, lats)
def map_commutes(G):
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.basemap import Basemap
import matplotlib as mpl
mpl.rcParams['font.size'] = 10.
mpl.rcParams['axes.labelsize'] = 8.
mpl.rcParams['xtick.labelsize'] = 6.
mpl.rcParams['ytick.labelsize'] = 6.
from shapelib import ShapeFile
import dbflib
from matplotlib.collections import LineCollection
from matplotlib import cm
###########################################################################################
fig = plt.figure(3)
#Custom adjust of the subplots
plt.subplots_adjust(left=0.05,right=0.95,top=0.90,bottom=0.05,wspace=0.15,hspace=0.05)
ax = plt.subplot(111)
m = Basemap(projection='cyl',\
llcrnrlon=-9, \
llcrnrlat=3.8, \
urcrnrlon=-1.5, \
urcrnrlat = 11, \
resolution = 'h', \
# projection = 'tmerc', \
lat_0 = 7.38, \
lon_0 = -5.30)
# read subpref coordinates
subpref_coord = read_subpref_lonlat()
shp = ShapeFile(r'/home/sscepano/DATA SET7S/D4D/SubPrefecture/GEOM_SOUS_PREFECTURE')
dbf = dbflib.open(r'/home/sscepano/DATA SET7S/D4D/SubPrefecture/GEOM_SOUS_PREFECTURE')
for npoly in range(shp.info()[0]):
shpsegs = []
shpinfo = []
shp_object = shp.read_object(npoly)
verts = shp_object.vertices()
rings = len(verts)
for ring in range(rings):
lons, lats = zip(*verts[ring])
# if max(lons) > 721. or min(lons) < -721. or max(lats) > 91. or min(lats) < -91:
# raise ValueError,msg
x, y = m(lons, lats)
shpsegs.append(zip(x,y))
if ring == 0:
shapedict = dbf.read_record(npoly)
#print shapedict
name = shapedict["ID_DEPART"]
subpref_id = shapedict["ID_SP"]
# add information about ring number to dictionary.
shapedict['RINGNUM'] = ring+1
shapedict['SHAPENUM'] = npoly+1
shpinfo.append(shapedict)
#print subpref_id
#print name
lines = LineCollection(shpsegs,antialiaseds=(1,))
# lines.set_facecolors(col[subpref_id])
lines.set_edgecolors('k')
lines.set_linewidth(0.2)
ax.add_collection(lines)
if G.has_node(-1):
G.remove_node(-1)
if G.has_node(0):
G.remove_node(0)
max7s = 1
min7s = 10000000
for u,v,d in G.edges(data=True):
if d['weight'] > max7s:
max7s = d['weight']
if d['weight'] < min7s:
min7s = d['weight']
max7s = float(max7s)
print "max", (max7s)
print "min", (min7s)
scaled_weight = defaultdict(int)
for i in range(256):
scaled_weight[i] = defaultdict(int)
for u,v,d in G.edges(data=True):
scaled_weight[u][v] = (d['weight'] - min7s) / (max7s - min7s)
for u, v, d in G.edges(data=True):
# lo = []
# la = []
# lo.append(subpref_coord[u][0])
# lo.append(subpref_coord[v][0])
# la.append(subpref_coord[u][1])
# la.append(subpref_coord[v][1])
# x, y = m(lo, la)
# linewidth7s = scaled_weight[u][v] * 2.5 + 0.25
# m.plot(x,y, linewidth= linewidth7s)
# if linewidth7s > 1:
# print (linewidth7s)
linewidth7s = scaled_weight[u][v] * 2.5 + 0.25
m.drawgreatcircle(subpref_coord[u][0], subpref_coord[u][1], \
subpref_coord[v][0], subpref_coord[v][1], linewidth= linewidth7s, color='r')
m.drawcoastlines()
m.fillcontinents()
plt.savefig('/home/sscepano/Project7s/D4D/CI/urban_rural/home_work/OUTPUT_files/maps/hw_commuting_cur.png',dpi=700)
#plt.show()
###################################################################################################3
return
def map_commutes(G):
mpl.rcParams['font.size'] = 10.
mpl.rcParams['axes.labelsize'] = 8.
mpl.rcParams['xtick.labelsize'] = 6.
mpl.rcParams['ytick.labelsize'] = 6.
###########################################################################################
fig = plt.figure(3)
#Custom adjust of the subplots
plt.subplots_adjust(left=0.05,
right=0.95,
top=0.90,
bottom=0.05,
wspace=0.15,
hspace=0.05)
ax = plt.subplot(111)
m = Basemap(projection='cyl',\
llcrnrlon=-9, \
llcrnrlat=3.8, \
urcrnrlon=-1.5, \
urcrnrlat = 11, \
resolution = 'h', \
# projection = 'tmerc', \
lat_0 = 7.38, \
lon_0 = -5.30)
# read subpref coordinates
subpref_coord = read_in_subpref_lonlat()
shp = ShapeFile(
r'/home/sscepano/DATA SET7S/D4D/SubPrefecture/GEOM_SOUS_PREFECTURE')
dbf = dbflib.open(
r'/home/sscepano/DATA SET7S/D4D/SubPrefecture/GEOM_SOUS_PREFECTURE')
for npoly in range(shp.info()[0]):
shpsegs = []
shpinfo = []
shp_object = shp.read_object(npoly)
verts = shp_object.vertices()
rings = len(verts)
for ring in range(rings):
lons, lats = zip(*verts[ring])
# if max(lons) > 721. or min(lons) < -721. or max(lats) > 91. or min(lats) < -91:
# raise ValueError,msg
x, y = m(lons, lats)
shpsegs.append(zip(x, y))
if ring == 0:
shapedict = dbf.read_record(npoly)
#print shapedict
name = shapedict["ID_DEPART"]
subpref_id = shapedict["ID_SP"]
# add information about ring number to dictionary.
shapedict['RINGNUM'] = ring + 1
shapedict['SHAPENUM'] = npoly + 1
shpinfo.append(shapedict)
#print subpref_id
#print name
lines = LineCollection(shpsegs, antialiaseds=(1, ))
# lines.set_facecolors(col[subpref_id])
lines.set_edgecolors('k')
lines.set_linewidth(0.2)
ax.add_collection(lines)
if G.has_node(-1):
G.remove_node(-1)
if G.has_node(0):
G.remove_node(0)
###########################################################################################################
### this is the main part: scale weights of # commuters, use colormap by work location and then plot
### all or just the routes with more that 10 commutes
###########################################################################################################
max7s = 1
min7s = 10000000
for u, v, d in G.edges(data=True):
if d['weight'] > max7s:
max7s = d['weight']
if d['weight'] < min7s:
min7s = d['weight']
max7s = float(max7s)
print "max", (max7s)
print "min", (min7s)
scaled_weight = defaultdict(int)
for i in range(256):
scaled_weight[i] = defaultdict(int)
for u, v, d in G.edges(data=True):
scaled_weight[u][v] = (d['weight'] - min7s) / (max7s - min7s)
values = range(256)
jet = cm = plt.get_cmap('jet')
cNorm = colors.Normalize(vmin=0, vmax=values[-1])
scalarMap = cmx.ScalarMappable(norm=cNorm, cmap=jet)
for u, v, d in G.edges(data=True):
# lo = []
# la = []
# lo.append(subpref_coord[u][0])
# lo.append(subpref_coord[v][0])
# la.append(subpref_coord[u][1])
# la.append(subpref_coord[v][1])
# x, y = m(lo, la)
# linewidth7s = scaled_weight[u][v] * 2.5 + 0.25
# m.plot(x,y, linewidth= linewidth7s)
# if linewidth7s > 1:
# print (linewidth7s)
if d['weight'] >= 10:
colorVal = scalarMap.to_rgba(values[v])
linewidth7s = scaled_weight[u][v] * 2.5 + 0.35
###########################################################################################################
### no scaling vs scaling the width of the line
###########################################################################################################
# linewidth7s = d['weight'] / 10
m.drawgreatcircle(subpref_coord[u][0], subpref_coord[u][1], \
subpref_coord[v][0], subpref_coord[v][1], linewidth= linewidth7s, color=colorVal)
m.drawcoastlines()
#m.fillcontinents()
plt.savefig(
'/home/sscepano/Project7s/D4D/CI/urban_rural/home_work/OUTPUT_files/maps/hw_commuting_colored_by_work_gr10commuters.png',
dpi=700)
#plt.show()
###################################################################################################3
return
def shpReadShapeFile(shapeFile):
"""
Read in shape file, return vertices and information
Based on the mpl_toolkit.basemap.Basemap.readshapefile() function
Changes include removal of the optional kwargs to draw bounds on an
axes object, and no conversion from lat/lon coords to map (x/y)
coords.
Input: shapeFile - path to shapefile components
Output: coords - array of shapefile vertices or points (lon,lat)
attributes - list of dictionaries, one for each shape,
containing attributes of each shape from the corresponding
dbf file.
"""
try:
shp = shapelib.ShapeFile(shapeFile)
except:
raise IOError, "Error reading shapefile %s.shp" % shapeFile
try:
dbf = dbflib.open(shapeFile)
except:
raise IOError, "Error reading dbffile %s.dbf" % shapeFile
info = shp.info()
if info[1] not in [1,3,5,8]:
raise ValueError, "shpReadShapeFile can only handle 2D shape types"
msg = """Shapefile must have lat/lon vertices - it appears this one
has vertices in map projection coordinates. Convert the shapefile to
geographic coordinates using the shpproj utility from the shapelib
tools"""
if info[1] in [1,8]:
coords = []
nelem = shp.info()[0]
for elem in xrange(nelem):
shpObj = shp.read_object(elem)
verts = shpObj.vertices()
lons, lats = zip(*verts)
if max(lons) > 721. or min(lons) < -721. or max(lats) > 91. or \
min(lats) < -91.:
raise ValueError, msg
if len(verts) > 1:
coords.append(zip(lons, lats))
else:
coords.append((lons[0], lats[0]))
attributes = [dbf.read_record(i) for i in xrange(nelem)]
else:
coords = []
attributes = []
for npoly in xrange(shp.info()[0]):
shpObj = shp.read_object(npoly)
verts = shpObj.vertices()
rings = len(verts)
for ring in xrange(rings):
lons, lats = zip(*verts[ring])
if max(lons) > 721. or min(lons) < -721. or max(lats) > 91. or \
min(lats) < -91.:
raise ValueError, msg
coords.append(zip(lons, lats))
if ring == 0:
shapedict = dbf.read_record(npoly)
shapedict['RINGNUM'] = ring+1
shapedict['SHAPENUM'] = npoly+1
attributes.append(shapedict)
shp.close()
dbf.close()
return coords, attributes
""" Create table with xref between gssurgo values and file names """
import dbflib
import psycopg2
PGCONN = psycopg2.connect(database='idep', host='iemdb')
cursor = PGCONN.cursor()
cursor.execute("""DELETE from xref_surgo""")
dbf = dbflib.open('../../data/s/iaMU.tif.vat.dbf')
for i in range(dbf.record_count()):
row = dbf.read_record(i)
cursor.execute("""INSERT into xref_surgo(surgo, soilfile) VALUES
(%s, %s)""", (row['Value'], row['WEPP_SOL']))
cursor.close()
PGCONN.commit()
PGCONN.close()
def map_wake_hr(thersholdX):
mpl.rcParams['font.size'] = 4.4
###########################################################################################
fig = plt.figure(3)
#Custom adjust of the subplots
plt.subplots_adjust(left=0.05,right=0.95,top=0.90,bottom=0.05,wspace=0.15,hspace=0.05)
ax = plt.subplot(111)
m = Basemap(llcrnrlon=-9, \
llcrnrlat=3.8, \
urcrnrlon=-1.5, \
urcrnrlat = 11, \
resolution = 'h', \
projection = 'tmerc', \
lat_0 = 7.38, \
lon_0 = -5.30)
subpref_wake_hr = read_in_subpref_wake_hr(thersholdX)
# read the shapefile archive
s = m.readshapefile('/home/sscepano/DATA SET7S/D4D/SubPrefecture/GEOM_SOUS_PREFECTURE', 'subpref')
max7s = 1
min7s = 10000000
for subpref in subpref_wake_hr.keys():
if subpref_wake_hr[subpref] > max7s:
max7s = subpref_wake_hr[subpref]
if subpref_wake_hr[subpref] < min7s:
min7s = subpref_wake_hr[subpref]
max7s = float(max7s)
print "max", (max7s)
print "min", (min7s)
# scaled_weight = defaultdict(int)
# for i in range(256):
# scaled_weight[i] = defaultdict(int)
# for subpref in subpref_wake_hr.keys():
# scaled_weight[subpref] = (subpref_wake_hr[subpref] - min7s) / (max7s - min7s)
# values = range(256)
# jet = cm = plt.get_cmap('jet')
# cNorm = colors.Normalize(vmin=0, vmax=values[-1])
# scalarMap = cmx.ScalarMappable(norm=cNorm, cmap=jet)
# define custom colormap, white -> nicered, #E6072A = RGB(0.9,0.03,0.16)
cdict = {'red': ( (0.0, 1.0, 1.0),
(1.0, 0.9, 1.0) ),
'green':( (0.0, 1.0, 1.0),
(1.0, 0.03, 0.0) ),
'blue': ( (0.0, 1.0, 1.0),
(1.0, 0.16, 0.0) ) }
custom_map = LinearSegmentedColormap('custom_map', cdict, N=33)
plt.register_cmap(cmap=custom_map)
subpref_coord = read_in_subpref_lonlat()
shp = ShapeFile(r'/home/sscepano/DATA SET7S/D4D/SubPrefecture/GEOM_SOUS_PREFECTURE')
dbf = dbflib.open(r'/home/sscepano/DATA SET7S/D4D/SubPrefecture/GEOM_SOUS_PREFECTURE')
for npoly in range(shp.info()[0]):
shpsegs = []
shpinfo = []
shp_object = shp.read_object(npoly)
verts = shp_object.vertices()
rings = len(verts)
for ring in range(rings):
lons, lats = zip(*verts[ring])
# if max(lons) > 721. or min(lons) < -721. or max(lats) > 91. or min(lats) < -91:
# raise ValueError,msg
x, y = m(lons, lats)
shpsegs.append(zip(x,y))
if ring == 0:
shapedict = dbf.read_record(npoly)
#print shapedict
name = shapedict["ID_DEPART"]
subpref_id = shapedict["ID_SP"]
print name, subpref_id
# add information about ring number to dictionary.
shapedict['RINGNUM'] = ring+1
shapedict['SHAPENUM'] = npoly+1
shpinfo.append(shapedict)
#print subpref_id
#print name
lines = LineCollection(shpsegs,antialiaseds=(1,))
colorVal = custom_map(subpref_wake_hr[subpref_id])
# colorVal = scaled_weight[subpef]
lines.set_facecolors(colorVal)
lines.set_edgecolors('gray')
lines.set_linewidth(0.1)
ax.add_collection(lines)
# data to plot on the map
lons = []
lats = []
num = []
for subpref in subpref_wake_hr.iterkeys():
print(subpref)
if subpref <> 0 and subpref <> -1:
lons.append(subpref_coord[subpref][0])
lats.append(subpref_coord[subpref][1])
num.append(subpref_wake_hr[subpref])
x, y = m(lons, lats)
m.scatter(x, y, color='white')
for name, xc, yc in zip(num, x, y):
# draw the pref name in a yellow (shaded) box
plt.text(xc, yc, name)
plt.savefig("/home/sscepano/Project7s/D4D/CI/call_wakeup_sleep_hour/subpref_wake_" + str(thersholdX) + "pct.png",dpi=350)
def map_communities_and_commutes(G):
import networkx as nx
G_mod = nx.read_gml("/home/sscepano/D4D res/allstuff/User movements graphs/commuting patterns/1/total_commuting_G_10com_775_269_v2.gml")
col = [str]*256
for i in range(256):
col[i] = 'w'
for node in G_mod.nodes_iter(data=True):
#print node[1]['label']
col_gephi = node[1]['graphics']['fill']
while (len(col_gephi) < 7):
col_gephi += '0'
col[int(float(node[1]['label']))] = col_gephi
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.basemap import Basemap
import matplotlib as mpl
mpl.rcParams['font.size'] = 10.
mpl.rcParams['axes.labelsize'] = 8.
mpl.rcParams['xtick.labelsize'] = 6.
mpl.rcParams['ytick.labelsize'] = 6.
from shapelib import ShapeFile
import dbflib
from matplotlib.collections import LineCollection
from matplotlib import cm
###########################################################################################
fig = plt.figure(3)
#Custom adjust of the subplots
plt.subplots_adjust(left=0.05,right=0.95,top=0.90,bottom=0.05,wspace=0.15,hspace=0.05)
ax = plt.subplot(111)
m = Basemap(llcrnrlon=-9, \
llcrnrlat=3.8, \
urcrnrlon=-1.5, \
urcrnrlat = 11, \
resolution = 'h', \
projection = 'tmerc', \
lat_0 = 7.38, \
lon_0 = -5.30)
m.drawcoastlines()
shp = ShapeFile(r'/home/sscepano/DATA SET7S/D4D/SubPrefecture/GEOM_SOUS_PREFECTURE')
dbf = dbflib.open(r'/home/sscepano/DATA SET7S/D4D/SubPrefecture/GEOM_SOUS_PREFECTURE')
for npoly in range(shp.info()[0]):
shpsegs = []
shpinfo = []
shp_object = shp.read_object(npoly)
verts = shp_object.vertices()
rings = len(verts)
for ring in range(rings):
lons, lats = zip(*verts[ring])
# if max(lons) > 721. or min(lons) < -721. or max(lats) > 91. or min(lats) < -91:
# raise ValueError,msg
x, y = m(lons, lats)
shpsegs.append(zip(x,y))
if ring == 0:
shapedict = dbf.read_record(npoly)
#print shapedict
name = shapedict["ID_DEPART"]
subpref_id = shapedict["ID_SP"]
#color_col
# add information about ring number to dictionary.
shapedict['RINGNUM'] = ring+1
shapedict['SHAPENUM'] = npoly+1
shpinfo.append(shapedict)
#print subpref_id
#print name
lines = LineCollection(shpsegs,antialiaseds=(1,))
lines.set_facecolors(col[subpref_id])
lines.set_edgecolors('k')
lines.set_linewidth(0.3)
ax.add_collection(lines)
from collections import defaultdict
if G.has_node(-1):
G.remove_node(-1)
max7s = 1
min7s = 1
for u,v,d in G.edges(data=True):
if d['weight'] > max7s:
max7s = d['weight']
if d['weight'] < min7s:
min7s = d['weight']
max7s = float(max7s)
print max7s
print min7s
scaled_weight = defaultdict(int)
for i in range(256):
scaled_weight[i] = defaultdict(int)
for u,v,d in G.edges(data=True):
scaled_weight[u][v] = (d['weight'] - min7s) / (max7s - min7s)
for u, v, d in G.edges(data=True):
lo = []
la = []
lo.append(lons[u])
lo.append(lons[v])
la.append(lats[u])
la.append(lats[v])
x, y = m(lo, la)
linewidth7s = scaled_weight[u][v] * 6.5 + 0.15
m.plot(x,y, linewidth= linewidth7s)
if linewidth7s > 1:
print linewidth7s
plt.savefig('/home/sscepano/D4D res/allstuff/User movements graphs/commuting patterns/1/maps/mod_classes_10com_775_269_v2.png',dpi=1000)
#plt.show()
###################################################################################################3
return
def map_diversity():
#subpref_avg_fq = rd.read_in_subpref_avg_fq()
rg = get_scaled_rg()
fig = plt.figure(1)
#Custom adjust of the subplots
plt.subplots_adjust(left=0.05,right=0.95,top=0.90,bottom=0.05,wspace=0.15,hspace=0.05)
ax = plt.subplot(111)
m = Basemap(llcrnrlon=-9, \
llcrnrlat=3.8, \
urcrnrlon=-1.5, \
urcrnrlat = 11, \
resolution = 'h', \
projection = 'tmerc', \
lat_0 = 7.38, \
lon_0 = -5.30)
m.drawcoastlines()
from shapelib import ShapeFile
import dbflib
from matplotlib.collections import LineCollection
from matplotlib import cm
shp = ShapeFile(r'/home/sscepano/DATA SET7S/D4D/SubPrefecture/GEOM_SOUS_PREFECTURE')
dbf = dbflib.open(r'/home/sscepano/DATA SET7S/D4D/SubPrefecture/GEOM_SOUS_PREFECTURE')
msg = "Out of bounds"
color_col = []
for npoly in range(shp.info()[0]):
shpsegs = []
shpinfo = []
shp_object = shp.read_object(npoly)
verts = shp_object.vertices()
rings = len(verts)
for ring in range(rings):
lons, lats = zip(*verts[ring])
x, y = m(lons, lats)
shpsegs.append(zip(x,y))
if ring == 0:
shapedict = dbf.read_record(npoly)
#print shapedict
name = shapedict["ID_DEPART"]
subpref_id = shapedict["ID_SP"]
num = ["%.2f" % rg[subpref_id]]
# add information about ring number to dictionary.
shapedict['RINGNUM'] = ring+1
shapedict['SHAPENUM'] = npoly+1
shpinfo.append(shapedict)
#print subpref_id
#print name
lines = LineCollection(shpsegs,antialiaseds=(1,))
lines.set_facecolors(str(rg[subpref_id]))
lines.set_edgecolors('k')
lines.set_linewidth(0.3)
ax.add_collection(lines)
plt.savefig('/home/sscepano/D4D res/allstuff/diversity of travel/diversity_map.png',dpi=500)
return
def map_commutes(G):
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.basemap import Basemap
import matplotlib as mpl
mpl.rcParams['font.size'] = 10.
mpl.rcParams['axes.labelsize'] = 8.
mpl.rcParams['xtick.labelsize'] = 6.
mpl.rcParams['ytick.labelsize'] = 6.
from shapelib import ShapeFile
import dbflib
from matplotlib.collections import LineCollection
from matplotlib import cm
###########################################################################################
fig = plt.figure(3)
#Custom adjust of the subplots
plt.subplots_adjust(left=0.05,
right=0.95,
top=0.90,
bottom=0.05,
wspace=0.15,
hspace=0.05)
ax = plt.subplot(111)
m = Basemap(projection='cyl',\
llcrnrlon=-9, \
llcrnrlat=3.8, \
urcrnrlon=-1.5, \
urcrnrlat = 11, \
resolution = 'h', \
# projection = 'tmerc', \
lat_0 = 7.38, \
lon_0 = -5.30)
# read subpref coordinates
subpref_coord = read_subpref_lonlat()
shp = ShapeFile(
r'/home/sscepano/DATA SET7S/D4D/SubPrefecture/GEOM_SOUS_PREFECTURE')
dbf = dbflib.open(
r'/home/sscepano/DATA SET7S/D4D/SubPrefecture/GEOM_SOUS_PREFECTURE')
for npoly in range(shp.info()[0]):
shpsegs = []
shpinfo = []
shp_object = shp.read_object(npoly)
verts = shp_object.vertices()
rings = len(verts)
for ring in range(rings):
lons, lats = zip(*verts[ring])
# if max(lons) > 721. or min(lons) < -721. or max(lats) > 91. or min(lats) < -91:
# raise ValueError,msg
x, y = m(lons, lats)
shpsegs.append(zip(x, y))
if ring == 0:
shapedict = dbf.read_record(npoly)
#print shapedict
name = shapedict["ID_DEPART"]
subpref_id = shapedict["ID_SP"]
# add information about ring number to dictionary.
shapedict['RINGNUM'] = ring + 1
shapedict['SHAPENUM'] = npoly + 1
shpinfo.append(shapedict)
#print subpref_id
#print name
lines = LineCollection(shpsegs, antialiaseds=(1, ))
# lines.set_facecolors(col[subpref_id])
lines.set_edgecolors('k')
lines.set_linewidth(0.2)
ax.add_collection(lines)
if G.has_node(-1):
G.remove_node(-1)
if G.has_node(0):
G.remove_node(0)
max7s = 1
min7s = 10000000
for u, v, d in G.edges(data=True):
if d['weight'] > max7s:
max7s = d['weight']
if d['weight'] < min7s:
min7s = d['weight']
max7s = float(max7s)
print "max", (max7s)
print "min", (min7s)
scaled_weight = defaultdict(int)
for i in range(256):
scaled_weight[i] = defaultdict(int)
for u, v, d in G.edges(data=True):
scaled_weight[u][v] = (d['weight'] - min7s) / (max7s - min7s)
for u, v, d in G.edges(data=True):
# lo = []
# la = []
# lo.append(subpref_coord[u][0])
# lo.append(subpref_coord[v][0])
# la.append(subpref_coord[u][1])
# la.append(subpref_coord[v][1])
# x, y = m(lo, la)
# linewidth7s = scaled_weight[u][v] * 2.5 + 0.25
# m.plot(x,y, linewidth= linewidth7s)
# if linewidth7s > 1:
# print (linewidth7s)
linewidth7s = scaled_weight[u][v] * 2.5 + 0.25
m.drawgreatcircle(subpref_coord[u][0], subpref_coord[u][1], \
subpref_coord[v][0], subpref_coord[v][1], linewidth= linewidth7s, color='r')
m.drawcoastlines()
m.fillcontinents()
plt.savefig(
'/home/sscepano/Project7s/D4D/CI/urban_rural/home_work/OUTPUT_files/maps/hw_commuting_cur.png',
dpi=700)
#plt.show()
###################################################################################################3
return
def map_num_users():
mpl.rcParams['font.size'] = 4.4
###########################################################################################
fig = plt.figure(3)
#Custom adjust of the subplots
plt.subplots_adjust(left=0.05,
right=0.95,
top=0.90,
bottom=0.05,
wspace=0.15,
hspace=0.05)
ax = plt.subplot(111)
m = Basemap(llcrnrlon=-9, \
llcrnrlat=3.8, \
urcrnrlon=-1.5, \
urcrnrlat = 11, \
resolution = 'h', \
projection = 'tmerc', \
lat_0 = 7.38, \
lon_0 = -5.30)
subpref_users = read_in_subpref_num_users()
# read the shapefile archive
s = m.readshapefile(
'/home/sscepano/DATA SET7S/D4D/SubPrefecture/GEOM_SOUS_PREFECTURE',
'subpref')
max7s = 1
min7s = 10000000
for subpref in subpref_users.keys():
if subpref_users[subpref] > max7s:
max7s = subpref_users[subpref]
if subpref_users[subpref] < min7s:
min7s = subpref_users[subpref]
max7s = float(max7s)
print "max", (max7s)
print "min", (min7s)
scaled_weight = defaultdict(int)
for i in range(256):
scaled_weight[i] = defaultdict(int)
for subpref in subpref_users.keys():
scaled_weight[subpref] = (subpref_users[subpref] - min7s) / (max7s -
min7s)
# values = range(256)
# jet = cm = plt.get_cmap('jet')
# cNorm = colors.Normalize(vmin=0, vmax=values[-1])
# scalarMap = cmx.ScalarMappable(norm=cNorm, cmap=jet)
# define custom colormap, white -> nicered, #E6072A = RGB(0.9,0.03,0.16)
cdict = {
'red': ((0.0, 1.0, 1.0), (1.0, 0.9, 1.0)),
'green': ((0.0, 1.0, 1.0), (1.0, 0.03, 0.0)),
'blue': ((0.0, 1.0, 1.0), (1.0, 0.16, 0.0))
}
custom_map = LinearSegmentedColormap('custom_map', cdict, N=10000)
plt.register_cmap(cmap=custom_map)
subpref_coord = read_in_subpref_lonlat()
shp = ShapeFile(
r'/home/sscepano/DATA SET7S/D4D/SubPrefecture/GEOM_SOUS_PREFECTURE')
dbf = dbflib.open(
r'/home/sscepano/DATA SET7S/D4D/SubPrefecture/GEOM_SOUS_PREFECTURE')
for npoly in range(shp.info()[0]):
shpsegs = []
shpinfo = []
shp_object = shp.read_object(npoly)
verts = shp_object.vertices()
rings = len(verts)
#print shp_object
for ring in range(rings):
print ring
lons, lats = zip(*verts[ring])
# if max(lons) > 721. or min(lons) < -721. or max(lats) > 91. or min(lats) < -91:
# raise ValueError,msg
x, y = m(lons, lats)
shpsegs.append(zip(x, y))
if ring == 0:
shapedict = dbf.read_record(npoly)
print shapedict
name = shapedict["ID_DEPART"]
subpref_id = shapedict["ID_SP"]
# print name, subpref_id
# add information about ring number to dictionary.
shapedict['RINGNUM'] = ring + 1
shapedict['SHAPENUM'] = npoly + 1
shpinfo.append(shapedict)
#print subpref_id
#print name
lines = LineCollection(shpsegs, antialiaseds=(1, ))
colorVal = custom_map(subpref_users[subpref_id])
# colorVal = scaled_weight[subpef]
lines.set_facecolors(colorVal)
lines.set_edgecolors('gray')
lines.set_linewidth(0.1)
ax.add_collection(lines)
obama = percent['Obama, Barack']
fig = plt.figure(figsize=(12, 12))
ax = fig.add_axes([0.1, 0.1, 0.8, 0.8])
lllat = 21; urlat = 53; lllon = -118; urlon = -62
m = Basemap(ax=ax, projection='stere',
lon_0=(urlon + lllon) / 2, lat_0=(urlat + lllat) / 2,
llcrnrlat=lllat, urcrnrlat=urlat, llcrnrlon=lllon,
urcrnrlon=urlon, resolution='1')
m.drawcoastlines()
m.drawcountries()
shp = ShapeFile('../states/statesp020')
dbf = dbflib.open('../states/statesp020')
for npoly in range(shp.info()[0]):
# draw colored polygons on the map
shpseqs = []
shp_object = shp.read_object(npoly)
verts = shp_object.vertices()
rings = len(verts)
for ring in range(rings):
lons, lats = zip(*verts[ring])
x, y = m(lons,lats)
shpsegs.append(zip(x,y))
if ring == 0:
shapedict = dbf.read_record(npoly)
name = shapedict['STATE']
lines = LineCollection(shpsegs,antialiased=(1,))
def map_G(G):
import matplotlib.pyplot as plt
from mpl_toolkits.basemap import Basemap
import matplotlib.cm as cmx
import matplotlib as mpl
mpl.rcParams['font.size'] = 10.
mpl.rcParams['axes.labelsize'] = 8.
mpl.rcParams['xtick.labelsize'] = 6.
mpl.rcParams['ytick.labelsize'] = 6.
from shapelib import ShapeFile
import dbflib
from matplotlib.collections import LineCollection
from matplotlib import cm
import matplotlib.colors as colors
###########################################################################################
fig = plt.figure(3)
#Custom adjust of the subplots
plt.subplots_adjust(left=0.05,right=0.95,top=0.90,bottom=0.05,wspace=0.15,hspace=0.05)
ax = plt.subplot(111)
m = Basemap(projection='cyl',\
llcrnrlon=-9, \
llcrnrlat=3.8, \
urcrnrlon=-1.5, \
urcrnrlat = 11, \
resolution = 'h', \
# projection = 'tmerc', \
lat_0 = 7.38, \
lon_0 = -5.30)
# read subpref coordinates
subpref_coord = read_in_subpref_lonlat()
shp = ShapeFile(r'/home/sscepano/DATA SET7S/D4D/SubPrefecture/GEOM_SOUS_PREFECTURE')
dbf = dbflib.open(r'/home/sscepano/DATA SET7S/D4D/SubPrefecture/GEOM_SOUS_PREFECTURE')
for npoly in range(shp.info()[0]):
shpsegs = []
shpinfo = []
shp_object = shp.read_object(npoly)
verts = shp_object.vertices()
rings = len(verts)
for ring in range(rings):
lons, lats = zip(*verts[ring])
# if max(lons) > 721. or min(lons) < -721. or max(lats) > 91. or min(lats) < -91:
# raise ValueError,msg
x, y = m(lons, lats)
shpsegs.append(zip(x,y))
if ring == 0:
shapedict = dbf.read_record(npoly)
#print shapedict
name = shapedict["ID_DEPART"]
subpref_id = shapedict["ID_SP"]
# add information about ring number to dictionary.
shapedict['RINGNUM'] = ring+1
shapedict['SHAPENUM'] = npoly+1
shpinfo.append(shapedict)
#print subpref_id
#print name
lines = LineCollection(shpsegs,antialiaseds=(1,))
# lines.set_facecolors(col[subpref_id])
lines.set_edgecolors('gray')
lines.set_linewidth(0.1)
ax.add_collection(lines)
if G.has_node(-1):
G.remove_node(-1)
if G.has_node(0):
G.remove_node(0)
###########################################################################################################
### this is the main part: scale weights of # commuters, use colormap by work location and then plot
### all or just the routes with more that 10 commutes
###########################################################################################################
# max7s = 0
# min7s = 10000000
# for u,v,d in G.edges(data=True):
# if d['weight'] > max7s:
# max7s = d['weight']
# if d['weight'] < min7s:
# min7s = d['weight']
#
# max7s = float(max7s)
# print "max", (max7s)
# print "min", (min7s)
#
# scaled_weight = defaultdict(int)
#
# for i in range(len(G.edges())):
# scaled_weight[i] = defaultdict(int)
# for u,v,d in G.edges(data=True):
# scaled_weight[u][v] = (d['weight'] - min7s) / (max7s - min7s)
values = range(256)
jet = cm = plt.get_cmap('jet')
cNorm = colors.Normalize(vmin=0, vmax=values[-1])
scalarMap = cmx.ScalarMappable(norm=cNorm, cmap=jet)
for u, v, d in G.edges(data=True):
# lo = []
# la = []
# lo.append(subpref_coord[u][0])
# lo.append(subpref_coord[v][0])
# la.append(subpref_coord[u][1])
# la.append(subpref_coord[v][1])
# x, y = m(lo, la)
# linewidth7s = scaled_weight[u][v] * 2.5 + 0.25
# m.plot(x,y, linewidth= linewidth7s)
# if linewidth7s > 1:
# print (linewidth7s)
if d['weight'] >= 0:
colorVal = scalarMap.to_rgba(values[v])
# linewidth7s = scaled_weight[u][v] * 2.5 + 0.35
# print u, v, d['weight'], scaled_weight[u][v]
###########################################################################################################
### no scaling vs scaling the width of the line
###########################################################################################################
# linewidth7s = d['weight'] / 10
m.drawgreatcircle(subpref_coord[u][0], subpref_coord[u][1], \
subpref_coord[v][0], subpref_coord[v][1], linewidth= d['weight']*100, color=colorVal)
# m.drawcoastlines()
#m.fillcontinents()
plt.savefig('/home/sscepano/Project7s/D4D/CI/commuting_centers/Igor/high_betweenes_routes4.png',dpi=700)
#plt.show()
###################################################################################################3
return
def main(pickleName, renderCities, sieverts, uncovered):
try:
[npts, x, y, z, zraw, xil, yil, grid, missing] = cPickle.load(open(pickleName,'rb'))
lakes = cPickle.load(open(lakesPickle,'rb'))
if uncovered:
coastline = cPickle.load(open(coastlinePickle,'rb'))
print "Pickle file loaded (%d points)" % npts
except:
print "An error occurs loading the pickle datas !!!"
sys.exit(1)
if renderCities:
administrativeShapefile = "%s/JPN_adm2" % dataFolder
distanceAngle = 5
else:
administrativeShapefile = "%s/JPN_adm1" % dataFolder
distanceAngle = 45
if sieverts:
print "Rendering using uSv/h as unit"
renderCPM = False
else:
print "Rendering using CPM as unit"
renderCPM = True
# Load locations names and position
print "Load locations"
shp = ShapeFile(locationShapefile)
dbf = dbflib.open(locationShapefile)
lx = []
ly = []
lname = []
for name in range(shp.info()[0]):
shp_object = shp.read_object(name)
shp_dict = dbf.read_record(name)
cx , cy = shp_object.vertices()[0]
names = re.split('[()]', shp_dict["NAME"])
if len(names)>1:
cityname = names[1]
else:
cityname = ""
lx.append(cx)
ly.append(cy)
lname.append(cityname)
if uncovered:
# Compute all uncovered areas
difference = coastline.difference(missing)
missing = difference
# Create the thread pool
pool = Pool()
# Load Japan administrative area
shp = ShapeFile(administrativeShapefile)
dbf = dbflib.open(administrativeShapefile)
# Process every shape from the ShapeFile
print "Processing shapes ..."
for npoly in range(shp.info()[0]):
shpsegs = []
shpinfo = []
vx = []
vy = []
shp_object = shp.read_object(npoly)
shp_dict = dbf.read_record(npoly)
verts = shp_object.vertices()
# Start building the city map
if renderCities:
name = "%s" % (shp_dict["NAME_2"])
folder = "%s" % (shp_dict["NAME_1"])
else:
name = "%s" % (shp_dict["NAME_1"])
folder = "All"
#if (name not in ["Koriyama", "Fukushima", "Tokyo"]):
#if (folder not in ["Fukushima"]):
#if (shp_dict["NAME_1"] not in ["Fukushima", "Miyagi", "Tokyo", "Chiba", "Ibaraki", "Shizuoka", "Iwate", "Kanagawa"]):
# print "Skipping %s" % name
# continue
# Extract city polygon vertices
# Biggest ring only
biggestRing = 0
biggestRingSize = 0
for ring in verts:
if len(ring) > biggestRingSize:
biggestRingSize = len(ring)
biggestRing = ring
for point in biggestRing:
vx.append(point[0])
vy.append(point[1])
poly_verts = zip(vx,vy)
# Compute intersections with the city
from matplotlib.nxutils import points_inside_poly
points = np.vstack((x,y)).T
intersection = points_inside_poly(points, poly_verts)
# Compute a small statistics for the city
measures = []
for i in range(len(intersection)-1):
if (intersection[i]):
measures.append(zraw[i])
measures = np.array(measures)
npts, minCPM, maxCPM, medianCPM = (0,0,0,0)
npts = np.size(measures)
if npts:
minCPM = measures.min()
maxCPM = measures.max()
medianCPM = np.median(measures)
# Only keep cities information inside the region
cities_verts = zip(lx,ly)
cities_inside = points_inside_poly(cities_verts, poly_verts)
cities = [(lx[i], ly[i], lname[i]) for i in range(len(cities_inside)-1) if (cities_inside[i])]
if npts>0:
title = 'Safecast %s - Griddata (%s points) - %s [%s]\n(min, median, max) = (%d, %d, %d) CPM' % (safecastDatasetName, npts, name, folder, minCPM, medianCPM, maxCPM)
#DrawMap(title, 2, vx, vy, min(vx), max(vx), min(vy), max(vy), npts, x, y, z, xil, yil, grid, name, folder, cities, missing, uncovered, lakes)
pool.apply_async(DrawMap, (title, 2, vx, vy, min(vx), max(vx), min(vy), max(vy), npts, x, y, z, xil, yil, grid, name, folder, cities, missing, uncovered, lakes))
else:
print "Skipping %s" % name
# Wait the pool to be completed
pool.close()
pool.join()
""" Create table with xref between gssurgo values and file names """
import dbflib
from pyiem.util import get_dbconn
PGCONN = get_dbconn("idep")
cursor = PGCONN.cursor()
cursor.execute("""DELETE from xref_surgo""")
dbf = dbflib.open("../../data/s/iaMU.tif.vat.dbf")
for i in range(dbf.record_count()):
row = dbf.read_record(i)
cursor.execute(
"""INSERT into xref_surgo(surgo, soilfile) VALUES
(%s, %s)""",
(row["Value"], row["WEPP_SOL"]),
)
cursor.close()
PGCONN.commit()
PGCONN.close()
