def run(f):
## track='data\\2420.csv'
## df = pd.read_csv(track)
## for track, trackdf in df.groupby("track"):
## trackdf.to_csv('data\\'+str(track)+".csv")
# df = pd.read_csv('data\\332541.csv')
# layer = 'results\\sampleLandUse.shp'
# land = gpd.GeoDataFrame.from_file(layer)
# track = df[['X','Y']]
# Crowd(track, land, layer)
# repo_dir = Path('__file__').parents[0]
# data_dir = repo_dir / 'data'
# layer = 'sampleLandUse.shp'
# land = gpd.GeoDataFrame.from_file(layer)
land = 'landUse.tif'
NDV, xsize, ysize, GeoT, Projection, DataType = gr.get_geo_info(land)
df = pd.read_csv(f)
numHome = 0
for index, row in df.iterrows():
if row['purpose'] == 'home': #or row['purto']=='home':
numHome += 1
track = df[['X', 'Y']]
start = time.clock()
home = Crowd(track, land, numHome, k=10, p=0.02, lag=50)
speed = time.clock() - start
return home, speed
def nightlights_values(self,
df,
lon_col='gpsLongitude',
lat_col='gpsLatitude'):
"""
Given a dataset with latitude and longitude columns, it returns the nightlight value at each point.
:param df: DataFrame
:param lon_col: column names for longitude
:param lat_col: column name of latitude
:return: Series
"""
import georasters as gr
try:
pop = gr.load_tiff(self.file)
except MemoryError:
print('Landuse Raster too big!')
raise
# Find location of point (x,y) on raster, e.g. to extract info at that location
NDV, xsize, ysize, GeoT, Projection, DataType = gr.get_geo_info(
self.file)
def lu_extract(row):
try:
c, r = gr.map_pixel(row[lon_col], row[lat_col], GeoT[1],
GeoT[-1], GeoT[0], GeoT[3])
lu = pop[c, r]
return lu
except IndexError:
pass
return df.apply(lu_extract, axis=1)
def test_get_dims(self):
DATA = "~/Downloads/datasets/elevation/viewfinder_dem3/15-J.tif" # from http://www.viewfinderpanoramas.org/dem3.html
DATA = os.path.expanduser(DATA)
data = gr.from_file(DATA)
print(data.geot)
NDV, xsize, ysize, GeoT, Projection, DataType = gr.get_geo_info(DATA)
print(NDV, xsize, ysize, GeoT, DataType)
def open(self, file=None):
"""Does what it says."""
self.ndv, self.xsize, self.ysize, self.geot, self.projection, datatype = georasters.get_geo_info(
file)
if self.ndv is None:
self.ndv = -99999
self.array = gdalnumeric.LoadFile(file)
self.y_cell_size = self.geot[1]
self.x_cell_size = self.geot[5]
self.array = numpy.ma.masked_array(self.array,
mask=self.array == self.ndv,
fill_value=self.ndv)
def open(self, file=None, dtype=None):
"""
Open a local file handle for reading and assignment
:param file:
:return: None
"""
# args[0]/file=
if file is None:
raise IndexError("invalid file= argument provided")
# grab raster meta information from GeoRasters
try:
self.ndv, self.x_cell_size, self.y_cell_size, self.geot, self.projection, self.dtype = \
get_geo_info(file)
except Exception:
raise AttributeError("problem processing file input -- is this",
"a raster file?")
# args[1]/dtype=
if dtype is not None:
# override our shadow'd value from GeoRasters if
# something was specified by the user
self.dtype = dtype
# re-cast our datatype as a numpy type, if needed
if type(self.dtype) == str:
self.dtype = NUMPY_TYPES[self.dtype.lower()]
if self.ndv is None:
self.ndv = _DEFAULT_NA_VALUE
# low-level call to gdal with explicit type specification
# that will store in memory or as a disc cache, depending
# on the state of our _using_disc_caching property
if self._using_disc_caching is not None:
# create a cache file
self.array = np.memmap(self._using_disc_caching,
dtype=dtype,
mode='w+',
shape=(_x_size, _y_size))
# load file contents into the cache
self.array[:] = gdalnumeric.LoadFile(
filename=self.filename,
buf_type=gdal_array.NumericTypeCodeToGDALTypeCode(
self.dtype))[:]
# by default, load the whole file into memory
else:
self.array = gdalnumeric.LoadFile(
filename=self.filename,
buf_type=gdal_array.NumericTypeCodeToGDALTypeCode(self.dtype))
# make sure we honor our no data value
self.array = np.ma.masked_array(self.array,
mask=self.array == self.ndv,
fill_value=self.ndv)
def getRastervalue(df, pop_raster, lat_col="gpsLatitude", lon_col="gpsLongitude", filter=1):
"""
when you pass dataframe with Lat, Long coordinates
it returns a vector of the corresponding population value at theses locations
It merges on the closest coordinates between the raster and the dataset.
Using the WorldPop Populaiton layers: http://www.worldpop.org.uk/data/data_sources/
use: data = getRastervalue(data,path_to_raster)
Parameters
----------
df : dataframe
pop_raster : string
filapath to the population raster
lat_col, lon_col : str
column names for the coordinates
filter : what treshold to consider valid population.
"""
print('-> finding landuse for {} points'.format(df.shape[0]))
import georasters as gr
try:
pop = gr.load_tiff(pop_raster)
except MemoryError:
print('Landuse Raster too big!')
raise
# Find location of point (x,y) on raster, e.g. to extract info at that location
NDV, xsize, ysize, GeoT, Projection, DataType = gr.get_geo_info(pop_raster)
def lu_extract(row):
try:
c, r = gr.map_pixel(row[lon_col], row[lat_col], GeoT[1], GeoT[-1], GeoT[0], GeoT[3])
lu = pop[c, r]
return lu
except IndexError:
pass
df['landuse'] = df.apply(lu_extract, axis=1)
# filter on population densities greater than filter
df = df[df.landuse > filter]
return df
def main():
# DATA = "../data/relief_san_andres.tif"
DATA = "~/Downloads/datasets/elevation/viewfinder_dem3/15-J.tif" # from http://www.viewfinderpanoramas.org/dem3.html
DATA = os.path.expanduser(DATA)
data = gr.from_file(DATA)
(xmin, xsize, x, ymax, y, ysize) = data.geot
print(data.geot)
NDV, xsize, ysize, GeoT, Projection, DataType = gr.get_geo_info(DATA)
print(NDV, xsize, ysize, GeoT, DataType)
print(Projection)
# ok, when looking again it looks like the max and min are the edges + half the difference.
# I might just try indexing for regions that are a multiple of the dimensions into the raster.
# find top coords of grid cell in map for lat/long
dx = 1.0 # in degrees
dy = 1.0 # in degrees
nw_corner = (round_to_nearest(xmin, dx), round_to_nearest(ymax, dy))
print(nw_corner)
se_corner = (round_to_nearest(nw_corner[0]+dx, dx), round_to_nearest(nw_corner[1]-dy, dy))
print(se_corner)
# data.plot()
# plt.show()
# get the array indexes for the map
print(type(data.raster))
print(data.raster.shape)
print(data.raster)
print(GeoT)
x_indexes = int(dx / GeoT[1])
y_indexes = int(dy / -GeoT[5])
print(x_indexes, y_indexes)
# determine the desired final raster size.
# wait, i want to figure out how to divide this up to give each chunk its own list of data to take stats on
# so I need to determine the next chunk. Or just iterate through the whole damn image and append the values to
# a dict for that chunk
# yeah let's do that, it's easy.
# lol never mind just get the map pixels for the corners and iterate over them
# col, row = gr.map_pixel(x,y,GeoT[1],GeoT[-1], GeoT[0],GeoT[3])
# col, row = gr.map_pixel()
print(data.map_pixel_location(13, 13))
row, col = data.map_pixel_location(13,13)
print(row, col)
T2 = r"./output/inyo_bcets/20140225_mojave_bcet/20140225_mojave_bcet_B1.tif"
T3 = r"./output/inyo_bcets/20151026_mojave_bcet/20151026_mojave_bcet_B1.tif"
T4 = r"./output/inyo_bcets/20160302_mojave_bcet/20160302_mojave_bcet_B1.tif"
T5 = r"./output/inyo_bcets/20160926_mojave_bcet/20160926_mojave_bcet_B1.tif"
T6 = r"./output/inyo_bcets/20170727_mojave_bcet/20170727_mojave_bcet_B1.tif"
T7 = r"./output/inyo_bcets/20170913_mojave_bcet/20170913_mojave_bcet_B1.tif"
T1_gr = gr.from_file(T1)
T2_gr = gr.from_file(T2)
T3_gr = gr.from_file(T3)
T4_gr = gr.from_file(T4)
T5_gr = gr.from_file(T5)
T6_gr = gr.from_file(T6)
T7_gr = gr.from_file(T7)
ndv, xsize, ysize, geot, projection, datatype = gr.get_geo_info(T1) # Raster information
Tstack = np.dstack((T1_gr.raster,T2_gr.raster,T3_gr.raster,T4_gr.raster,T5_gr.raster,T6_gr.raster,T7_gr.raster))
def difference(x):
return np.max(x)-np.min(x)
Rdiff = np.apply_along_axis(difference,axis=2, arr=Tstack)
output_gr = gr.GeoRaster(Rdiff,
T1_gr.geot,
nodata_value=ndv,
projection=T1_gr.projection,
datatype=T1_gr.datatype)
output_gr.to_tiff('./output/diff')
def open(self):
"""
Read a raster file from disc as a formatted numpy array
:rval none:
"""
# grab raster meta informationP from GeoRasters
try:
self.ndv, self.x_size, self.y_size, self.geot, self.projection, self.dtype = get_geo_info(
self.filename)
except Exception:
raise AttributeError("problem processing file input -- is this" +
" a raster file?")
# call gdal with explicit type specification
# that will store in memory or as a disc cache, depending
# on the state of our use_disc_caching property
if self.use_disc_caching is True:
# create a cache file and load file contents into the cache
_cached_file = NdArrayDiscCache(
input=self.filename,
dtype=self.dtype,
x_size=self.x_size,
y_size=self.y_size,
)
self.array = _cached_file.array
self.disc_cache_file = _cached_file.disc_cache_file
# by default, load the whole file into memory
else:
self.array = gdalnumeric.LoadFile(
filename=self.filename,
buf_type=gdal_array.NumericTypeCodeToGDALTypeCode(
_to_numpy_type(self.dtype)),
)
# make sure we honor our no data value
self.array = np.ma.masked_array(self.array,
mask=self.array == self.ndv,
fill_value=self.ndv)
#coding=utf-8
# @author Oleg Urzhumtcev aka NetBUG
# @desc Fiddling around GeoTIFF
import georasters as gr
import numpy as np
import matplotlib.pyplot as plt
if __name__ == '__main__':
pass
# Load data
#raster = './data/slope.tif' #nasa-worldview-2016-07-09.tiff'
raster = './data/nasa-worldview-2016-07-20.tiff'
data = gr.from_file(raster)
NDV, xsize, ysize, GeoT, Projection, DataType = gr.get_geo_info(raster)
print("NDV: %s; size: %dx%d; GeoT: %s; Projection: %s; Data: %s" %
(NDV, xsize, ysize, GeoT, Projection, DataType))
print()
x = 101.45
y = 66
#col, row = gr.map_pixel(x,y,GeoT[1],GeoT[-1], GeoT[0],GeoT[3])
#value = data[0,row,col]
#print("%d, %d: %s" % (col, row, repr(value)))
# Plot data
#print(data)
#plt.figure(figsize=(12, 8))
data.raster = data.raster[0]
data.plot()
plt.show()
def watershed(flow_dir_file, lake_data, val=47):
"""
__orig_author__ = Rdebbout
Description
----------
given a flow direction grid, develop tif representing lake watershed
Parameters
----------
flow_dir_file : grid file representing flow direction. DIR file within HydroSHEDS
lake_data : object representing lake information including id and pour point lat/lon
val : value to set watershed cells
Output
---------
tif file of a lake's watershed including value attribute table
"""
#get information about original flow direction raster
#NDV = no data values , xysize= raster size , tf = geotransform
NDV, xsize, ysize, tf, Projection, DataType = gr.get_geo_info(
flow_dir_file)
#print (tf)
#load into georaster from source file
arr = gr.from_file(flow_dir_file)
(xmin, xsize, x, ymax, y, ysize) = arr.geot
#get pour point lon and lat from lake object
pp_lon, pp_lat = lake_data.pour_pt_lon, lake_data.pour_pt_lat
# Function to map location in pixel of raster array: https://github.com/ozak/georasters/blob/master/georasters/georasters.py
here = gr.map_pixel(pp_lon, pp_lat, xsize, ysize, xmin, ymax)
init = []
init = ring_cells(here)
hold = []
temp_hold = eval_ring(init, arr)
hold = temp_hold
#Altered this to deal with an issue when iterating across multiple lakes in original code
#for every cell in ring evaluate if it is flowing to the pour point, if so add it to the hold, temp_hold continues until exhausted
while len(temp_hold) > 0:
for h in temp_hold:
init = ring_cells(h)
temp_hold = eval_ring(init, arr)
hold += temp_hold
#Add target cell
hold.append(here)
hold = list(set(hold))
# try to update transform info(tf) and shrink array
dd = np.array(hold)
r_min, col_min = np.amin(dd, axis=0)
r_max, col_max = np.amax(dd, axis=0)
#added step for WGS84 data that is not projected
lon, lat = gr.map_pixel_inv(r_min, col_min, tf[1], tf[-1], tf[0], tf[3])
# shift transform to reduce NoData in the output raster
shifted_tf = (lon, tf[1], tf[2], lat, tf[4], tf[5])
# insert val into the cells of the raster that represent watershed
out = np.zeros(arr.shape)
for idx in hold:
out[idx] = val
#plus 1 deals with index starting at 0, these are counts of rows, columns
new = out[r_min:(r_max + 1), col_min:(col_max + 1)]
go = gr.GeoRaster(new, shifted_tf, 0)
go.projection = Projection
go.datatype = DataType
#Set name of files and create
lk_id = str(lake_data.id)
#create tif file
go.to_tiff(f"output/lkshed/{lk_id}_watershed")
df = make_rat(f"output/lkshed/{lk_id}_watershed.tif")
#export value attribute table
df2dbf(df, f"output/lkshed/{lk_id}_watershed.tif.vat.dbf")
# #cleanup to help take care of issues on iterative processing
# arr = 0
go = None
here = None
arr = None
out = None
new = None
dd = None
shifted_tf = None
gc.collect()
table_y['SOILCLASS'] = table_y['TAXNWRB.f'].apply(lambda x: x.split(" ")[1])
table = table_y.drop(labels=['TIMESTRR'], axis=1).drop_duplicates()
table_y = table_y[['SOILCLASS', 'LOC_ID', 'LONWGS84',
'LATWGS84']].drop_duplicates()
soil_class_encoder = sklearn.preprocessing.LabelEncoder()
print("A total of " + repr(len(table_y)) +
" instances with information on soil classes are available...")
print("The data instances refer to a total of " +
repr(table_y['LOC_ID'].nunique()) + " unique locations...")
print("The data instances contain " +
repr(table_y['LOC_ID'].nunique() - table['LOC_ID'].nunique()) +
" locations with different classes at different time instants...")
print("Reading information on land coverage...")
table_y['LANDCOV'] = str("210_")
NDV, xsize, ysize, GeoT, Projection, DataType = gr.get_geo_info(
"./globcover/GLOBCOVER_L4_200901_200912_V2.3.tif")
raster_aux = dict()
table = gr.from_file("./globcover/GLOBCOVER_L4_200901_200912_V2.3.tif")
cnt = 0
for index, row in table_y.iterrows():
try:
val = table.map_pixel(row['LONWGS84'], row['LATWGS84'])
table_y.loc[index, 'LANDCOV'] = str(val) + "_"
cnt += 1
l = table.map_pixel_location(row['LONWGS84'], row['LATWGS84'])
raster_aux[(l[0], l[1])] = table_y.loc[index, 'LANDCOV']
except:
table_y.loc[index, 'LANDCOV'] = str("210_")
land_coverage_encoder = sklearn.preprocessing.OneHotEncoder()
print("Added land coverage information to " + repr(cnt) +
" instances out of " + repr(len(table_y)) + " data instances...")
'../HRM/HRM/Data/datasets/WFP_ENSAN_Senegal_2013_cluster.csv',
'../HRM/HRM/Data/datasets/WB_Uganda_2011_cluster.csv'],
['FCS_mean',
'cons']):
dataset = pd.read_csv(filename)
try:
dataset = dataset[dataset.cons <= 5]
except AttributeError:
dataset = dataset[dataset.FCS_mean <= 30]
import georasters as gr
nightlights = 'data/nightlights.tif'
esa = gr.load_tiff(nightlights)
# Find location of point (x,y) on raster, e.g. to extract info at that location
NDV, xsize, ysize, GeoT, Projection, DataType = gr.get_geo_info(nightlights)
def lu_extract(row):
try:
c, r = gr.map_pixel(row['gpsLongitude'], row['gpsLatitude'], GeoT[1], GeoT[-1], GeoT[0], GeoT[3])
lu = esa[c, r]
return lu
except IndexError:
print('coordinates {} {} at sea!'.format(row['gpsLongitude'], row['gpsLatitude']))
dataset['nightlights'] = dataset.apply(lu_extract, axis=1)
print('file {} correlates {} with nightlights.'.format(
filename, np.round(dataset[indicator].corr(dataset['nightlights']), 2)))
