def __call__(self, coord):
"""
Extracts the 21x21 sub-array from the the full VIIRS tile set
corresponding to the provided coordinates,
and returns a normalized 3D tensor.
Normalization:
output = ln(input + 1)/ln(max)
Where max = 92,000 radiance in our dataset.
Maps to [0,1] and reduces outsize influence of outliers.
Input:
coord (tuple of 2 floats)
Returns a 3D tensor.
"""
lon, lat = coord
min_lat, min_lon, max_lat, max_lon = create_space(lat, lon)
row, col = self.tif.index(min_lon, max_lat)
row -= self.row_offset
col -= self.col_offset
array = self.tif_data[:, row:row + 21, col:col + 21]
viirs_tensor = torch.tensor(array.reshape(
(-1, 21, 21))).type(torch.FloatTensor)
return torch.clamp(torch.log(viirs_tensor + 1) / 11.43, min=0, max=1)
def create_poly(r):
lat = r.cluster_lat
lon = r.cluster_lon
min_lat, min_lon, max_lat, max_lon = create_space(lat, lon)
points = [(min_lon, min_lat), (min_lon, max_lat), (max_lon, max_lat),
(max_lon, max_lat)]
return Polygon(points)
def __call__(self, coord, pathrow, img_urls):
"""
Extracts the 21x21 sub-array from the Landsat scene corresponding
to the provided coordinates, and returns a normalized 3D tensor.
Input:
coord (tuple of 2 floats)
pathrow (str)
img_urls: tuple of urls to R, G, and B TIF bands to a single scene
Returns a 3D tensor.
"""
r_url, b_url, g_url = img_urls
lon, lat = coord
# If we already have the 3d tensor, just return it
if coord in self.coord_imgs:
return self.coord_imgs[coord].new_tensor()
else:
# Get TIF with 3 bands
tif = rasterio.open(self._get_tif(img_urls, pathrow), 'r')
# Extract 224x224 subarray from landsat scene
min_lat, min_lon, max_lat, max_lon = create_space(lat, lon)
utm = pyproj.Proj(tif.crs)
lonlat = pyproj.Proj(init='epsg:4326')
east, north = pyproj.transform(lonlat, utm, max_lon, max_lat)
west, south = pyproj.transform(lonlat, utm, min_lon, min_lat)
north_idx, west_idx = tif.index(west, north)
south_idx, east_idx = tif.index(east, south)
raw_array = tif.read(window=Window(
west_idx, north_idx, abs(west_idx -
east_idx), abs(north_idx - south_idx)))
tif.close()
# Convert array to tensor
landsat_tensor = torch.tensor(raw_array).type(torch.FloatTensor)
# resize tensor
landsat_tensor = TF.resize(landsat_tensor, size=(self.ydim, self.xdim))
# Add tensor to dict
self.coord_imgs[coord] = landsat_tensor
return landsat_tensor.new_tensor()
def generate_download_locations(df, ipc=50):
'''
Takes a dataframe with columns cluster_lat, cluster_lon
Generates a 10km x 10km bounding box around the cluster and samples
ipc images per cluster. First samples in a grid fashion, then any
remaining points are randomly (uniformly) chosen
'''
np.random.seed(RANDOM_SEED) # for reproducability
df_download = {'image_name': [], 'image_lat': [], 'image_lon': []}
for c in df.columns:
df_download[c] = []
# side length of square for uniform distribution
edge_num = math.floor(math.sqrt(ipc))
for _, r in df.iterrows():
min_lat, min_lon, max_lat, max_lon = create_space(r.cluster_lat, r.cluster_lon)
lats = np.linspace(min_lat, max_lat, edge_num).tolist()
lons = np.linspace(min_lon, max_lon, edge_num).tolist()
# performs cartesian product
uniform_points = np.transpose([np.tile(lats, len(lons)), np.repeat(lons, len(lats))])
lats = uniform_points[:,0].tolist()
lons = uniform_points[:,1].tolist()
# fills the remainder with random points
for _ in range(ipc - edge_num * edge_num):
lat = np.random.uniform(min_lat, max_lat)
lon = np.random.uniform(min_lon, max_lon)
lats.append(lat)
lons.append(lon)
# add to dict
for lat, lon in zip(lats, lons):
# image name is going to be image_lat_image_lon_cluster_lat_cluster_lon.png
image_name = str(lat) + '_' + str(lon) + '_' + str(r.cluster_lat) + '_' + str(r.cluster_lon) + '.png'
df_download['image_name'].append(image_name)
df_download['image_lat'].append(lat)
df_download['image_lon'].append(lon)
for c in df.columns:
df_download[c].append(r[c])
return pd.DataFrame.from_dict(df_download)
def add_nightlights(df):
'''
This takes a dataframe with columns cluster_lat, cluster_lon and finds the average
nightlights in 2015 using a 10km x 10km box around the point
'''
return
tif_array = None
print('loading tif...')
if COUNTRY == 'malawi_2016':
print('loading tif...')
tif_array = np.squeeze(tifs[0].get_data())
add_nightlights(df_c, TIFS[0], tif_array)
elif COUNTRY == 'ethiopia_2015':
print('loading tif...')
tif_array = np.squeeze(tifs[1].get_data())
add_nightlights(df_c, TIFS[1], tif_array)
else:
raise ValueError('Unrecognized country')
cluster_nightlights = []
for i, r in df.iterrows():
min_lat, min_lon, max_lat, max_lon = create_space(
r.cluster_lat, r.cluster_lon)
xminPixel, ymaxPixel = tif.proj_to_raster(min_lon, min_lat)
xmaxPixel, yminPixel = tif.proj_to_raster(max_lon, max_lat)
assert xminPixel < xmaxPixel, print(r.cluster_lat, r.cluster_lon)
assert yminPixel < ymaxPixel, print(r.cluster_lat, r.cluster_lon)
if xminPixel < 0 or xmaxPixel >= tif_array.shape[1]:
print(f"no match for {r.cluster_lat}, {r.cluster_lon}")
raise ValueError()
elif yminPixel < 0 or ymaxPixel >= tif_array.shape[0]:
print(f"no match for {r.cluster_lat}, {r.cluster_lon}")
raise ValueError()
xminPixel, yminPixel, xmaxPixel, ymaxPixel = int(xminPixel), int(
yminPixel), int(xmaxPixel), int(ymaxPixel)
cluster_nightlights.append(tif_array[yminPixel:ymaxPixel,
xminPixel:xmaxPixel].mean())
df['nightlights'] = cluster_nightlights
return
def __call__(self, coord, country):
"""
Extracts the 21x21 sub-array from the the full VIIRS tile set
corresponding to the provided coordinates,
and returns a normalized 3D tensor.
Normalization:
output = ln(input + 1)/ln(max)
Where max = 92,000 radiance in our dataset.
Maps to [0,1] and reduces outsize influence of outliers.
Input:
coord (tuple of 2 floats)
country (str): One of ['eth', 'mw', 'ng']
Returns a 3D tensor.
"""
min_lat, min_lon, max_lat, max_lon = create_space(
coord[0], coord[1])
xminPixel, ymaxPixel = self.tifs[country].proj_to_raster(min_lon, min_lat)
xminPixel, ymaxPixel = int(xminPixel), int(ymaxPixel)
array = self.arrays[country][:, ymaxPixel-21:ymaxPixel, xminPixel:xminPixel+21]
viirs_tensor = torch.tensor(array.reshape((-1,21,21))).type(torch.FloatTensor)
return torch.log(viirs_tensor + 1) / 11.43