def download_landsat_scene(url, directory, filename):
'''
This method downloads a scene directly from usgs. In order to do so, it
pretends to be a browser to build a request that is accepted by the server.
We added the headers so we don't get banned when the server detects that we
are doing lots of requests. This idea is based on the landsat downloader:
https://github.com/olivierhagolle/LANDSAT-Download
'''
cookies = urllib2.HTTPCookieProcessor()
opener = urllib2.build_opener(cookies)
urllib2.install_opener(opener)
data = urllib2.urlopen("https://ers.cr.usgs.gov").read()
token_group = re.search(r'<input .*?name="csrf_token".*?value="(.*?)"',
data)
if token_group:
token = token_group.group(1)
else:
LOGGER.error('The cross site request forgery token was not found.')
sys.exit(1)
usgs = {
'account': getattr(SETTINGS, 'USGS_USER'),
'passwd': getattr(SETTINGS, 'USGS_PASSWORD')
}
params = urllib.urlencode(
dict(username=usgs['account'],
password=usgs['passwd'],
csrf_token=token))
request = urllib2.Request(
"https://ers.cr.usgs.gov/login",
params,
headers={
'User-Agent':
'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.7) Gecko/2009021910 Firefox/3.0.7'
})
f = urllib2.urlopen(request)
data = f.read()
f.close()
download_chunks(url, directory, filename)
def download_chunks(scene_url, directory, filename):
'''
This method downloads a file in chunks.
'''
req = urllib2.urlopen(scene_url)
downloaded = 0
CHUNK = 1024 * 1024 * 8
total_size = int(req.info().getheader('Content-Length').strip())
total_size_fmt = size_of_fmt(total_size)
filepath = create_filename(directory, filename)
with open(filepath, 'wb') as fp:
start = time.clock()
LOGGER.info('Downloading %s %s:' % (filename, total_size_fmt))
while True:
chunk = req.read(CHUNK)
downloaded += len(chunk)
done = int(50 * downloaded / total_size)
sys.stdout.write('\r[{1}{2}]{0:3.0f}% {3}ps'.format(
math.floor((float(downloaded) / total_size) * 100), '=' * done,
' ' * (50 - done),
size_of_fmt((downloaded // (time.clock() - start)) / 8)))
sys.stdout.flush()
if not chunk: break
fp.write(chunk)
def calculate_top_of_atmosphere_spot6(self):
'''
Calculates the top of atmosphere for the image that is object represents.
'''
LOGGER.info('folder correctly identified')
LOGGER.info("Starting DN to TOA")
LOGGER.info("Start folder: %s" % self.path)
LOGGER.info('calculating TOA')
self.number_of_bands = self.get_raster().get_attribute(raster.DATA_SHAPE)[2]
metadata_band_order = self.get_sensor().get_attribute(spot6.BAND_DISPLAY_ORDER)
image_band_order = self.get_sensor().get_attribute(spot6.BAND_INDEX)
LOGGER.debug('band metadata order: %s' % self.get_sensor().get_attribute(spot6.BAND_DISPLAY_ORDER))
LOGGER.debug('band image order: %s' % self.get_sensor().get_attribute(spot6.BAND_INDEX))
band_order_reference = [u'B0', u'B1', u'B2', u'B3']
if image_band_order == metadata_band_order and metadata_band_order != band_order_reference:
band_solar_irradiance = self.get_sensor().get_attribute(spot6.BAND_SOLAR_IRRADIANCE_VALUE)
band_solar_irradiance = list(numpy.array(band_solar_irradiance)[map(lambda x: metadata_band_order.index(x), band_order_reference)])
LOGGER.info('band_solar_irradiance: %s' % band_solar_irradiance)
gain = map(float, self.get_sensor().get_attribute(spot6.PHYSICAL_GAIN))
offset = map(float, self.get_sensor().get_attribute(spot6.PHYSICAL_BIAS))
gain = list(numpy.array(gain)[map(lambda x: metadata_band_order.index(x), band_order_reference)])
offset = list(numpy.array(offset)[map(lambda x: metadata_band_order.index(x), band_order_reference)])
LOGGER.info('gain: %s' % gain)
LOGGER.info('offset: %s' % offset)
LOGGER.info('reading data_array')
data_array = self.get_raster().read_data_file_as_array()
sun_elevation = numpy.deg2rad(float(numpy.median(self.get_sensor().get_attribute(spot6.SUN_ELEVATION))))
LOGGER.debug('sun_elevation: %s' % sun_elevation)
imaging_date = datetime.date(self.sensor.get_attribute(spot6.ACQUISITION_DATE))
self.toa = calculate_rad_toa_spot6(data_array, gain, offset, imaging_date, sun_elevation, band_solar_irradiance, self.number_of_bands)
LOGGER.info('finished TOA')
LOGGER.info('exporting to tif')
outname = re.sub(r'.JP2', '', self.file_dictionary[self.get_image_file()]) + '_TOA.TIF'
LOGGER.info('Results of folder %s is %s' % (self.path, outname))
data_file = self.get_raster().create_from_reference(outname, self.toa.shape[2], self.toa.shape[1], self.toa.shape[0], self.get_raster().get_attribute(raster.GEOTRANSFORM), self.get_raster().get_attribute(raster.PROJECTION))
self.get_raster().write_raster(data_file, self.toa)
data_file = None
LOGGER.info('finished export')
LOGGER.info('finished DN to TOA')
def extract_extremes(data, image_file, make_plot, steps=1000):
# TODO: the two for and the last one takes too long to finish
CENTER = 50
xtiles = 5000 / steps
ytiles = 5000 / steps
counter = 0
b = 0
global_quant = list()
for ycount in range(0, 5000, steps):
for xcount in range(0, 5000, steps):
subset = data[:, ycount:ycount + steps, xcount:xcount + steps]
LOGGER.info("Extent: %dx%d" % (xcount, ycount))
z, y, x = subset.shape
rgb = numpy.zeros((y, x, z))
for i in range(0, z):
rgb[:, :, i] = subset[i, :, :] / numpy.max(data[i, :, :])
col_space1 = color.rgb2lab(rgb[:, :, 0:3])
quant = calculate_quantiles(col_space1[ :, :, 0])
if len(quant) > 0:
points = calculate_breaking_points(quant)
break_points = numpy.array(calculate_continuity(points.keys()))
else:
break_points = []
subset = data[b, ycount:ycount + steps, xcount:xcount + steps]
if make_plot:
pylab.subplot(xtiles, ytiles, counter + 1)
fig = pylab.plot(quant, c="k", alpha=0.5)
if len(break_points) > 1:
if make_plot:
for point in points.keys():
pylab.plot(point, numpy.array(quant)[point], 'r.')
if len(break_points[break_points < CENTER]) > 0:
min_bp = max(break_points[break_points < CENTER])
if len(break_points[break_points > CENTER]) > 0:
max_bp = min(break_points[break_points > CENTER])
if numpy.min(subset) != 0.0 or numpy.max(subset) != 0.0:
LOGGER.info("%d, %3.2f, %3.2f, %3.2f * %d, %3.2f, %3.2f, %3.2f" % (min_bp, quant[min_bp], numpy.min(subset), quant[0] / numpy.min(subset), max_bp, quant[99] , numpy.max(subset), quant[99] / numpy.max(subset)))
for i, value in enumerate(range(max_bp, 100, 1)):
modelled = f_lin(value, points[max_bp]["slope"], points[max_bp]["offset"])
diff = abs(quant[value] - modelled)
global_quant.append((value, quant[value], diff))
for i, value in enumerate(range(0, min_bp,)):
modelled = f_lin(value, points[max_bp]["slope"], points[max_bp]["offset"])
diff = abs(quant[value] - modelled)
global_quant.append((value, quant[value], diff))
counter += 1
if make_plot:
fig = pylab.gcf()
fig.set_size_inches(18.5, 10.5)
name = image_file.replace(".tif", "_local.png")
fig.savefig(name, bbox_inches='tight', dpi=150)
z, y, x = data.shape
rgb = numpy.zeros((y, x, z))
for i in range(0, z):
rgb[:, :, i] = data[i, :, :] / numpy.max(data[i, :, :])
col_space1 = color.rgb2lab(rgb[:, :, 0:3])
subset_result = numpy.zeros((3, y, x), dtype=numpy.float)
total_q = len(global_quant)
for counter, item in enumerate(global_quant):
LOGGER.info("%d: %d, %s" % (counter, total_q, str(item)))
q, value, diff = item
if diff > MAX_ERROR:
if q < 50.0:
subset_result[0, :, :] = numpy.where(col_space1[ :, :, 0] < value, 1 + col_space1[ :, :, 0] - diff, subset_result[0, :, :])
subset_result[1, :, :] = numpy.where(col_space1[ :, :, 0] < value, subset_result[1, :, :] + diff, subset_result[1, :, :])
subset_result[2, :, :] = numpy.where(col_space1[ :, :, 2] < value, subset_result[2, :, :] - 1, subset_result[2, :, :])
else:
subset_result[0, :, :] = numpy.where(col_space1[ :, :, 0] > value, 100. + col_space1[ :, :, 0] - diff, subset_result[0, :, :])
subset_result[1, :, :] = numpy.where(col_space1[ :, :, 0] > value, subset_result[1, :, :] + diff, subset_result[1, :, :])
subset_result[2, :, :] = numpy.where(col_space1[ :, :, 2] > value, subset_result[2, :, :] + 1, subset_result[2, :, :])
return subset_result
def calculate_top_of_atmosphere_spot6(self):
'''
Calculates the top of atmosphere for the image that is object represents.
'''
LOGGER.info('folder correctly identified')
LOGGER.info("Starting DN to TOA")
LOGGER.info("Start folder: %s" % self.path)
LOGGER.info('calculating TOA')
self.number_of_bands = self.get_raster().get_attribute(
raster.DATA_SHAPE)[2]
metadata_band_order = self.get_sensor().get_attribute(
spot6.BAND_DISPLAY_ORDER)
image_band_order = self.get_sensor().get_attribute(spot6.BAND_INDEX)
LOGGER.debug('band metadata order: %s' %
self.get_sensor().get_attribute(spot6.BAND_DISPLAY_ORDER))
LOGGER.debug('band image order: %s' %
self.get_sensor().get_attribute(spot6.BAND_INDEX))
band_order_reference = [u'B0', u'B1', u'B2', u'B3']
if image_band_order == metadata_band_order and metadata_band_order != band_order_reference:
band_solar_irradiance = self.get_sensor().get_attribute(
spot6.BAND_SOLAR_IRRADIANCE_VALUE)
band_solar_irradiance = list(
numpy.array(band_solar_irradiance)[map(
lambda x: metadata_band_order.index(x),
band_order_reference)])
LOGGER.info('band_solar_irradiance: %s' % band_solar_irradiance)
gain = map(float,
self.get_sensor().get_attribute(spot6.PHYSICAL_GAIN))
offset = map(float,
self.get_sensor().get_attribute(spot6.PHYSICAL_BIAS))
gain = list(
numpy.array(gain)[map(lambda x: metadata_band_order.index(x),
band_order_reference)])
offset = list(
numpy.array(offset)[map(lambda x: metadata_band_order.index(x),
band_order_reference)])
LOGGER.info('gain: %s' % gain)
LOGGER.info('offset: %s' % offset)
LOGGER.info('reading data_array')
data_array = self.get_raster().read_data_file_as_array()
sun_elevation = numpy.deg2rad(
float(
numpy.median(self.get_sensor().get_attribute(
spot6.SUN_ELEVATION))))
LOGGER.debug('sun_elevation: %s' % sun_elevation)
imaging_date = datetime.date(
self.sensor.get_attribute(spot6.ACQUISITION_DATE))
self.toa = calculate_rad_toa_spot6(data_array, gain, offset,
imaging_date, sun_elevation,
band_solar_irradiance,
self.number_of_bands)
LOGGER.info('finished TOA')
LOGGER.info('exporting to tif')
outname = re.sub(
r'.JP2', '',
self.file_dictionary[self.get_image_file()]) + '_TOA.TIF'
LOGGER.info('Results of folder %s is %s' % (self.path, outname))
data_file = self.get_raster().create_from_reference(
outname, self.toa.shape[2], self.toa.shape[1], self.toa.shape[0],
self.get_raster().get_attribute(raster.GEOTRANSFORM),
self.get_raster().get_attribute(raster.PROJECTION))
self.get_raster().write_raster(data_file, self.toa)
data_file = None
LOGGER.info('finished export')
LOGGER.info('finished DN to TOA')
def calculate_top_of_atmosphere_spot5(self):
'''
Calculates the top of atmosphere for the image that is object represents.
'''
LOGGER.info('folder correctly identified')
LOGGER.info("Starting DN to TOA")
LOGGER.info("Start folder: %s" % self.path)
LOGGER.info('calculating TOA')
self.number_of_bands = self.get_raster().get_attribute(raster.DATA_SHAPE)[2]
metadata_band_order = self.get_sensor().get_attribute(spot5.BAND_DESCRIPTION)
image_band_order = self.get_raster().get_attribute(raster.METADATA_FILE)['TIFFTAG_IMAGEDESCRIPTION'].split(" ")[:self.number_of_bands]
LOGGER.debug('band metadata order: %s' % self.get_sensor().get_attribute(spot5.BAND_DESCRIPTION))
LOGGER.debug('band image order: %s' % image_band_order)
self.geotransform_from_gcps = self.get_raster().gcps_to_geotransform()
LOGGER.info('reordering data_array')
data_array = self.get_raster().read_data_file_as_array()[map(lambda x: image_band_order.index(x), metadata_band_order), :, :]
self.projection = osr.SpatialReference()
self.projection.ImportFromEPSG(int(self.get_sensor().get_attribute(spot5.HORIZONTAL_CS_CODE).replace("epsg:", "")))
LOGGER.debug('projection: %s' % self.projection.ExportToWkt())
sun_elevation = float(self.get_sensor().get_attribute(spot5.SUN_ELEVATION))
LOGGER.debug('sun_elevation: %s' % sun_elevation)
gain = map(float, self.get_sensor().get_attribute(spot5.PHYSICAL_GAIN))
offset = [0] * len(gain)
hrg = self.get_sensor().get_attribute(spot5.HRG)
LOGGER.debug('HRG type: %s' % hrg)
imaging_date = datetime.date(self.sensor.get_attribute(spot5.ACQUISITION_DATE))
self.toa = calculate_rad_toa_spot5(data_array, gain, offset, imaging_date, sun_elevation, hrg, self.number_of_bands)
LOGGER.info('finished TOA')
LOGGER.info('exporting to tif')
outname = re.sub(r'.TIF', '', self.file_dictionary[self.get_image_file()]) + '_TOA.tif'
LOGGER.info('Result of folder %s is %s' % (self.path, outname))
data_file = self.get_raster().create_from_reference(outname, self.toa.shape[2], self.toa.shape[1], self.toa.shape[0], self.geotransform_from_gcps, self.projection.ExportToWkt())
self.get_raster().write_raster(data_file, self.toa)
data_file = None
LOGGER.info('finished export')
LOGGER.info('finished DN to TOA')