def download_chunks(scene_url, directory, filename):
'''
This method downloads a file in chunks.
'''
req = urllib2.urlopen(scene_url)
downloaded = 0
CHUNK = 1024 * 1024 * 8
total_size = int(req.info().getheader('Content-Length').strip())
total_size_fmt = size_of_fmt(total_size)
filepath = create_filename(directory, filename)
with open(filepath, 'wb') as fp:
start = time.clock()
LOGGER.info('Downloading %s %s:' % (filename, total_size_fmt))
while True:
chunk = req.read(CHUNK)
downloaded += len(chunk)
done = int(50 * downloaded / total_size)
sys.stdout.write('\r[{1}{2}]{0:3.0f}% {3}ps'.format(
math.floor((float(downloaded) / total_size) * 100), '=' * done,
' ' * (50 - done),
size_of_fmt((downloaded // (time.clock() - start)) / 8)))
sys.stdout.flush()
if not chunk: break
fp.write(chunk)
def calculate_top_of_atmosphere_spot6(self):
'''
Calculates the top of atmosphere for the image that is object represents.
'''
LOGGER.info('folder correctly identified')
LOGGER.info("Starting DN to TOA")
LOGGER.info("Start folder: %s" % self.path)
LOGGER.info('calculating TOA')
self.number_of_bands = self.get_raster().get_attribute(raster.DATA_SHAPE)[2]
metadata_band_order = self.get_sensor().get_attribute(spot6.BAND_DISPLAY_ORDER)
image_band_order = self.get_sensor().get_attribute(spot6.BAND_INDEX)
LOGGER.debug('band metadata order: %s' % self.get_sensor().get_attribute(spot6.BAND_DISPLAY_ORDER))
LOGGER.debug('band image order: %s' % self.get_sensor().get_attribute(spot6.BAND_INDEX))
band_order_reference = [u'B0', u'B1', u'B2', u'B3']
if image_band_order == metadata_band_order and metadata_band_order != band_order_reference:
band_solar_irradiance = self.get_sensor().get_attribute(spot6.BAND_SOLAR_IRRADIANCE_VALUE)
band_solar_irradiance = list(numpy.array(band_solar_irradiance)[map(lambda x: metadata_band_order.index(x), band_order_reference)])
LOGGER.info('band_solar_irradiance: %s' % band_solar_irradiance)
gain = map(float, self.get_sensor().get_attribute(spot6.PHYSICAL_GAIN))
offset = map(float, self.get_sensor().get_attribute(spot6.PHYSICAL_BIAS))
gain = list(numpy.array(gain)[map(lambda x: metadata_band_order.index(x), band_order_reference)])
offset = list(numpy.array(offset)[map(lambda x: metadata_band_order.index(x), band_order_reference)])
LOGGER.info('gain: %s' % gain)
LOGGER.info('offset: %s' % offset)
LOGGER.info('reading data_array')
data_array = self.get_raster().read_data_file_as_array()
sun_elevation = numpy.deg2rad(float(numpy.median(self.get_sensor().get_attribute(spot6.SUN_ELEVATION))))
LOGGER.debug('sun_elevation: %s' % sun_elevation)
imaging_date = datetime.date(self.sensor.get_attribute(spot6.ACQUISITION_DATE))
self.toa = calculate_rad_toa_spot6(data_array, gain, offset, imaging_date, sun_elevation, band_solar_irradiance, self.number_of_bands)
LOGGER.info('finished TOA')
LOGGER.info('exporting to tif')
outname = re.sub(r'.JP2', '', self.file_dictionary[self.get_image_file()]) + '_TOA.TIF'
LOGGER.info('Results of folder %s is %s' % (self.path, outname))
data_file = self.get_raster().create_from_reference(outname, self.toa.shape[2], self.toa.shape[1], self.toa.shape[0], self.get_raster().get_attribute(raster.GEOTRANSFORM), self.get_raster().get_attribute(raster.PROJECTION))
self.get_raster().write_raster(data_file, self.toa)
data_file = None
LOGGER.info('finished export')
LOGGER.info('finished DN to TOA')
def extract_extremes(data, image_file, make_plot, steps=1000):
# TODO: the two for and the last one takes too long to finish
CENTER = 50
xtiles = 5000 / steps
ytiles = 5000 / steps
counter = 0
b = 0
global_quant = list()
for ycount in range(0, 5000, steps):
for xcount in range(0, 5000, steps):
subset = data[:, ycount:ycount + steps, xcount:xcount + steps]
LOGGER.info("Extent: %dx%d" % (xcount, ycount))
z, y, x = subset.shape
rgb = numpy.zeros((y, x, z))
for i in range(0, z):
rgb[:, :, i] = subset[i, :, :] / numpy.max(data[i, :, :])
col_space1 = color.rgb2lab(rgb[:, :, 0:3])
quant = calculate_quantiles(col_space1[ :, :, 0])
if len(quant) > 0:
points = calculate_breaking_points(quant)
break_points = numpy.array(calculate_continuity(points.keys()))
else:
break_points = []
subset = data[b, ycount:ycount + steps, xcount:xcount + steps]
if make_plot:
pylab.subplot(xtiles, ytiles, counter + 1)
fig = pylab.plot(quant, c="k", alpha=0.5)
if len(break_points) > 1:
if make_plot:
for point in points.keys():
pylab.plot(point, numpy.array(quant)[point], 'r.')
if len(break_points[break_points < CENTER]) > 0:
min_bp = max(break_points[break_points < CENTER])
if len(break_points[break_points > CENTER]) > 0:
max_bp = min(break_points[break_points > CENTER])
if numpy.min(subset) != 0.0 or numpy.max(subset) != 0.0:
LOGGER.info("%d, %3.2f, %3.2f, %3.2f * %d, %3.2f, %3.2f, %3.2f" % (min_bp, quant[min_bp], numpy.min(subset), quant[0] / numpy.min(subset), max_bp, quant[99] , numpy.max(subset), quant[99] / numpy.max(subset)))
for i, value in enumerate(range(max_bp, 100, 1)):
modelled = f_lin(value, points[max_bp]["slope"], points[max_bp]["offset"])
diff = abs(quant[value] - modelled)
global_quant.append((value, quant[value], diff))
for i, value in enumerate(range(0, min_bp,)):
modelled = f_lin(value, points[max_bp]["slope"], points[max_bp]["offset"])
diff = abs(quant[value] - modelled)
global_quant.append((value, quant[value], diff))
counter += 1
if make_plot:
fig = pylab.gcf()
fig.set_size_inches(18.5, 10.5)
name = image_file.replace(".tif", "_local.png")
fig.savefig(name, bbox_inches='tight', dpi=150)
z, y, x = data.shape
rgb = numpy.zeros((y, x, z))
for i in range(0, z):
rgb[:, :, i] = data[i, :, :] / numpy.max(data[i, :, :])
col_space1 = color.rgb2lab(rgb[:, :, 0:3])
subset_result = numpy.zeros((3, y, x), dtype=numpy.float)
total_q = len(global_quant)
for counter, item in enumerate(global_quant):
LOGGER.info("%d: %d, %s" % (counter, total_q, str(item)))
q, value, diff = item
if diff > MAX_ERROR:
if q < 50.0:
subset_result[0, :, :] = numpy.where(col_space1[ :, :, 0] < value, 1 + col_space1[ :, :, 0] - diff, subset_result[0, :, :])
subset_result[1, :, :] = numpy.where(col_space1[ :, :, 0] < value, subset_result[1, :, :] + diff, subset_result[1, :, :])
subset_result[2, :, :] = numpy.where(col_space1[ :, :, 2] < value, subset_result[2, :, :] - 1, subset_result[2, :, :])
else:
subset_result[0, :, :] = numpy.where(col_space1[ :, :, 0] > value, 100. + col_space1[ :, :, 0] - diff, subset_result[0, :, :])
subset_result[1, :, :] = numpy.where(col_space1[ :, :, 0] > value, subset_result[1, :, :] + diff, subset_result[1, :, :])
subset_result[2, :, :] = numpy.where(col_space1[ :, :, 2] > value, subset_result[2, :, :] + 1, subset_result[2, :, :])
return subset_result
def calculate_top_of_atmosphere_spot6(self):
'''
Calculates the top of atmosphere for the image that is object represents.
'''
LOGGER.info('folder correctly identified')
LOGGER.info("Starting DN to TOA")
LOGGER.info("Start folder: %s" % self.path)
LOGGER.info('calculating TOA')
self.number_of_bands = self.get_raster().get_attribute(
raster.DATA_SHAPE)[2]
metadata_band_order = self.get_sensor().get_attribute(
spot6.BAND_DISPLAY_ORDER)
image_band_order = self.get_sensor().get_attribute(spot6.BAND_INDEX)
LOGGER.debug('band metadata order: %s' %
self.get_sensor().get_attribute(spot6.BAND_DISPLAY_ORDER))
LOGGER.debug('band image order: %s' %
self.get_sensor().get_attribute(spot6.BAND_INDEX))
band_order_reference = [u'B0', u'B1', u'B2', u'B3']
if image_band_order == metadata_band_order and metadata_band_order != band_order_reference:
band_solar_irradiance = self.get_sensor().get_attribute(
spot6.BAND_SOLAR_IRRADIANCE_VALUE)
band_solar_irradiance = list(
numpy.array(band_solar_irradiance)[map(
lambda x: metadata_band_order.index(x),
band_order_reference)])
LOGGER.info('band_solar_irradiance: %s' % band_solar_irradiance)
gain = map(float,
self.get_sensor().get_attribute(spot6.PHYSICAL_GAIN))
offset = map(float,
self.get_sensor().get_attribute(spot6.PHYSICAL_BIAS))
gain = list(
numpy.array(gain)[map(lambda x: metadata_band_order.index(x),
band_order_reference)])
offset = list(
numpy.array(offset)[map(lambda x: metadata_band_order.index(x),
band_order_reference)])
LOGGER.info('gain: %s' % gain)
LOGGER.info('offset: %s' % offset)
LOGGER.info('reading data_array')
data_array = self.get_raster().read_data_file_as_array()
sun_elevation = numpy.deg2rad(
float(
numpy.median(self.get_sensor().get_attribute(
spot6.SUN_ELEVATION))))
LOGGER.debug('sun_elevation: %s' % sun_elevation)
imaging_date = datetime.date(
self.sensor.get_attribute(spot6.ACQUISITION_DATE))
self.toa = calculate_rad_toa_spot6(data_array, gain, offset,
imaging_date, sun_elevation,
band_solar_irradiance,
self.number_of_bands)
LOGGER.info('finished TOA')
LOGGER.info('exporting to tif')
outname = re.sub(
r'.JP2', '',
self.file_dictionary[self.get_image_file()]) + '_TOA.TIF'
LOGGER.info('Results of folder %s is %s' % (self.path, outname))
data_file = self.get_raster().create_from_reference(
outname, self.toa.shape[2], self.toa.shape[1], self.toa.shape[0],
self.get_raster().get_attribute(raster.GEOTRANSFORM),
self.get_raster().get_attribute(raster.PROJECTION))
self.get_raster().write_raster(data_file, self.toa)
data_file = None
LOGGER.info('finished export')
LOGGER.info('finished DN to TOA')
def calculate_top_of_atmosphere_spot5(self):
'''
Calculates the top of atmosphere for the image that is object represents.
'''
LOGGER.info('folder correctly identified')
LOGGER.info("Starting DN to TOA")
LOGGER.info("Start folder: %s" % self.path)
LOGGER.info('calculating TOA')
self.number_of_bands = self.get_raster().get_attribute(raster.DATA_SHAPE)[2]
metadata_band_order = self.get_sensor().get_attribute(spot5.BAND_DESCRIPTION)
image_band_order = self.get_raster().get_attribute(raster.METADATA_FILE)['TIFFTAG_IMAGEDESCRIPTION'].split(" ")[:self.number_of_bands]
LOGGER.debug('band metadata order: %s' % self.get_sensor().get_attribute(spot5.BAND_DESCRIPTION))
LOGGER.debug('band image order: %s' % image_band_order)
self.geotransform_from_gcps = self.get_raster().gcps_to_geotransform()
LOGGER.info('reordering data_array')
data_array = self.get_raster().read_data_file_as_array()[map(lambda x: image_band_order.index(x), metadata_band_order), :, :]
self.projection = osr.SpatialReference()
self.projection.ImportFromEPSG(int(self.get_sensor().get_attribute(spot5.HORIZONTAL_CS_CODE).replace("epsg:", "")))
LOGGER.debug('projection: %s' % self.projection.ExportToWkt())
sun_elevation = float(self.get_sensor().get_attribute(spot5.SUN_ELEVATION))
LOGGER.debug('sun_elevation: %s' % sun_elevation)
gain = map(float, self.get_sensor().get_attribute(spot5.PHYSICAL_GAIN))
offset = [0] * len(gain)
hrg = self.get_sensor().get_attribute(spot5.HRG)
LOGGER.debug('HRG type: %s' % hrg)
imaging_date = datetime.date(self.sensor.get_attribute(spot5.ACQUISITION_DATE))
self.toa = calculate_rad_toa_spot5(data_array, gain, offset, imaging_date, sun_elevation, hrg, self.number_of_bands)
LOGGER.info('finished TOA')
LOGGER.info('exporting to tif')
outname = re.sub(r'.TIF', '', self.file_dictionary[self.get_image_file()]) + '_TOA.tif'
LOGGER.info('Result of folder %s is %s' % (self.path, outname))
data_file = self.get_raster().create_from_reference(outname, self.toa.shape[2], self.toa.shape[1], self.toa.shape[0], self.geotransform_from_gcps, self.projection.ExportToWkt())
self.get_raster().write_raster(data_file, self.toa)
data_file = None
LOGGER.info('finished export')
LOGGER.info('finished DN to TOA')
