def download_chunks(scene_url, directory, filename): ''' This method downloads a file in chunks. ''' req = urllib2.urlopen(scene_url) downloaded = 0 CHUNK = 1024 * 1024 * 8 total_size = int(req.info().getheader('Content-Length').strip()) total_size_fmt = size_of_fmt(total_size) filepath = create_filename(directory, filename) with open(filepath, 'wb') as fp: start = time.clock() LOGGER.info('Downloading %s %s:' % (filename, total_size_fmt)) while True: chunk = req.read(CHUNK) downloaded += len(chunk) done = int(50 * downloaded / total_size) sys.stdout.write('\r[{1}{2}]{0:3.0f}% {3}ps'.format( math.floor((float(downloaded) / total_size) * 100), '=' * done, ' ' * (50 - done), size_of_fmt((downloaded // (time.clock() - start)) / 8))) sys.stdout.flush() if not chunk: break fp.write(chunk)
def calculate_top_of_atmosphere_spot6(self): ''' Calculates the top of atmosphere for the image that is object represents. ''' LOGGER.info('folder correctly identified') LOGGER.info("Starting DN to TOA") LOGGER.info("Start folder: %s" % self.path) LOGGER.info('calculating TOA') self.number_of_bands = self.get_raster().get_attribute(raster.DATA_SHAPE)[2] metadata_band_order = self.get_sensor().get_attribute(spot6.BAND_DISPLAY_ORDER) image_band_order = self.get_sensor().get_attribute(spot6.BAND_INDEX) LOGGER.debug('band metadata order: %s' % self.get_sensor().get_attribute(spot6.BAND_DISPLAY_ORDER)) LOGGER.debug('band image order: %s' % self.get_sensor().get_attribute(spot6.BAND_INDEX)) band_order_reference = [u'B0', u'B1', u'B2', u'B3'] if image_band_order == metadata_band_order and metadata_band_order != band_order_reference: band_solar_irradiance = self.get_sensor().get_attribute(spot6.BAND_SOLAR_IRRADIANCE_VALUE) band_solar_irradiance = list(numpy.array(band_solar_irradiance)[map(lambda x: metadata_band_order.index(x), band_order_reference)]) LOGGER.info('band_solar_irradiance: %s' % band_solar_irradiance) gain = map(float, self.get_sensor().get_attribute(spot6.PHYSICAL_GAIN)) offset = map(float, self.get_sensor().get_attribute(spot6.PHYSICAL_BIAS)) gain = list(numpy.array(gain)[map(lambda x: metadata_band_order.index(x), band_order_reference)]) offset = list(numpy.array(offset)[map(lambda x: metadata_band_order.index(x), band_order_reference)]) LOGGER.info('gain: %s' % gain) LOGGER.info('offset: %s' % offset) LOGGER.info('reading data_array') data_array = self.get_raster().read_data_file_as_array() sun_elevation = numpy.deg2rad(float(numpy.median(self.get_sensor().get_attribute(spot6.SUN_ELEVATION)))) LOGGER.debug('sun_elevation: %s' % sun_elevation) imaging_date = datetime.date(self.sensor.get_attribute(spot6.ACQUISITION_DATE)) self.toa = calculate_rad_toa_spot6(data_array, gain, offset, imaging_date, sun_elevation, band_solar_irradiance, self.number_of_bands) LOGGER.info('finished TOA') LOGGER.info('exporting to tif') outname = re.sub(r'.JP2', '', self.file_dictionary[self.get_image_file()]) + '_TOA.TIF' LOGGER.info('Results of folder %s is %s' % (self.path, outname)) data_file = self.get_raster().create_from_reference(outname, self.toa.shape[2], self.toa.shape[1], self.toa.shape[0], self.get_raster().get_attribute(raster.GEOTRANSFORM), self.get_raster().get_attribute(raster.PROJECTION)) self.get_raster().write_raster(data_file, self.toa) data_file = None LOGGER.info('finished export') LOGGER.info('finished DN to TOA')
def extract_extremes(data, image_file, make_plot, steps=1000): # TODO: the two for and the last one takes too long to finish CENTER = 50 xtiles = 5000 / steps ytiles = 5000 / steps counter = 0 b = 0 global_quant = list() for ycount in range(0, 5000, steps): for xcount in range(0, 5000, steps): subset = data[:, ycount:ycount + steps, xcount:xcount + steps] LOGGER.info("Extent: %dx%d" % (xcount, ycount)) z, y, x = subset.shape rgb = numpy.zeros((y, x, z)) for i in range(0, z): rgb[:, :, i] = subset[i, :, :] / numpy.max(data[i, :, :]) col_space1 = color.rgb2lab(rgb[:, :, 0:3]) quant = calculate_quantiles(col_space1[ :, :, 0]) if len(quant) > 0: points = calculate_breaking_points(quant) break_points = numpy.array(calculate_continuity(points.keys())) else: break_points = [] subset = data[b, ycount:ycount + steps, xcount:xcount + steps] if make_plot: pylab.subplot(xtiles, ytiles, counter + 1) fig = pylab.plot(quant, c="k", alpha=0.5) if len(break_points) > 1: if make_plot: for point in points.keys(): pylab.plot(point, numpy.array(quant)[point], 'r.') if len(break_points[break_points < CENTER]) > 0: min_bp = max(break_points[break_points < CENTER]) if len(break_points[break_points > CENTER]) > 0: max_bp = min(break_points[break_points > CENTER]) if numpy.min(subset) != 0.0 or numpy.max(subset) != 0.0: LOGGER.info("%d, %3.2f, %3.2f, %3.2f * %d, %3.2f, %3.2f, %3.2f" % (min_bp, quant[min_bp], numpy.min(subset), quant[0] / numpy.min(subset), max_bp, quant[99] , numpy.max(subset), quant[99] / numpy.max(subset))) for i, value in enumerate(range(max_bp, 100, 1)): modelled = f_lin(value, points[max_bp]["slope"], points[max_bp]["offset"]) diff = abs(quant[value] - modelled) global_quant.append((value, quant[value], diff)) for i, value in enumerate(range(0, min_bp,)): modelled = f_lin(value, points[max_bp]["slope"], points[max_bp]["offset"]) diff = abs(quant[value] - modelled) global_quant.append((value, quant[value], diff)) counter += 1 if make_plot: fig = pylab.gcf() fig.set_size_inches(18.5, 10.5) name = image_file.replace(".tif", "_local.png") fig.savefig(name, bbox_inches='tight', dpi=150) z, y, x = data.shape rgb = numpy.zeros((y, x, z)) for i in range(0, z): rgb[:, :, i] = data[i, :, :] / numpy.max(data[i, :, :]) col_space1 = color.rgb2lab(rgb[:, :, 0:3]) subset_result = numpy.zeros((3, y, x), dtype=numpy.float) total_q = len(global_quant) for counter, item in enumerate(global_quant): LOGGER.info("%d: %d, %s" % (counter, total_q, str(item))) q, value, diff = item if diff > MAX_ERROR: if q < 50.0: subset_result[0, :, :] = numpy.where(col_space1[ :, :, 0] < value, 1 + col_space1[ :, :, 0] - diff, subset_result[0, :, :]) subset_result[1, :, :] = numpy.where(col_space1[ :, :, 0] < value, subset_result[1, :, :] + diff, subset_result[1, :, :]) subset_result[2, :, :] = numpy.where(col_space1[ :, :, 2] < value, subset_result[2, :, :] - 1, subset_result[2, :, :]) else: subset_result[0, :, :] = numpy.where(col_space1[ :, :, 0] > value, 100. + col_space1[ :, :, 0] - diff, subset_result[0, :, :]) subset_result[1, :, :] = numpy.where(col_space1[ :, :, 0] > value, subset_result[1, :, :] + diff, subset_result[1, :, :]) subset_result[2, :, :] = numpy.where(col_space1[ :, :, 2] > value, subset_result[2, :, :] + 1, subset_result[2, :, :]) return subset_result
def calculate_top_of_atmosphere_spot6(self): ''' Calculates the top of atmosphere for the image that is object represents. ''' LOGGER.info('folder correctly identified') LOGGER.info("Starting DN to TOA") LOGGER.info("Start folder: %s" % self.path) LOGGER.info('calculating TOA') self.number_of_bands = self.get_raster().get_attribute( raster.DATA_SHAPE)[2] metadata_band_order = self.get_sensor().get_attribute( spot6.BAND_DISPLAY_ORDER) image_band_order = self.get_sensor().get_attribute(spot6.BAND_INDEX) LOGGER.debug('band metadata order: %s' % self.get_sensor().get_attribute(spot6.BAND_DISPLAY_ORDER)) LOGGER.debug('band image order: %s' % self.get_sensor().get_attribute(spot6.BAND_INDEX)) band_order_reference = [u'B0', u'B1', u'B2', u'B3'] if image_band_order == metadata_band_order and metadata_band_order != band_order_reference: band_solar_irradiance = self.get_sensor().get_attribute( spot6.BAND_SOLAR_IRRADIANCE_VALUE) band_solar_irradiance = list( numpy.array(band_solar_irradiance)[map( lambda x: metadata_band_order.index(x), band_order_reference)]) LOGGER.info('band_solar_irradiance: %s' % band_solar_irradiance) gain = map(float, self.get_sensor().get_attribute(spot6.PHYSICAL_GAIN)) offset = map(float, self.get_sensor().get_attribute(spot6.PHYSICAL_BIAS)) gain = list( numpy.array(gain)[map(lambda x: metadata_band_order.index(x), band_order_reference)]) offset = list( numpy.array(offset)[map(lambda x: metadata_band_order.index(x), band_order_reference)]) LOGGER.info('gain: %s' % gain) LOGGER.info('offset: %s' % offset) LOGGER.info('reading data_array') data_array = self.get_raster().read_data_file_as_array() sun_elevation = numpy.deg2rad( float( numpy.median(self.get_sensor().get_attribute( spot6.SUN_ELEVATION)))) LOGGER.debug('sun_elevation: %s' % sun_elevation) imaging_date = datetime.date( self.sensor.get_attribute(spot6.ACQUISITION_DATE)) self.toa = calculate_rad_toa_spot6(data_array, gain, offset, imaging_date, sun_elevation, band_solar_irradiance, self.number_of_bands) LOGGER.info('finished TOA') LOGGER.info('exporting to tif') outname = re.sub( r'.JP2', '', self.file_dictionary[self.get_image_file()]) + '_TOA.TIF' LOGGER.info('Results of folder %s is %s' % (self.path, outname)) data_file = self.get_raster().create_from_reference( outname, self.toa.shape[2], self.toa.shape[1], self.toa.shape[0], self.get_raster().get_attribute(raster.GEOTRANSFORM), self.get_raster().get_attribute(raster.PROJECTION)) self.get_raster().write_raster(data_file, self.toa) data_file = None LOGGER.info('finished export') LOGGER.info('finished DN to TOA')
def calculate_top_of_atmosphere_spot5(self): ''' Calculates the top of atmosphere for the image that is object represents. ''' LOGGER.info('folder correctly identified') LOGGER.info("Starting DN to TOA") LOGGER.info("Start folder: %s" % self.path) LOGGER.info('calculating TOA') self.number_of_bands = self.get_raster().get_attribute(raster.DATA_SHAPE)[2] metadata_band_order = self.get_sensor().get_attribute(spot5.BAND_DESCRIPTION) image_band_order = self.get_raster().get_attribute(raster.METADATA_FILE)['TIFFTAG_IMAGEDESCRIPTION'].split(" ")[:self.number_of_bands] LOGGER.debug('band metadata order: %s' % self.get_sensor().get_attribute(spot5.BAND_DESCRIPTION)) LOGGER.debug('band image order: %s' % image_band_order) self.geotransform_from_gcps = self.get_raster().gcps_to_geotransform() LOGGER.info('reordering data_array') data_array = self.get_raster().read_data_file_as_array()[map(lambda x: image_band_order.index(x), metadata_band_order), :, :] self.projection = osr.SpatialReference() self.projection.ImportFromEPSG(int(self.get_sensor().get_attribute(spot5.HORIZONTAL_CS_CODE).replace("epsg:", ""))) LOGGER.debug('projection: %s' % self.projection.ExportToWkt()) sun_elevation = float(self.get_sensor().get_attribute(spot5.SUN_ELEVATION)) LOGGER.debug('sun_elevation: %s' % sun_elevation) gain = map(float, self.get_sensor().get_attribute(spot5.PHYSICAL_GAIN)) offset = [0] * len(gain) hrg = self.get_sensor().get_attribute(spot5.HRG) LOGGER.debug('HRG type: %s' % hrg) imaging_date = datetime.date(self.sensor.get_attribute(spot5.ACQUISITION_DATE)) self.toa = calculate_rad_toa_spot5(data_array, gain, offset, imaging_date, sun_elevation, hrg, self.number_of_bands) LOGGER.info('finished TOA') LOGGER.info('exporting to tif') outname = re.sub(r'.TIF', '', self.file_dictionary[self.get_image_file()]) + '_TOA.tif' LOGGER.info('Result of folder %s is %s' % (self.path, outname)) data_file = self.get_raster().create_from_reference(outname, self.toa.shape[2], self.toa.shape[1], self.toa.shape[0], self.geotransform_from_gcps, self.projection.ExportToWkt()) self.get_raster().write_raster(data_file, self.toa) data_file = None LOGGER.info('finished export') LOGGER.info('finished DN to TOA')