Ejemplo n.º 1
0
def download_landsat_scene(url, directory, filename):
    '''
    This method downloads a scene directly from usgs. In order to do so, it
    pretends to be a browser to build a request that is accepted by the server.
    We added the headers so we don't get banned when the server detects that we
    are doing lots of requests. This idea is based on the landsat downloader:
    https://github.com/olivierhagolle/LANDSAT-Download
    '''
    cookies = urllib2.HTTPCookieProcessor()
    opener = urllib2.build_opener(cookies)
    urllib2.install_opener(opener)
    data = urllib2.urlopen("https://ers.cr.usgs.gov").read()
    token_group = re.search(r'<input .*?name="csrf_token".*?value="(.*?)"',
                            data)
    if token_group:
        token = token_group.group(1)
    else:
        LOGGER.error('The cross site request forgery token was not found.')
        sys.exit(1)
    usgs = {
        'account': getattr(SETTINGS, 'USGS_USER'),
        'passwd': getattr(SETTINGS, 'USGS_PASSWORD')
    }
    params = urllib.urlencode(
        dict(username=usgs['account'],
             password=usgs['passwd'],
             csrf_token=token))
    request = urllib2.Request(
        "https://ers.cr.usgs.gov/login",
        params,
        headers={
            'User-Agent':
            'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.7) Gecko/2009021910 Firefox/3.0.7'
        })
    f = urllib2.urlopen(request)
    data = f.read()
    f.close()
    download_chunks(url, directory, filename)
Ejemplo n.º 2
0
def download_chunks(scene_url, directory, filename):
    '''
    This method downloads a file in chunks.
    '''
    req = urllib2.urlopen(scene_url)
    downloaded = 0
    CHUNK = 1024 * 1024 * 8
    total_size = int(req.info().getheader('Content-Length').strip())
    total_size_fmt = size_of_fmt(total_size)
    filepath = create_filename(directory, filename)
    with open(filepath, 'wb') as fp:
        start = time.clock()
        LOGGER.info('Downloading %s %s:' % (filename, total_size_fmt))
        while True:
            chunk = req.read(CHUNK)
            downloaded += len(chunk)
            done = int(50 * downloaded / total_size)
            sys.stdout.write('\r[{1}{2}]{0:3.0f}% {3}ps'.format(
                math.floor((float(downloaded) / total_size) * 100), '=' * done,
                ' ' * (50 - done),
                size_of_fmt((downloaded // (time.clock() - start)) / 8)))
            sys.stdout.flush()
            if not chunk: break
            fp.write(chunk)
Ejemplo n.º 3
0
 def calculate_top_of_atmosphere_spot6(self):
     '''
     Calculates the top of atmosphere for the image that is object represents.
     '''
     LOGGER.info('folder correctly identified')
     LOGGER.info("Starting DN to TOA")
     LOGGER.info("Start folder: %s" % self.path)
     LOGGER.info('calculating TOA')
     self.number_of_bands = self.get_raster().get_attribute(raster.DATA_SHAPE)[2]
     metadata_band_order = self.get_sensor().get_attribute(spot6.BAND_DISPLAY_ORDER)
     image_band_order = self.get_sensor().get_attribute(spot6.BAND_INDEX)
     LOGGER.debug('band metadata order: %s' % self.get_sensor().get_attribute(spot6.BAND_DISPLAY_ORDER)) 
     LOGGER.debug('band image order: %s' % self.get_sensor().get_attribute(spot6.BAND_INDEX)) 
     band_order_reference = [u'B0', u'B1', u'B2', u'B3']
     if image_band_order == metadata_band_order and metadata_band_order != band_order_reference:
         band_solar_irradiance = self.get_sensor().get_attribute(spot6.BAND_SOLAR_IRRADIANCE_VALUE)
         band_solar_irradiance = list(numpy.array(band_solar_irradiance)[map(lambda x: metadata_band_order.index(x), band_order_reference)])
         LOGGER.info('band_solar_irradiance: %s' % band_solar_irradiance)
         gain = map(float, self.get_sensor().get_attribute(spot6.PHYSICAL_GAIN))
         offset = map(float, self.get_sensor().get_attribute(spot6.PHYSICAL_BIAS))
         gain = list(numpy.array(gain)[map(lambda x: metadata_band_order.index(x), band_order_reference)])
         offset = list(numpy.array(offset)[map(lambda x: metadata_band_order.index(x), band_order_reference)]) 
         LOGGER.info('gain: %s' % gain)
         LOGGER.info('offset: %s' % offset)         
         LOGGER.info('reading data_array')
         data_array = self.get_raster().read_data_file_as_array()
     sun_elevation = numpy.deg2rad(float(numpy.median(self.get_sensor().get_attribute(spot6.SUN_ELEVATION))))
     LOGGER.debug('sun_elevation: %s' % sun_elevation)
     imaging_date = datetime.date(self.sensor.get_attribute(spot6.ACQUISITION_DATE))
     self.toa = calculate_rad_toa_spot6(data_array, gain, offset, imaging_date, sun_elevation, band_solar_irradiance, self.number_of_bands)
     LOGGER.info('finished TOA')
     LOGGER.info('exporting to tif')
     outname = re.sub(r'.JP2', '', self.file_dictionary[self.get_image_file()]) + '_TOA.TIF'
     LOGGER.info('Results of folder %s is %s' % (self.path, outname))
     data_file = self.get_raster().create_from_reference(outname, self.toa.shape[2], self.toa.shape[1], self.toa.shape[0], self.get_raster().get_attribute(raster.GEOTRANSFORM), self.get_raster().get_attribute(raster.PROJECTION))
     self.get_raster().write_raster(data_file, self.toa) 
     data_file = None
     LOGGER.info('finished export')
     LOGGER.info('finished DN to TOA') 
Ejemplo n.º 4
0
def extract_extremes(data, image_file, make_plot, steps=1000):
    # TODO: the two for and the last one takes too long to finish
    CENTER = 50
    xtiles = 5000 / steps
    ytiles = 5000 / steps
    counter = 0
    b = 0
    global_quant = list()
    for ycount in range(0, 5000, steps):
        for xcount in range(0, 5000, steps):
            subset = data[:, ycount:ycount + steps, xcount:xcount + steps]
            LOGGER.info("Extent: %dx%d" % (xcount, ycount))
            z, y, x = subset.shape
            rgb = numpy.zeros((y, x, z))
            for i in range(0, z):
                rgb[:, :, i] = subset[i, :, :] / numpy.max(data[i, :, :])
            col_space1 = color.rgb2lab(rgb[:, :, 0:3])
            quant = calculate_quantiles(col_space1[ :, :, 0])
            if len(quant) > 0:
                points = calculate_breaking_points(quant)
                break_points = numpy.array(calculate_continuity(points.keys()))
            else:
                break_points = []
            subset = data[b, ycount:ycount + steps, xcount:xcount + steps]
            if make_plot:
                pylab.subplot(xtiles, ytiles, counter + 1)
                fig = pylab.plot(quant, c="k", alpha=0.5)
            if len(break_points) > 1:
                if make_plot:
                    for point in points.keys():
                        pylab.plot(point, numpy.array(quant)[point], 'r.')
                if len(break_points[break_points < CENTER]) > 0:
                    min_bp = max(break_points[break_points < CENTER])
                if len(break_points[break_points > CENTER]) > 0:
                    max_bp = min(break_points[break_points > CENTER])
                if numpy.min(subset) != 0.0 or numpy.max(subset) != 0.0:
                    LOGGER.info("%d, %3.2f, %3.2f, %3.2f * %d, %3.2f, %3.2f, %3.2f" % (min_bp, quant[min_bp], numpy.min(subset), quant[0] / numpy.min(subset), max_bp, quant[99] , numpy.max(subset), quant[99] / numpy.max(subset)))
                for i, value in enumerate(range(max_bp, 100, 1)):
                    modelled = f_lin(value, points[max_bp]["slope"], points[max_bp]["offset"])
                    diff = abs(quant[value] - modelled)
                    global_quant.append((value, quant[value], diff))
                for i, value in enumerate(range(0, min_bp,)):
                    modelled = f_lin(value, points[max_bp]["slope"], points[max_bp]["offset"])
                    diff = abs(quant[value] - modelled)
                    global_quant.append((value, quant[value], diff))
            counter += 1
            if make_plot:
                fig = pylab.gcf()
                fig.set_size_inches(18.5, 10.5)
                name = image_file.replace(".tif", "_local.png")
                fig.savefig(name, bbox_inches='tight', dpi=150)
    z, y, x = data.shape
    rgb = numpy.zeros((y, x, z))
    for i in range(0, z):
        rgb[:, :, i] = data[i, :, :] / numpy.max(data[i, :, :])
    col_space1 = color.rgb2lab(rgb[:, :, 0:3])
    subset_result = numpy.zeros((3, y, x), dtype=numpy.float)
    total_q = len(global_quant)
    for counter, item in enumerate(global_quant):
        LOGGER.info("%d: %d, %s" % (counter, total_q, str(item)))
        q, value, diff = item
        if diff > MAX_ERROR:
            if q < 50.0:
                subset_result[0, :, :] = numpy.where(col_space1[ :, :, 0] < value, 1 + col_space1[ :, :, 0] - diff, subset_result[0, :, :])
                subset_result[1, :, :] = numpy.where(col_space1[ :, :, 0] < value, subset_result[1, :, :] + diff, subset_result[1, :, :])
                subset_result[2, :, :] = numpy.where(col_space1[ :, :, 2] < value, subset_result[2, :, :] - 1, subset_result[2, :, :])
            else:                        
                subset_result[0, :, :] = numpy.where(col_space1[ :, :, 0] > value, 100. + col_space1[ :, :, 0] - diff, subset_result[0, :, :])
                subset_result[1, :, :] = numpy.where(col_space1[ :, :, 0] > value, subset_result[1, :, :] + diff, subset_result[1, :, :])
                subset_result[2, :, :] = numpy.where(col_space1[ :, :, 2] > value, subset_result[2, :, :] + 1, subset_result[2, :, :])
    return subset_result
Ejemplo n.º 5
0
 def calculate_top_of_atmosphere_spot6(self):
     '''
     Calculates the top of atmosphere for the image that is object represents.
     '''
     LOGGER.info('folder correctly identified')
     LOGGER.info("Starting DN to TOA")
     LOGGER.info("Start folder: %s" % self.path)
     LOGGER.info('calculating TOA')
     self.number_of_bands = self.get_raster().get_attribute(
         raster.DATA_SHAPE)[2]
     metadata_band_order = self.get_sensor().get_attribute(
         spot6.BAND_DISPLAY_ORDER)
     image_band_order = self.get_sensor().get_attribute(spot6.BAND_INDEX)
     LOGGER.debug('band metadata order: %s' %
                  self.get_sensor().get_attribute(spot6.BAND_DISPLAY_ORDER))
     LOGGER.debug('band image order: %s' %
                  self.get_sensor().get_attribute(spot6.BAND_INDEX))
     band_order_reference = [u'B0', u'B1', u'B2', u'B3']
     if image_band_order == metadata_band_order and metadata_band_order != band_order_reference:
         band_solar_irradiance = self.get_sensor().get_attribute(
             spot6.BAND_SOLAR_IRRADIANCE_VALUE)
         band_solar_irradiance = list(
             numpy.array(band_solar_irradiance)[map(
                 lambda x: metadata_band_order.index(x),
                 band_order_reference)])
         LOGGER.info('band_solar_irradiance: %s' % band_solar_irradiance)
         gain = map(float,
                    self.get_sensor().get_attribute(spot6.PHYSICAL_GAIN))
         offset = map(float,
                      self.get_sensor().get_attribute(spot6.PHYSICAL_BIAS))
         gain = list(
             numpy.array(gain)[map(lambda x: metadata_band_order.index(x),
                                   band_order_reference)])
         offset = list(
             numpy.array(offset)[map(lambda x: metadata_band_order.index(x),
                                     band_order_reference)])
         LOGGER.info('gain: %s' % gain)
         LOGGER.info('offset: %s' % offset)
         LOGGER.info('reading data_array')
         data_array = self.get_raster().read_data_file_as_array()
     sun_elevation = numpy.deg2rad(
         float(
             numpy.median(self.get_sensor().get_attribute(
                 spot6.SUN_ELEVATION))))
     LOGGER.debug('sun_elevation: %s' % sun_elevation)
     imaging_date = datetime.date(
         self.sensor.get_attribute(spot6.ACQUISITION_DATE))
     self.toa = calculate_rad_toa_spot6(data_array, gain, offset,
                                        imaging_date, sun_elevation,
                                        band_solar_irradiance,
                                        self.number_of_bands)
     LOGGER.info('finished TOA')
     LOGGER.info('exporting to tif')
     outname = re.sub(
         r'.JP2', '',
         self.file_dictionary[self.get_image_file()]) + '_TOA.TIF'
     LOGGER.info('Results of folder %s is %s' % (self.path, outname))
     data_file = self.get_raster().create_from_reference(
         outname, self.toa.shape[2], self.toa.shape[1], self.toa.shape[0],
         self.get_raster().get_attribute(raster.GEOTRANSFORM),
         self.get_raster().get_attribute(raster.PROJECTION))
     self.get_raster().write_raster(data_file, self.toa)
     data_file = None
     LOGGER.info('finished export')
     LOGGER.info('finished DN to TOA')
Ejemplo n.º 6
0
 def calculate_top_of_atmosphere_spot5(self):
     '''
     Calculates the top of atmosphere for the image that is object represents.
     '''
     LOGGER.info('folder correctly identified')
     LOGGER.info("Starting DN to TOA")
     LOGGER.info("Start folder: %s" % self.path)
     LOGGER.info('calculating TOA')
     self.number_of_bands = self.get_raster().get_attribute(raster.DATA_SHAPE)[2]
     metadata_band_order = self.get_sensor().get_attribute(spot5.BAND_DESCRIPTION)
     image_band_order = self.get_raster().get_attribute(raster.METADATA_FILE)['TIFFTAG_IMAGEDESCRIPTION'].split(" ")[:self.number_of_bands]
     LOGGER.debug('band metadata order: %s' % self.get_sensor().get_attribute(spot5.BAND_DESCRIPTION)) 
     LOGGER.debug('band image order: %s' % image_band_order)
     self.geotransform_from_gcps = self.get_raster().gcps_to_geotransform()
     LOGGER.info('reordering data_array')
     data_array = self.get_raster().read_data_file_as_array()[map(lambda x: image_band_order.index(x), metadata_band_order), :, :]
     self.projection = osr.SpatialReference()
     self.projection.ImportFromEPSG(int(self.get_sensor().get_attribute(spot5.HORIZONTAL_CS_CODE).replace("epsg:", "")))
     LOGGER.debug('projection: %s' % self.projection.ExportToWkt())  
     sun_elevation = float(self.get_sensor().get_attribute(spot5.SUN_ELEVATION))
     LOGGER.debug('sun_elevation: %s' % sun_elevation) 
     gain = map(float, self.get_sensor().get_attribute(spot5.PHYSICAL_GAIN))
     offset = [0] * len(gain)
     hrg = self.get_sensor().get_attribute(spot5.HRG)
     LOGGER.debug('HRG type: %s' % hrg)
     imaging_date = datetime.date(self.sensor.get_attribute(spot5.ACQUISITION_DATE))
     self.toa = calculate_rad_toa_spot5(data_array, gain, offset, imaging_date, sun_elevation, hrg, self.number_of_bands)   
     LOGGER.info('finished TOA')
     LOGGER.info('exporting to tif')       
     outname = re.sub(r'.TIF', '', self.file_dictionary[self.get_image_file()]) + '_TOA.tif'    
     LOGGER.info('Result of folder %s is %s' % (self.path, outname))
     data_file = self.get_raster().create_from_reference(outname, self.toa.shape[2], self.toa.shape[1], self.toa.shape[0], self.geotransform_from_gcps, self.projection.ExportToWkt())
     self.get_raster().write_raster(data_file, self.toa) 
     data_file = None
     LOGGER.info('finished export')
     LOGGER.info('finished DN to TOA')