Example #1
0
def persist_bundle(bundle, keep=False):
    '''
    This function persist a bundle in both the database and the file system. It
    is responsibility of the bundle to provide information about which files
    should be persisted in the file system, and to build the database object
    that will be inserted in the database. The database is configured using the
    session maker in the connection module.
    In order to achieve its purpose, this method creates a list of the actions
    to perform. Once the list is fully populated, it calls the act method for
    each element in the list. If any of the actions in the list fails, a
    rollback is performed in all of them, the result is the same state as before
    the method was called.
    '''
    destination = bundle.get_output_directory()
    create_directory_path(destination)
    actions = []
    session = SESSION_MAKER()
    try:
        if not session.query(Product).filter(
                Product.product_path ==
                bundle.get_database_object().product_path).count():
            if not keep:
                LOGGER.debug(
                    'This process will move the files to a new destination.')
                for file_name in bundle.get_files():
                    actions.append(
                        filesystem.InsertAction(file_name, destination))
            else:
                LOGGER.debug(
                    'This process will keep the original path for the files.')
            actions.append(
                database.InsertAction(bundle.get_database_object(), session))
            import madmex.mapper.bundle.rapideye as re
            if isinstance(bundle, re.Bundle):
                actions.append(
                    database.InsertAction(bundle.get_features_object(),
                                          session))

            def do_result(action):
                '''
                Lambda function to perform an action and return the result.
                '''
                action.act()
                return action.success

            if not reduce(lambda x, y: x and y, map(do_result, actions)):
                LOGGER.debug('Some action went wrong at persistence process, '
                             'rollback will be performed.')
                for action in actions:
                    action.undo()
            else:
                LOGGER.info('Ingestion was successful.')
        else:
            LOGGER.info(
                'An instance of this object already exist in the database.')
    except Exception:
        LOGGER.error('Not expected error at persistence.driver')
        raise
    finally:
        session.close()
Example #2
0
 def test_imad_pair_images(self):
     '''
     Perform an imad transformation with two images
     '''
     from madmex.transformation import imad
     from madmex.mapper.data import harmonized
     from madmex.mapper.data import raster
     image1 = '/LUSTRE/MADMEX/eodata/rapideye/1147524/2012/2012-10-18/l3a/2012-10-18T191005_RE3_3A-NAC_11137283_149747.tif'
     image2 = '/LUSTRE/MADMEX/eodata/rapideye/1147524/2013/2013-09-09/l3a/1147524_2013-09-09_RE5_3A_175826.tif'
     #image2 = '/LUSTRE/MADMEX/eodata/spot/556297/2010/2010-01-26/1a/556_297_260110_SP5.img'
     gdal_format = "GTiff"
     image1_data_class =raster.Data(image1, gdal_format)
     image2_data_class = raster.Data(image2, gdal_format)
     harmonized_class = harmonized.Data(image1_data_class, image2_data_class)
     if harmonized_class:
         data_shape_harmonized = harmonized_class.get_attribute(harmonized.DATA_SHAPE)
         width, height, bands = data_shape_harmonized
         geotransform_harmonized = harmonized_class.get_attribute(harmonized.GEOTRANSFORM)
         projection_harmonized = harmonized_class.get_attribute(harmonized.PROJECTION)
         image1_data_array, image2_data_array = harmonized_class.harmonized_arrays(image1_data_class, image2_data_class)
         imad_class = imad.Transformation([image1_data_array, image2_data_array])
         imad_class.execute()
         output = os.path.join(os.path.expanduser('~'),'test_imad_pair_images')
         create_directory_path(output)
         output+= '/result_mad.tif' 
         mad_image = harmonized_class.create_from_reference(output, width, height, (bands+1), geotransform_harmonized, projection_harmonized)
         harmonized_class.write_raster(mad_image, imad_class.output)
         print 'corrlist'
         print imad_class.outcorrlist
Example #3
0
def split_shape_into_features(shape_name, destination_directory, column, name,
                              sep):
    '''
    This method will take a input shape and iterate over its features, creating
    a new shape file with each one of them. It copies all the fields and the
    same spatial reference from the original file. The created files are saved
    in the destination directory using the number of the field given. 
    '''
    driver = ogr.GetDriverByName(str('ESRI Shapefile'))
    shape = driver.Open(shape_name, 0)
    layer = shape.GetLayer()
    layer_name = layer.GetName()
    spatial_reference = layer.GetSpatialRef()
    in_feature = layer.GetNextFeature()
    shape_files = []

    while in_feature:
        encoding = 'utf-8'
        in_feature_name = in_feature.GetField(column)
        in_name = create_filename_from_string(
            in_feature.GetField(name).decode(encoding))

        final_path = destination_directory + str(in_feature.GetField(0))
        create_directory_path(final_path)
        output_name = create_filename(
            final_path, '%s__%s.shp' % (in_feature_name, in_name))
        shape_files.append(output_name)

        if os.path.exists(output_name):
            driver.DeleteDataSource(output_name)
        datasource = driver.CreateDataSource(output_name)

        outLayer = datasource.CreateLayer(layer_name,
                                          spatial_reference,
                                          geom_type=ogr.wkbPolygon)

        inLayerDefn = layer.GetLayerDefn()
        for i in range(0, inLayerDefn.GetFieldCount()):
            fieldDefn = inLayerDefn.GetFieldDefn(i)
            #LOGGER.debug(fieldDefn.GetName())
            outLayer.CreateField(fieldDefn)

        outLayerDefn = outLayer.GetLayerDefn()
        geometry = in_feature.GetGeometryRef()

        out_feature = ogr.Feature(outLayerDefn)
        out_feature.SetGeometry(geometry)

        for i in range(0, outLayerDefn.GetFieldCount()):
            out_feature.SetField(
                outLayerDefn.GetFieldDefn(i).GetNameRef(),
                in_feature.GetField(i))

        outLayer.CreateFeature(out_feature)
        out_feature.Destroy()
        in_feature.Destroy()
        in_feature = layer.GetNextFeature()
    shape.Destroy()
    datasource.Destroy()
    return shape_files
Example #4
0
 def get_thumbnail(self):
     '''
     Creates a thumbnail for the scene in true color.
     '''
     from subprocess import call
     file_1 = self.file_dictionary[_BASE %
                                   (self.get_mission(), 'B1[0-9].TIF')]
     file_2 = self.file_dictionary[_BASE %
                                   (self.get_mission(), 'B2[0-9].TIF')]
     file_3 = self.file_dictionary[_BASE %
                                   (self.get_mission(), 'B3[0-9].TIF')]
     parent_directory = get_parent(self.path)
     thumnail_directory = create_filename(parent_directory, 'thumbnail')
     create_directory_path(thumnail_directory)
     parent_directory
     filename = create_filename(thumnail_directory, 'vrt.tif')
     merge_command = [
         '/Library/Frameworks/GDAL.framework/Programs/gdalbuildvrt',
         '-separate', '-o', filename, file_3, file_2, file_1
     ]
     call(merge_command)
     thumbnail = create_filename(thumnail_directory, 'thumbnail.jpg')
     resize_command = [
         '/Library/Frameworks/GDAL.framework/Programs/gdal_translate',
         filename, '-of', 'JPEG', '-outsize', '5%', '5%', thumbnail
     ]
     call(resize_command)
Example #5
0
    def preprocess(self):
        '''
        Top of atmosphere is calculated and persisted into a file. Then a cloud
        mask is created with the given algorithm.
        '''
        solar_zenith = self.get_sensor().parser.get_attribute(
            rapideye.SOLAR_ZENITH)
        data_acquisition_date = self.get_sensor().parser.get_attribute(
            rapideye.ACQUISITION_DATE)
        solar_azimuth = self.get_sensor().parser.get_attribute(
            rapideye.SOLAR_AZIMUTH)
        geotransform = self.get_raster().get_attribute(raster.GEOTRANSFORM)
        data = self.get_raster().read_data_file_as_array()

        sun_earth_distance = calculate_distance_sun_earth(
            data_acquisition_date)
        top_of_atmosphere_data = calculate_toa_rapideye(
            calculate_rad_rapideye(data), sun_earth_distance, solar_zenith)
        top_of_atmosphere_directory = create_filename(get_parent(self.path),
                                                      'TOA')

        create_directory_path(top_of_atmosphere_directory)
        output_file = create_filename(
            top_of_atmosphere_directory,
            get_basename(self.get_files()[2]) +
            '_toa.tif')  # TODO: change [2] in self.get_files()[2]

        create_raster_from_reference(output_file,
                                     top_of_atmosphere_data,
                                     self.file_dictionary[_IMAGE],
                                     data_type=NumericTypeCodeToGDALTypeCode(
                                         numpy.float32))
        LOGGER.debug('Top of atmosphere file was created.')
        cloud_output_file = create_filename(
            top_of_atmosphere_directory,
            get_basename(self.get_files()[2]) + '_cloud.tif')

        if self.algorithm == ANOMALY_DETECTION:
            LOGGER.debug('Cloud mask by anomaly detection process.')
            clouds = self.anomaly_detection_cloud_mask(top_of_atmosphere_data,
                                                       cloud_output_file,
                                                       solar_zenith,
                                                       solar_azimuth,
                                                       geotransform)
        elif self.algorithm == TIME_SERIES:
            LOGGER.debug('Cloud mask by reference with time series process.')
            tile_id = self.get_sensor().get_attribute(TILE_ID)
            clouds = self.masking_with_time_series(data, cloud_output_file,
                                                   solar_zenith, solar_azimuth,
                                                   geotransform, tile_id)

        create_raster_from_reference(cloud_output_file,
                                     clouds,
                                     self.file_dictionary[_IMAGE],
                                     data_type=NumericTypeCodeToGDALTypeCode(
                                         numpy.float32))
        LOGGER.info('Cloud mask was created.')
Example #6
0
def train_model(X_train, X_test, y_train, y_test, output, model_name):
    model = load_model(model_name)
    persistence_directory = create_filename(output, model_name)
    create_directory_path(persistence_directory)
    model_instance = model.Model(persistence_directory)
    model_instance.fit(X_train, y_train)
    model_instance.save(persistence_directory)
    predicted = model_instance.predict(X_test)
    model_instance.create_report(
        y_test, predicted, create_filename(persistence_directory,
                                           'report.txt'))
Example #7
0
    def handle(self, **options):
        '''
        In this example command, the values that come from the user input are
        added up and the result is printed in the screen.
        '''
        output = options['output'][0]
        models = options['modelname']
        model_directory = options['modeldir'][0]

        pca_model = pca.Model(5)
        pca_model.load(create_filename(model_directory, 'pca'))

        for model_name in models:
            persistence_directory = create_filename(model_directory,
                                                    model_name)
            model = load_model(model_name)
            model_instance = model.Model(persistence_directory)
            model_instance.load(persistence_directory)
            block_size = 500
            for path in options['path']:
                image_array = open_handle(path)
                y_size = image_array.shape[1]
                x_size = image_array.shape[2]
                basename = get_basename(path)[:7]
                warnings.filterwarnings('ignore')
                final = numpy.zeros((x_size, y_size))
                import time
                start_time = time.time()
                for i in range(0, y_size, block_size):
                    if i + block_size < y_size:
                        rows = block_size
                    else:
                        rows = y_size - i
                    for j in range(0, x_size, block_size):
                        if j + block_size < x_size:
                            cols = block_size
                        else:
                            cols = x_size - j
                        step = image_array[:, i:i + rows, j:j + cols]
                        step_ravel = step.reshape(10, -1)
                        prediction = model_instance.predict(
                            pca_model.transform(numpy.transpose(step_ravel)))
                        final[i:i + rows, j:j + cols] = prediction.reshape(
                            (rows, cols))
                print("--- %s seconds ---" % (time.time() - start_time))
                create_directory_path(output)
                classification = create_filename(
                    output, '%s-%s.tif' % (basename, model_name))
                create_raster_from_reference(classification,
                                             final.reshape(x_size, y_size),
                                             path,
                                             data_type=gdal.GDT_Byte,
                                             creating_options=['COMPRESS=LZW'])
Example #8
0
 def get_thumbnail_with_path(self, thumbnail_path):
     '''
     Creates a thumbnail for the scene in true color.
     '''
     from subprocess import call
     thumnail_directory = create_file_name(thumbnail_path, 'thumbnail')
     create_directory_path(thumnail_directory)
     filename = self.file_dictionary[_BROWSE]
     
     thumbnail = create_file_name(thumnail_directory, '%s.jpg' % get_base_name(filename))
     
     resize_command = ['/Library/Frameworks/GDAL.framework/Programs/gdal_translate', filename, '-of', 'JPEG', thumbnail]
     call(resize_command)
Example #9
0
def persist_bundle(bundle, keep=False):
    """
    This function persist a bundle in both the database and the file system. It
    is responsibility of the bundle to provide information about which files
    should be persisted in the file system, and to build the database object
    that will be inserted in the database. The database is configured using the
    session maker in the connection module.
    In order to achieve its purpose, this method creates a list of the actions
    to perform. Once the list is fully populated, it calls the act method for
    each element in the list. If any of the actions in the list fails, a
    rollback is performed in all of them, the result is the same state as before
    the method was called.
    """
    destination = bundle.get_output_directory()
    create_directory_path(destination)
    actions = []
    session = SESSION_MAKER()
    try:
        if not session.query(Product).filter(Product.product_path == bundle.get_database_object().product_path).count():
            if not keep:
                LOGGER.debug("This process will move the files to a new destination.")
                for file_name in bundle.get_files():
                    actions.append(filesystem.InsertAction(file_name, destination))
            else:
                LOGGER.debug("This process will keep the original path for the files.")
            actions.append(database.InsertAction(bundle.get_database_object(), session))
            actions.append(database.InsertAction(bundle.get_information_object(), session))
            actions.append(database.InsertAction(bundle.get_features_object(), session))

            def do_result(action):
                """
                Lambda function to perform an action and return the result.
                """
                action.act()
                return action.success

            if not reduce(lambda x, y: x and y, map(do_result, actions)):
                LOGGER.debug("Some action went wrong at persistence process, " "rollback will be performed.")
                for action in actions:
                    action.undo()
            else:
                LOGGER.info("Ingestion was successful.")
        else:
            LOGGER.info("An instance of this object already exist in the database.")
    except Exception:
        LOGGER.error("Not expected error at persistence.driver")
        raise
    finally:
        session.close()
Example #10
0
def split_shape_into_features(shape_name, destination_directory, column):
    '''
    This method will take a input shape and iterate over its features, creating
    a new shape file with each one of them. It copies all the fields and the
    same spatial reference from the original file. The created files are saved
    in the destination directory using the number of the field given. 
    '''
    driver = ogr.GetDriverByName(str('ESRI Shapefile'))
    shape = driver.Open(shape_name, 0)
    layer = shape.GetLayer()
    layer_name = layer.GetName()
    spatial_reference = layer.GetSpatialRef()
    in_feature = layer.GetNextFeature()

    while in_feature:
        encoding = 'utf-8'
        in_feature_name = create_filename_from_string(in_feature.GetField(column).decode(encoding))

        final_path = destination_directory + str(in_feature.GetField(0))
        create_directory_path(final_path)
        output_name = create_file_name(final_path, in_feature_name + '.shp')

        if os.path.exists(output_name):
            driver.DeleteDataSource(output_name)
        datasource = driver.CreateDataSource(output_name)

        outLayer = datasource.CreateLayer(layer_name, spatial_reference, geom_type=ogr.wkbPolygon)

        inLayerDefn = layer.GetLayerDefn()
        for i in range(0, inLayerDefn.GetFieldCount()):
            fieldDefn = inLayerDefn.GetFieldDefn(i)
            LOGGER.debug(fieldDefn.GetName())
            outLayer.CreateField(fieldDefn)
        
        outLayerDefn = outLayer.GetLayerDefn()
        geometry = in_feature.GetGeometryRef()

        out_feature = ogr.Feature(outLayerDefn)
        out_feature.SetGeometry(geometry)

        for i in range(0, outLayerDefn.GetFieldCount()):
            out_feature.SetField(outLayerDefn.GetFieldDefn(i).GetNameRef(), in_feature.GetField(i))
  
        outLayer.CreateFeature(out_feature)
        out_feature.Destroy()
        in_feature.Destroy()
        in_feature = layer.GetNextFeature()
    shape.Destroy()
    datasource.Destroy()
Example #11
0
 def intersect(self, output_directory, geometry):
     layer = self.get_layer()
     layer_name = layer.GetName()
     spatial_reference = layer.GetSpatialRef()
     
     inSpatialRef = osr.SpatialReference()
     inSpatialRef.ImportFromEPSG(4326)
     
     
     coordTransform = osr.CoordinateTransformation(inSpatialRef, spatial_reference)
 
     geometry.Transform(coordTransform)
             
     in_feature = layer.GetNextFeature()
     layer_definition = layer.GetLayerDefn()
     field_definition = layer_definition.GetFieldDefn(0)
     column_name = field_definition.GetName() 
     shape_files = []
     create_directory_path(output_directory)
     in_layer_definition = layer.GetLayerDefn()
     output_name = output_directory       
     print output_name
     if is_file(output_name):
         self.driver.DeleteDataSource(output_name)
     data_source = self.driver.CreateDataSource(output_name)
     out_layer = data_source.CreateLayer(layer_name, spatial_reference, geom_type=ogr.wkbPolygon)
     for i in range(0, in_layer_definition.GetFieldCount()):
         fieldDefn = in_layer_definition.GetFieldDefn(i)
         out_layer.CreateField(fieldDefn)
     while in_feature:            
         outLayerDefn = out_layer.GetLayerDefn()
         feature_geometry = in_feature.GetGeometryRef()
         out_feature = ogr.Feature(outLayerDefn)            
         
         print geometry.ExportToWkt()
         print feature_geometry.ExportToWkt()
         if geometry.Intersect(feature_geometry):
             print 'It intersects!!!!'
             intersection = geometry.Intersection(feature_geometry)
             out_feature.SetGeometry(intersection)
             for i in range(0, outLayerDefn.GetFieldCount()):
                 out_feature.SetField(outLayerDefn.GetFieldDefn(i).GetNameRef(), in_feature.GetField(i))
             out_layer.CreateFeature(out_feature)
         out_feature = None
         in_feature = None
         in_feature = layer.GetNextFeature()
     self.close()
     return Data(output_name)
Example #12
0
    def get_thumbnail_with_path(self, thumbnail_path):
        '''
        Creates a thumbnail for the scene in true color.
        '''
        from subprocess import call
        thumnail_directory = create_filename(thumbnail_path, 'thumbnail')
        create_directory_path(thumnail_directory)
        filename = self.file_dictionary[_BROWSE]

        thumbnail = create_filename(thumnail_directory,
                                    '%s.jpg' % get_basename(filename))

        resize_command = [
            '/Library/Frameworks/GDAL.framework/Programs/gdal_translate',
            filename, '-of', 'JPEG', thumbnail
        ]
        call(resize_command)
Example #13
0
 def get_thumbnail(self):
     '''
     Creates a thumbnail for the scene in true color.
     '''
     from subprocess import call
     file_1 = self.file_dictionary[_BASE % (self.get_mission(), 'B1[0-9].TIF')]
     file_2 = self.file_dictionary[_BASE % (self.get_mission(), 'B2[0-9].TIF')]
     file_3 = self.file_dictionary[_BASE % (self.get_mission(), 'B3[0-9].TIF')]
     parent_directory = get_parent(self.path)
     thumnail_directory = create_file_name(parent_directory, 'thumbnail')
     create_directory_path(thumnail_directory)
     parent_directory
     filename = create_file_name(thumnail_directory, 'vrt.tif')
     merge_command = ['/Library/Frameworks/GDAL.framework/Programs/gdalbuildvrt', '-separate', '-o', filename, file_3, file_2, file_1]
     call(merge_command)
     thumbnail = create_file_name(thumnail_directory, 'thumbnail.jpg')
     resize_command = ['/Library/Frameworks/GDAL.framework/Programs/gdal_translate', filename, '-of', 'JPEG', '-outsize', '5%', '5%', thumbnail]
     call(resize_command)
Example #14
0
 def handle(self, **options):
     '''
     In this example command, the values that come from the user input are
     added up and the result is printed in the screen.
     '''
     output = options['output'][0]
     zip = options['path'][0]
     basename = get_basename(zip)
     aux_name = create_filename(output, 'aux_%s' % basename)
     real_name = create_filename(output, basename)
     with ZipFile(zip, 'r') as unzipped:
         unzipped.extractall(create_filename(output, 'aux_%s' % basename))
     path = create_filename(aux_name, 'trend_tiles')
     tifs = get_contents_from_folder(path)
     create_directory_path(real_name)
     for tif in tifs:
         source = create_filename(path, tif)
         target = create_filename(real_name, tif)
         tile_map(source, target, 4000, 4000, 2, 2)
     remove_directory(aux_name)
Example #15
0
 def split(self, output_directory, column=0):
     '''
     This method will take a input shape and iterate over its features, creating
     a new shape file with each one of them. It copies all the fields and the
     same spatial reference from the original file. The created files are saved
     in the destination directory using the number of the field given. 
     '''
     layer = self.get_layer()
     layer_name = layer.GetName()
     spatial_reference = layer.GetSpatialRef()
     in_feature = layer.GetNextFeature()
     layer_definition = layer.GetLayerDefn()
     field_definition = layer_definition.GetFieldDefn(0)
     column_name = field_definition.GetName() 
     shape_files = []
     create_directory_path(output_directory)
     in_layer_definition = layer.GetLayerDefn()
     while in_feature:            
         in_feature_name = in_feature.GetField(column_name)
         output_name = create_filename(output_directory, '%s.shp' % in_feature_name)
         shape_files.append(output_name)
         if is_file(output_name):
             self.driver.DeleteDataSource(output_name)
         data_source = self.driver.CreateDataSource(output_name)
         out_layer = data_source.CreateLayer(layer_name, spatial_reference, geom_type=ogr.wkbPolygon)
         for i in range(0, in_layer_definition.GetFieldCount()):
             fieldDefn = in_layer_definition.GetFieldDefn(i)
             out_layer.CreateField(fieldDefn)
         outLayerDefn = out_layer.GetLayerDefn()
         geometry = in_feature.GetGeometryRef()
         out_feature = ogr.Feature(outLayerDefn)
         out_feature.SetGeometry(geometry)
         for i in range(0, outLayerDefn.GetFieldCount()):
             out_feature.SetField(outLayerDefn.GetFieldDefn(i).GetNameRef(), in_feature.GetField(i))
         out_layer.CreateFeature(out_feature)
         
         out_feature = None
         in_feature = None
         in_feature = layer.GetNextFeature()
     self.close()
     return [Data(filename) for filename in shape_files]
Example #16
0
def create_shape_from_json(id, json, output_directory):
    '''
    Given a json string containing coordinates, this method creates a shape file.
    '''
    create_directory_path(output_directory)
    filename = create_filename(output_directory, '%s.shp' % id)
    shape = Data(filename)
    if is_file(filename):
        shape.driver.DeleteDataSource(filename)
    data_source = shape.driver.CreateDataSource(filename)
    spatial_reference = osr.SpatialReference()
    spatial_reference.ImportFromEPSG(4326)    
    layer = data_source.CreateLayer(str('layer'), spatial_reference, geom_type=ogr.wkbPolygon)
    layer.CreateField(ogr.FieldDefn(str('id'), ogr.OFTString))
    feature = ogr.Feature(layer.GetLayerDefn())
    feature.SetField(str('id'), str(id))
    geometry = ogr.CreateGeometryFromJson(str(json))
    feature.SetGeometry(geometry)
    layer.CreateFeature(feature)
    shape.close()
    return shape
Example #17
0
    def preprocess(self):
        '''
        Top of atmosphere is calculated and persisted into a file. Then a cloud
        mask is created with the given algorithm.
        '''
        solar_zenith = self.get_sensor().parser.get_attribute(rapideye.SOLAR_ZENITH)
        data_acquisition_date = self.get_sensor().parser.get_attribute(rapideye.ACQUISITION_DATE)
        solar_azimuth = self.get_sensor().parser.get_attribute(rapideye.SOLAR_AZIMUTH)
        geotransform = self.get_raster().get_attribute(raster.GEOTRANSFORM)
        data = self.get_raster().read_data_file_as_array()

        sun_earth_distance = calculate_distance_sun_earth(data_acquisition_date)
        top_of_atmosphere_data = calculate_toa_rapideye(calculate_rad_rapideye(data), sun_earth_distance, solar_zenith)
        top_of_atmosphere_directory = create_file_name(get_parent(self.path), 'TOA')

        create_directory_path(top_of_atmosphere_directory)
        output_file = create_file_name(top_of_atmosphere_directory, get_base_name(self.get_files()[2]) + '_toa.tif')  # TODO: change [2] in self.get_files()[2] 

        create_raster_from_reference(output_file,
                                     top_of_atmosphere_data,
                                     self.file_dictionary[_IMAGE],
                                     data_type=NumericTypeCodeToGDALTypeCode(numpy.float32)
                                     )
        LOGGER.debug('Top of atmosphere file was created.')
        cloud_output_file = create_file_name(top_of_atmosphere_directory, get_base_name(self.get_files()[2]) + '_cloud.tif')

        if self.algorithm == ANOMALY_DETECTION:            
            LOGGER.debug('Cloud mask by anomaly detection process.')
            clouds = self.anomaly_detection_cloud_mask(top_of_atmosphere_data, cloud_output_file, solar_zenith, solar_azimuth, geotransform)
        elif self.algorithm == TIME_SERIES:
            LOGGER.debug('Cloud mask by reference with time series process.')
            tile_id = self.get_sensor().get_attribute(TILE_ID)
            clouds = self.masking_with_time_series(data, cloud_output_file, solar_zenith, solar_azimuth, geotransform, tile_id)

        create_raster_from_reference(cloud_output_file,
                     clouds,
                     self.file_dictionary[_IMAGE],
                     data_type=NumericTypeCodeToGDALTypeCode(numpy.float32)
                     )
        LOGGER.info('Cloud mask was created.')
Example #18
0
 def test_imad_pair_images(self):
     '''
     Perform an imad transformation with two images
     '''
     from madmex.transformation import imad
     from madmex.mapper.data import harmonized
     from madmex.mapper.data import raster
     image1 = '/LUSTRE/MADMEX/eodata/rapideye/1147524/2012/2012-10-18/l3a/2012-10-18T191005_RE3_3A-NAC_11137283_149747.tif'
     image2 = '/LUSTRE/MADMEX/eodata/rapideye/1147524/2013/2013-09-09/l3a/1147524_2013-09-09_RE5_3A_175826.tif'
     #image2 = '/LUSTRE/MADMEX/eodata/spot/556297/2010/2010-01-26/1a/556_297_260110_SP5.img'
     gdal_format = "GTiff"
     image1_data_class = raster.Data(image1, gdal_format)
     image2_data_class = raster.Data(image2, gdal_format)
     harmonized_class = harmonized.Data(image1_data_class,
                                        image2_data_class)
     if harmonized_class:
         data_shape_harmonized = harmonized_class.get_attribute(
             harmonized.DATA_SHAPE)
         width, height, bands = data_shape_harmonized
         geotransform_harmonized = harmonized_class.get_attribute(
             harmonized.GEOTRANSFORM)
         projection_harmonized = harmonized_class.get_attribute(
             harmonized.PROJECTION)
         image1_data_array, image2_data_array = harmonized_class.harmonized_arrays(
             image1_data_class, image2_data_class)
         imad_class = imad.Transformation(
             [image1_data_array, image2_data_array])
         imad_class.execute()
         output = os.path.join(os.path.expanduser('~'),
                               'test_imad_pair_images')
         create_directory_path(output)
         output += '/result_mad.tif'
         mad_image = harmonized_class.create_from_reference(
             output, width, height, (bands + 1), geotransform_harmonized,
             projection_harmonized)
         harmonized_class.write_raster(mad_image, imad_class.output)
         print 'corrlist'
         print imad_class.outcorrlist
Example #19
0
    def handle(self, **options):
        '''
        This process will call the change detection process from a set of two
        individual images. It will perform the harmonization and the multivariate
        alteration detection on the images. It will then perform a maximum
        correlation factor on them and work with the resulting bands.
        '''
        image_a = options['ima'][0]
        image_b = options['imb'][0]
        output_image = options['output'][0]

        LOGGER.info('Image %s will be compared against image %s. Output will be available' \
              ' at %s.', image_a, image_b, output_image)
        
        gdal_format = "GTiff"
         
        image_a_data_class = raster.Data(image_a, gdal_format)
        image_b_data_class = raster.Data(image_b, gdal_format)

        # TODO : remove references to class harmonized
        harmonized_class = harmonized.Data(image_a_data_class, image_b_data_class)
        
        #band1 = image_a_data_class.GetRasterBand(1)
        #band1 = band1.ReadAsArray(0, 0,image_a_data_class.RasterXSize,image_a_data_class.RasterYSize).astype(float)
        
        #print(band1)
 
        
        if harmonized_class:
            #data_shape_harmonized = harmonized_class.get_attribute(harmonized.DATA_SHAPE)
            #width, height, bands = data_shape_harmonized
            #geotransform_harmonized = harmonized_class.get_attribute(raster.GEOTRANSFORM)
            #projection_harmonized = harmonized_class.get_attribute(harmonized.PROJECTION)    
            
            image_a_data_array, image_b_data_array = harmonized_class.harmonized_arrays(image_a_data_class, image_b_data_class)            
               
            imad_class = imad.Transformation([image_a_data_array, image_b_data_array])
            imad_class.execute()
            
            mad_result = imad_class.output
            LOGGER.debug('mad_result.shape: %s', mad_result.shape)
            
            create_directory_path(getattr(SETTINGS, 'TEST_FOLDER'))
            
            mad_output_file = create_filename(getattr(SETTINGS, 'TEST_FOLDER'), 'mad.tif')
            create_raster_from_reference(mad_output_file, mad_result,image_a)
           
           
            maf_class = maf.Transformation(imad_class.output)
            
            maf_class.execute()
            
            
            
            maf_result = maf_class.output
            
            pdf_file = create_filename(getattr(SETTINGS, 'TEST_FOLDER'), 'maf_pdf.png')

            thresholds = calc_threshold_grid(maf_result, pdf_file)
            
            class_result = recode_classes_grid(maf_result, thresholds)
            

           
            LOGGER.debug('maf_result.shape: %s', maf_result.shape)
            LOGGER.debug('class_result.shape: %s', class_result.shape)

            maf_outputfile = create_filename(getattr(SETTINGS, 'TEST_FOLDER'), 'maf.tif')
            class_outputfile = create_filename(getattr(SETTINGS, 'TEST_FOLDER'), 'class.tif') 

            
            
            create_raster_from_reference(maf_outputfile, maf_result, image_a)
            create_raster_from_reference(class_outputfile, class_result, image_a)
            
            
            
            
            print 'Output written in: %s' % mad_output_file
            print 'Shape is ', imad_class.output.shape
Example #20
0
    def handle(self, **options):
        """
        This is the code that does the ingestion.        
        indexes --path /LUSTRE/MADMEX/staging/2016_tasks/Humedales_for_ctroche/LT/Landsat_2000_2008/L5_021_047_2000_022_sr/ --shape /LUSTRE/MADMEX/staging/2016_tasks/Humedales_for_ctroche/LT/AE_LT_new.shp --output 
        """
        path = options["path"][0]

        for mask_file in os.listdir(path):
            if mask_file.endswith("_cfmask.tif"):
                print mask_file
                basename = mask_file.replace("_cfmask.tif", "")
                print basename
                cloud_file_name = mask_file

        LOGGER.info("Calculating indexes for Landsat scenes.")

        band_name = basename + "_sr_band%s.tif"

        final_path = options["output"][0]
        create_directory_path(final_path)
        cloud_file = create_file_name(path, cloud_file_name)
        cloud_array = open_handle(cloud_file)
        cloud_mask = cloud_array == 4

        LOGGER.debug("Recognize sensor depending on file name.")

        if "L8" in path:
            LOGGER.debug("Landsat 8 scene was detected.")
            blue_file = create_file_name(path, band_name % BLUE_L8)
            green_file = create_file_name(path, band_name % GREEN_L8)
            red_file = create_file_name(path, band_name % RED_L8)
            nir_file = create_file_name(path, band_name % NIR_L8)
            swir_file = create_file_name(path, band_name % SWIR_L8)
        else:
            LOGGER.debug("Landsat 4, 5 or scene was detected.")
            blue_file = create_file_name(path, band_name % BLUE)
            green_file = create_file_name(path, band_name % GREEN)
            red_file = create_file_name(path, band_name % RED)
            nir_file = create_file_name(path, band_name % NIR)
            swir_file = create_file_name(path, band_name % SWIR)

        LOGGER.debug("Loading bands of interest.")

        green_array = open_handle(green_file)
        red_array = open_handle(red_file)
        nir_array = open_handle(nir_file)
        swir_array = open_handle(swir_file)

        LOGGER.debug("Calculating indexes.")

        ndvi_array = calculate_index(nir_array, red_array)
        mndwi_array = calculate_index(green_array, swir_array)
        ndwig_array = calculate_index(nir_array, swir_array)
        ndwim_array = calculate_index(green_array, nir_array)

        LOGGER.debug("Setting cloud mask values to -999.")

        ndvi_array[cloud_mask] = -999
        mndwi_array[cloud_mask] = -999
        ndwig_array[cloud_mask] = -999
        ndwim_array[cloud_mask] = -999

        LOGGER.debug("Creating files for indexes.")

        ndvi_final_file = create_file_name(final_path, basename + "_ndvi.tif")
        mndwi_final_file = create_file_name(final_path, basename + "_mndwi.tif")
        ndwig_final_file = create_file_name(final_path, basename + "_ndwig.tif")
        ndwim_final_file = create_file_name(final_path, basename + "_ndwim.tif")

        ndvi_clipped_file = create_file_name(final_path, basename + "_ndvi_clipped.tif")
        mndwi_clipped_file = create_file_name(final_path, basename + "_mndwi_clipped.tif")
        ndwig_clipped_file = create_file_name(final_path, basename + "_ndwig_clipped.tif")
        ndwim_clipped_file = create_file_name(final_path, basename + "_ndwim_clipped.tif")

        ndvi_file = create_file_name(final_path, basename + "_ndvi_pre.tif")
        mndwi_file = create_file_name(final_path, basename + "_mndwi_pre.tif")
        ndwig_file = create_file_name(final_path, basename + "_ndwig_pre.tif")
        ndwim_file = create_file_name(final_path, basename + "_ndwim_pre.tif")

        files = [ndvi_file, mndwi_file, ndwig_file, ndwim_file]
        clipped_files = [ndvi_clipped_file, mndwi_clipped_file, ndwig_clipped_file, ndwim_clipped_file]
        final_files = [ndvi_final_file, mndwi_final_file, ndwig_final_file, ndwim_final_file]

        LOGGER.debug("Writing information to files.")

        create_raster_from_reference(ndvi_file, ndvi_array, green_file)
        create_raster_from_reference(mndwi_file, mndwi_array, green_file)
        create_raster_from_reference(ndwig_file, ndwig_array, green_file)
        create_raster_from_reference(ndwim_file, ndwim_array, green_file)

        LOGGER.debug("Deleting arrays to release memory.")

        del ndvi_array
        del mndwi_array
        del ndwig_array
        del ndwim_array
        del cloud_array

        LOGGER.debug("Reference rgb file creation.")

        rgb_file = create_file_name(final_path, basename + "_rgb.tif")
        merge_command = [
            "/Library/Frameworks/GDAL.framework/Programs/gdalbuildvrt",
            "-separate",
            "-o",
            rgb_file,
            red_file,
            green_file,
            blue_file,
        ]
        call(merge_command)
        shape = options["shape"][0]

        LOGGER.debug("Cookie cut files using the given shape.")

        for i in range(4):
            clip_command = [
                "/Library/Frameworks/GDAL.framework/Programs/gdalwarp",
                "-crop_to_cutline",
                "-cutline",
                shape,
                files[i],
                clipped_files[i],
            ]
            call(clip_command)

        for i in range(4):
            aux_array = open_handle(clipped_files[i])
            aux_array[(aux_array == 0)] = -9999
            create_raster_from_reference(final_files[i], aux_array, ndvi_clipped_file)

        if not options["debug"]:
            LOGGER.info("Remove auxiliary files.")
            os.remove(ndvi_clipped_file)
            os.remove(mndwi_clipped_file)
            os.remove(ndwig_clipped_file)
            os.remove(ndwim_clipped_file)
            os.remove(ndvi_file)
            os.remove(mndwi_file)
            os.remove(ndwig_file)
            os.remove(ndwim_file)

        print "Done"
Example #21
0
    def handle(self, **options):
        '''
        In this example command, the values that come from the user input are
        added up and the result is printed in the screen.
        '''
        shape_name = options['shape'][0]
        destination = options['dest'][0]
        paths = options['path']

        years = []
        for path in paths:
            years.append(self.get_year_from_path(path))

        if os.path.exists(shape_name):
            LOGGER.info('The file %s was found.' % shape_name)
        shape_files = split_shape_into_features(shape_name, destination,
                                                str('id'), str('nombre'), '__')
        process_launcher = LocalProcessLauncher()

        import time
        start_time = time.time()

        cover_array = []
        tth_array = []
        cover_file = 'cover-stats.json'
        tth_name = 'tth-stats.json'
        json_directory = create_filename(destination, 'cover_stats')
        json_file = create_filename(json_directory, cover_file.lower())
        tth_file = create_filename(json_directory, tth_name.lower())

        for shape_file in shape_files:
            shape_name = get_basename(shape_file).split('__')
            anp_id = shape_name[0]
            anp_name = shape_name[1]
            basename = '%s.tif' % anp_name

            dataframe = DataFrame(index=years,
                                  columns=[0, 1, 2, 3, 4, 5, 6, 7, 8])

            create_directory_path(json_directory)

            print shape_file
            for path in paths:
                #pixel_resolution, dataset = pixel_info(path)

                year = self.get_year_from_path(path)
                raster_dir = create_filename(
                    create_filename(destination, 'raster'), year)
                create_directory_path(raster_dir)
                raster_file = create_filename(raster_dir, basename)

                shell_command = 'gdalwarp -ot Byte -co COMPRESS=LZW -cutline %s -crop_to_cutline %s %s' % (
                    shape_file, path, raster_file)
                print shell_command
                print process_launcher.execute(shell_command)
                raster_array = open_handle(raster_file)

                ds = gdal.Open(raster_file)
                geotransform = ds.GetGeoTransform()
                x_resolution = geotransform[1]
                y_resolution = geotransform[5]
                pixel_area = abs(x_resolution * y_resolution)

                unique_values = numpy.unique(raster_array, return_counts=True)

                indexes = unique_values[0]
                counts = unique_values[1]

                for i in range(len(indexes)):
                    key = indexes[i]
                    dataframe.set_value(year, key,
                                        area(int(counts[i]), pixel_area))
            dataframe = dataframe.sort_index()
            columns = list(dataframe.columns.values)
            index = list(dataframe.index.values)

            print dataframe
            cover_array.append(
                self.dataframe_to_json(dataframe, anp_id, anp_name))

            tth_dataframe = DataFrame(columns=columns)

            for i in range(len(index) - 1):
                label = '%s-%s' % (index[i], index[i + 1])
                tth_column = calculate_thh(dataframe.ix[i],
                                           dataframe.ix[i + 1],
                                           int(index[i + 1]) - int(index[i]))
                for j in range(len(tth_column)):
                    tth_dataframe.set_value(label, j, tth_column[j])
            tth_dataframe = tth_dataframe.sort_index()

            print tth_dataframe

            tth_array.append(
                self.dataframe_to_json(tth_dataframe, anp_id, anp_name))

        self.json_to_file(json_file, cover_array)
        self.json_to_file(tth_file, tth_array)

        print("--- %s seconds ---" % (time.time() - start_time))
Example #22
0
    def handle(self, **options):
        image_to_be_classified = options['image'][0]
        #landmask_path = options['landmask_path'][0]
        outlier = options['outlier'][0]
        folder_results =  getattr(SETTINGS, 'BIG_FOLDER')
        shutil.copy(image_to_be_classified, folder_results)
        image_to_be_classified = folder_results +  get_basename_of_file(image_to_be_classified)
        
        landmask_path = getattr(SETTINGS, 'LANDMASK_PATH')
        image_for_segmentation = '/results/' +  get_basename_of_file(image_to_be_classified)
        LOGGER.info('Starting segmentation with: %s' % image_for_segmentation)   
        val_t = 50
        val_s = 0.7
        val_c = 0.3
        val_xt = 40
        val_rows = 625
        val_tile = True
        val_mp = True
        
        folder_and_bind_segmentation = getattr(SETTINGS, 'FOLDER_SEGMENTATION')
        folder_and_bind_license = getattr(SETTINGS, 'FOLDER_SEGMENTATION_LICENSE')
        folder_and_bind_image = getattr(SETTINGS, 'BIG_FOLDER_HOST')
        

        LOGGER.info('starting segmentation')
        command = 'run_container'
        hosts_from_command = get_host_from_command(command)
        LOGGER.info('The command to be executed is %s in the host %s' % (command, hosts_from_command[0].hostname))
        remote = RemoteProcessLauncher(hosts_from_command[0])
        arguments = 'docker  run --rm -v ' + folder_and_bind_segmentation + ' -v ' + folder_and_bind_license + ' -v ' + folder_and_bind_image + ' madmex/segmentation python /segmentation/segment.py ' + image_for_segmentation
        arguments+=  ' -t ' + str(val_t) + ' -s ' + str(val_s) + ' -c ' + str(val_c) + ' --tile ' + str(val_tile) + ' --mp ' + str(val_mp) + ' --xt ' + str(val_xt) + ' --rows ' + str(val_rows)
        remote.execute(arguments)
        
        LOGGER.info('Finished segmentation')
        
        
        image_segmentation_file = image_to_be_classified + '_' + str(val_t) + '_' + ''.join(str(val_s).split('.'))+ '_' + ''.join(str(val_c).split('.')) + '.tif'
        LOGGER.info('Starting vectorization of segmentation file: %s' % image_segmentation_file)
        image_segmentation_shp_file = image_segmentation_file + '.shp'
        vectorize_raster(image_segmentation_file, 1, image_segmentation_shp_file, 'objects', 'id')
        LOGGER.info('Finished vectorization: %s' % image_segmentation_shp_file)
        
        gdal_format = 'GTiff'
        image_to_be_classified_class = raster.Data(image_to_be_classified, gdal_format)
        width, height, bands = image_to_be_classified_class.get_attribute(raster.DATA_SHAPE)
        LOGGER.info('Identifying landmask %s' % landmask_path)
        bundle = _get_bundle_from_path(landmask_path)
        if bundle:
            LOGGER.info('Directory %s is a %s bundle', landmask_path, bundle.get_name())
            LOGGER.info('Rasterizing vector shape')
            options_to_create = new_options_for_create_raster_from_reference(image_to_be_classified_class.metadata, raster.DATA_SHAPE, (width,height, 1), {})
            dataset_landmask_rasterized = create_raster_tiff_from_reference(image_to_be_classified_class.metadata, '', None, options_to_create)
            bundle.rasterize(dataset_landmask_rasterized, [1], [1]) #the rasterized process changes the dataset
            options_to_create = new_options_for_create_raster_from_reference(image_to_be_classified_class.metadata, raster.GDAL_CREATE_OPTIONS, ['COMPRESS=LZW'], {})            
            image = folder_results + 'landmask_rasterized.tif'
            create_raster_tiff_from_reference(image_to_be_classified_class.metadata, image, dataset_landmask_rasterized.ReadAsArray(), options_to_create)
            LOGGER.info('Finished rasterizing vector shape')        
        
        LOGGER.info('Polygonizing the landmask rasterized')
        landmask_folder = folder_results  + 'landmask_from_rasterize/'
        layer_landmask = 'landmask'
        landmask_file = landmask_folder + layer_landmask + '.shp'  
        layer_landmask = 'landmask'

        vectorize_raster(folder_results + 'landmask_rasterized.tif', 1, landmask_folder, layer_landmask, 'id')
        LOGGER.info('Folder of polygon: %s' % landmask_folder)
        image_segmentation_file_class = raster.Data(image_segmentation_file, gdal_format)
        LOGGER.info('Reading array of %s' % image_for_segmentation)
        array_sg_raster = image_segmentation_file_class.read_data_file_as_array()
        unique_labels_for_objects = numpy.unique(array_sg_raster)
        LOGGER.info('Calculating zonal stats for :%s' % image_to_be_classified)
        LOGGER.info('Reading array of %s' % image_to_be_classified)
        array_image_to_be_classified = image_to_be_classified_class.read_data_file_as_array()       
        array_zonal_statistics = calculate_zonal_statistics(array_image_to_be_classified, array_sg_raster, unique_labels_for_objects)
        LOGGER.info('finished zonal statistics')

        array_zonal_statistics_labeled = append_labels_to_array(array_zonal_statistics, unique_labels_for_objects)
        LOGGER.info('Shape of array of zonal statistics labeled %s %s' % (array_zonal_statistics_labeled.shape[0], array_zonal_statistics_labeled.shape[1]))
        LOGGER.info('Building data frame')
        dataframe_zonal_statistics = create_names_of_dataframe_from_filename(build_dataframe_from_array(array_zonal_statistics_labeled.T), array_zonal_statistics_labeled.shape[0], get_basename_of_file(image_to_be_classified))
        LOGGER.info('Filling NaN with zeros')            
        dataframe_zonal_statistics = dataframe_zonal_statistics.fillna(0)
        file_name = folder_results + 'dataframe_zonal_statistics'
        dataframe_zonal_statistics.to_csv(file_name, sep='\t', encoding='utf-8', index = False)
        
        
        LOGGER.info('Working with the training data')
        training_data_file = getattr(SETTINGS, 'TRAINING_DATA')
        LOGGER.info('Clipping training_data_file: %s with: %s' % (training_data_file, landmask_file))
        
        training_data_file_clipped = folder_results  + get_basename_of_file(training_data_file) + '_cropped_subprocess_call.tif'
        command = [
                    'gdalwarp', '-cutline', landmask_file,
                    '-crop_to_cutline', '-of', 'GTiff','-co', 'compress=lzw', '-co', 'tiled=yes', training_data_file, training_data_file_clipped
                    ]
        subprocess.call(command)
        LOGGER.info('Finished clipping of training data file')
            
        LOGGER.info('Starting warping of file: %s according to %s ' % (training_data_file_clipped, image_segmentation_file))
        dataset_warped_training_data_file = warp_raster_from_reference(training_data_file_clipped, image_segmentation_file_class.data_file, None)
        LOGGER.info('Starting resizing of array of training file: %s' % training_data_file_clipped)

        array_resized_and_warped_training_data_file = get_array_resized_from_reference_dataset(dataset_warped_training_data_file, image_segmentation_file_class.data_file)

        import gdal
        training_data_file_resized_and_warped =  folder_results + get_basename_of_file(training_data_file) + '_resized_and_warped.tif'
        options_to_create = new_options_for_create_raster_from_reference(image_to_be_classified_class.metadata,  raster.GDAL_CREATE_OPTIONS, ['TILED=YES', 'COMPRESS=LZW', 'INTERLEAVE=BAND'], {})
        create_raster_tiff_from_reference(image_to_be_classified_class.metadata, training_data_file_resized_and_warped, array_resized_and_warped_training_data_file, options_to_create, data_type = gdal.GDT_Int32)
        LOGGER.info('Starting resampling')
        array_training_data_resampled = resample_numpy_array(array_resized_and_warped_training_data_file, width, height, interpolation = 'nearest')
        training_data_file_resampled = folder_results + get_basename_of_file(training_data_file) + '_resampled_from_resized_and_warped.tif'        
        create_raster_tiff_from_reference(image_to_be_classified_class.metadata, training_data_file_resampled, array_training_data_resampled, options_to_create, data_type = gdal.GDT_Int32)
 
        LOGGER.info('Calculating zonal histograms for file: %s according to: %s' % (training_data_file_resampled, image_segmentation_file))
        unique_classes = numpy.unique(array_training_data_resampled)
        array_of_distribution_of_classes_per_object_segmentation = calculate_zonal_histograms(array_training_data_resampled, unique_classes, array_sg_raster, unique_labels_for_objects)
        LOGGER.info('Shape of zonal histogram: %s %s' % (array_of_distribution_of_classes_per_object_segmentation.shape[0], array_of_distribution_of_classes_per_object_segmentation.shape[1]))
        array_training_data_resampled = None
        LOGGER.info('Getting objects that have a class of at least .75 proportion within zonal histogram')
        dataframe_of_objects_for_training_data = get_objects_by_relative_proportion_from_raster_as_dataframe(array_of_distribution_of_classes_per_object_segmentation, unique_labels_for_objects, unique_classes, ["id", "given"], 0.75)
        file_name = folder_results + 'dataframe_of_objects_for_training_data'
        dataframe_of_objects_for_training_data.to_csv(file_name, sep='\t', encoding='utf-8', index = False)
        LOGGER.info('Number of rows and columns of dataframe of pure objects of training data %s %s' % (len(dataframe_of_objects_for_training_data.index), len(dataframe_of_objects_for_training_data.columns) ))        
        array_of_distribution_of_classes_per_object_segmentation = None
        dataframe_of_objects_for_training_data = None
        dataframe_of_objects_for_training_data = pandas.read_csv(file_name, sep='\t')
        LOGGER.info('Joining dataframe of dataframe zonal statistics and dataframe of objects of training data')
        dataframe_all_joined_classified = join_dataframes_by_column_name([dataframe_zonal_statistics, dataframe_of_objects_for_training_data], 'id')
        LOGGER.info('Number of rows and columns of dataframe joined %s %s' % (len(dataframe_all_joined_classified.index), len(dataframe_all_joined_classified.columns) ))        
            
        if outlier == 'True':
                LOGGER.info('Starting outlier elimination with dataframe of zonal statistics and dataframe of pure objects of training data')
                LOGGER.info('Starting principal component analysis')
                array_reduced_pca = reduce_dimensionality(dataframe_all_joined_classified, .95, ['id', 'given'])
                LOGGER.info('Shape of reduced array of zonal statistics and pure objects of training data by pca: %s %s' %(array_reduced_pca.shape[0], array_reduced_pca.shape[1]) )
                labels_of_objects_reduced_dataframe = dataframe_all_joined_classified['id'].values
                LOGGER.info('Appending labels')
                array_reduced_pca_labeled = append_labels_to_array(array_reduced_pca.T, labels_of_objects_reduced_dataframe)
                LOGGER.info('Shape of array reduced by pca and labeled: %s %s' %(array_reduced_pca_labeled.shape[0], array_reduced_pca_labeled.shape[1]))
                LOGGER.info('Building data frame')
                dataframe_reduced_pca_file = folder_results + 'dataframe_joined_for_zonal_statistics_and_pure_objects_of_training_data_reduced_by_pca'
                dataframe_reduced_pca = create_names_of_dataframe_from_filename(build_dataframe_from_array(array_reduced_pca_labeled.T), array_reduced_pca_labeled.shape[0], get_basename_of_file(dataframe_reduced_pca_file))
                dataframe_reduced_pca.to_csv(dataframe_reduced_pca_file, sep=',', encoding='utf-8', index = False)
                LOGGER.info('Starting with elimination of outliers')
                LOGGER.info('Joining reduced dataframe by pca with object ids and dataframe of pure objects of training data')
                dataframe_reduced_pca_with_classes= join_dataframes_by_column_name([dataframe_reduced_pca, dataframe_of_objects_for_training_data], 'id')
                LOGGER.info('Number of rows and columns of dataframe joined: (%s,%s)' %(len(dataframe_reduced_pca_with_classes.index), len(dataframe_reduced_pca_with_classes.columns)))
                dataframe_reduced_pca_with_classes.to_csv(dataframe_reduced_pca_file + 'classes', sep = ',', encoding = 'utf8', index = False)
                unique_classes = numpy.unique(dataframe_of_objects_for_training_data['given'].values)
                object_ids_outlier_elimination = outlier_elimination_for_dataframe(dataframe_reduced_pca_with_classes, 'id', 'given', 'id', 3, unique_classes, 0.15)
                object_ids_outlier_elimination_file = folder_results + 'dataframe_object_ids_outlier_elimination'
                object_ids_outlier_elimination.to_csv(object_ids_outlier_elimination_file, sep = ',', encoding = 'utf-8', index = False)
                LOGGER.info('Joining all dataframes according to ids of outlier elimination ')
                dataframe_all_joined_classified = join_dataframes_by_column_name([object_ids_outlier_elimination, dataframe_all_joined_classified], 'id')
                LOGGER.info('Number of rows and columns of dataframe joined classified: (%s,%s)' %(len(dataframe_all_joined_classified.index), len(dataframe_all_joined_classified.columns)))
            
        dataframe_zonal_statistics['given'] = '?'
        LOGGER.info('Number of rows and columns of dataframe for classifying: (%s,%s)' %(len(dataframe_zonal_statistics.index), len(dataframe_zonal_statistics.columns)))
        index_of_objects_not_id_zero = dataframe_zonal_statistics['id'] > 0
        dataframe_all_joined_for_classifying = dataframe_zonal_statistics[index_of_objects_not_id_zero]
        LOGGER.info('Number of rows and columns of dataframe for classifying after removing object with id zero: (%s,%s)' %(len(dataframe_all_joined_for_classifying.index), len(dataframe_all_joined_for_classifying.columns)))
            
        LOGGER.info('Generating data file')
        dataframe_all_joined_classified_file = folder_results + 'C5.data'
        dataframe_all_joined_classified.to_csv(dataframe_all_joined_classified_file, sep = ',', encoding = 'utf-8', index = False, header = False)
        LOGGER.info('Generating cases file')
        dataframe_all_joined_for_classifying_file = folder_results + 'C5.cases'
        dataframe_all_joined_for_classifying.to_csv(dataframe_all_joined_for_classifying_file, sep = ',', encoding = 'utf-8', index = False, header = False)
        LOGGER.info('Generating names file')
        unique_classes = numpy.unique(dataframe_all_joined_classified['given'].values)
        name_namesfile = folder_results + 'C5.names'
        generate_namesfile(dataframe_all_joined_classified.columns, unique_classes,name_namesfile, 'id', 'given')


        command = 'run_container'
        hosts_from_command = get_host_from_command(command)
        LOGGER.info('The command to be executed is %s in the host %s' % (command, hosts_from_command[0].hostname))
        remote = RemoteProcessLauncher(hosts_from_command[0])
        folder_and_bind_c5 = getattr(SETTINGS, 'BIG_FOLDER_HOST')
        arguments = 'docker  run --rm -v ' + folder_and_bind_c5  + ' madmex/c5_execution ' + 'c5.0 -b -f /results/C5'
        LOGGER.info('Beginning C5') 
        remote.execute(arguments)

        LOGGER.info('Begining predict')
        arguments = 'docker  run --rm -v ' + folder_and_bind_c5  + ' madmex/c5_execution ' + 'predict -f /results/C5'
        remote = RemoteProcessLauncher(hosts_from_command[0])
        output = remote.execute(arguments, True)
        
        LOGGER.info('Writing C5 result to csv')
        C5_result = write_C5_result_to_csv(output, folder_results)
        LOGGER.info('Using result of C5: %s for generating land cover shapefile and raster image' % C5_result)
        LOGGER.info('Using result of C5 for generating land cover shapefile and raster image')        
        C5_result = folder_results + 'C5_result.csv'
        dataframe_c5_result = pandas.read_csv(C5_result)
        FORMAT =  'ESRI Shapefile'
        image_segmentation_shp_class = vector.Data(image_segmentation_shp_file,  FORMAT)
        LOGGER.info('Joining dataframe %s to %s' %(C5_result, image_segmentation_shp_file))
        dataframe_joined_shp_segmentation_and_c5_result = join_C5_dataframe_and_shape(image_segmentation_shp_class, 'id', dataframe_c5_result, 'id')
        LOGGER.info('Number of rows and columns of dataframe joined: (%s,%s)' %(len(dataframe_joined_shp_segmentation_and_c5_result.index), len(dataframe_joined_shp_segmentation_and_c5_result.columns)))
        dataframe_joined_shp_segmentation_and_c5_result_file = folder_results + 'dataframe_joined_shp_segmentation_and_c5_result.csv'
        LOGGER.info('Writing csv of join between c5 result and segmentation shape: %s' % dataframe_joined_shp_segmentation_and_c5_result_file)            
        dataframe_joined_shp_segmentation_and_c5_result.to_csv(dataframe_joined_shp_segmentation_and_c5_result_file, sep =',', encoding = 'utf8', index = False)
        LOGGER.info('Writing C5 result joined with segmentation shape to shapefile')
        segmentation_and_c5_result_file_vectorized_folder = folder_results + 'segmentation_and_c5_result_vectorized/'
        create_directory_path(segmentation_and_c5_result_file_vectorized_folder)
        sql = "SELECT a.id, a.predicted, a.confidence, st_geomfromtext(a.geom," + image_segmentation_shp_class.srid+ ") as geometry "
        sql+= "from dataframe_joined_shp_segmentation_and_c5_result a"
        shp_result = segmentation_and_c5_result_file_vectorized_folder + '/C5_result_joined_segmentation_shape.shp'
        command = [
                    'ogr2ogr', shp_result,
                    dataframe_joined_shp_segmentation_and_c5_result_file,
                    '-dialect', 'sqlite', '-sql', sql
                    ]
        subprocess.call(command)       
        LOGGER.info('Rasterizing segmentation and c5 result shape of folder %s' % segmentation_and_c5_result_file_vectorized_folder)
        LOGGER.info('Identifying segmentation and c5 shape folder %s' % segmentation_and_c5_result_file_vectorized_folder)
        bundle = _get_bundle_from_path(segmentation_and_c5_result_file_vectorized_folder)
        if bundle:
            LOGGER.info('Directory %s is a %s bundle', segmentation_and_c5_result_file_vectorized_folder, bundle.get_name())
            LOGGER.info('Rasterizing vector shape to get land cover tif')
            options_to_create = new_options_for_create_raster_from_reference(image_to_be_classified_class.metadata, raster.DATA_SHAPE, (width,height, 1), {})
            dataset_shape_sg_and_c5_rasterized = create_raster_tiff_from_reference(image_to_be_classified_class.metadata, '', None, options_to_create)
            bundle.rasterize(dataset_shape_sg_and_c5_rasterized, [1], None, ["ATTRIBUTE=predicted" ]) #the rasterized process changes the dataset
            options_to_create = new_options_for_create_raster_from_reference(image_to_be_classified_class.metadata, raster.GDAL_CREATE_OPTIONS, ['COMPRESS=LZW'], {})            
            image =folder_results + 'madmex_lcc_prueba.tif'
            create_raster_tiff_from_reference(image_to_be_classified_class.metadata, image, dataset_shape_sg_and_c5_rasterized.ReadAsArray(), options_to_create, data_type = gdal.GDT_Int32)
            LOGGER.info('Finished rasterizing vector shape')
            LOGGER.info('Rasterizing vector shape to get confidence tif')
            bundle.rasterize(dataset_shape_sg_and_c5_rasterized, [1], None, ["ATTRIBUTE=confidence" ])
            image =folder_results + 'madmex_lcc_confidence_prueba.tif'            
            create_raster_tiff_from_reference(image_to_be_classified_class.metadata, image, dataset_shape_sg_and_c5_rasterized.ReadAsArray(), options_to_create)
            LOGGER.info('Finished rasterizing vector shape')
        
        LOGGER.info('Finished workflow classification :)')
Example #23
0
    def handle(self, **options):
        '''
        In this example command, the values that come from the user input are
        added up and the result is printed in the screen.
        '''
        target_tag = 'DN'
        start_time_all = time.time()
        shape_name = options['shape'][0]
        raster_paths = options['path']
        destination = options['dest']
        models = options['model']
        dataframe_features = None
        temporary_directory = getattr(SETTINGS, 'TEMPORARY')
        create_directory_path(temporary_directory)
        # I read the training data in shape form
        training_shape = vector.Data(shape_name)
        training_dataframe = training_shape.to_dataframe()
        training_path = create_filename(temporary_directory,
                                        'training_raster.tif')
        categories_file = create_filename(temporary_directory,
                                          'categories.json')
        training_warped_path = create_filename(temporary_directory,
                                               'training_warped_raster.tif')
        pixel_size = 0.000462175996292

        if not is_file(training_warped_path):
            training_raster = vector_to_raster(
                training_shape, training_path, pixel_size, -pixel_size,
                ['ATTRIBUTE=OBJECTID', 'COMPRESS=LZW'])
            training_raster_warped = training_raster.reproject(
                training_warped_path, epgs=32617)
        else:
            training_raster_warped = raster.Data(training_warped_path)

        dem_file = getattr(SETTINGS, 'DEM')

        dem_raster = raster.Data(dem_file)
        print dem_raster.get_spatial_reference()
        print 'reproyecting raster'
        #dem_raster_warped = dem_raster.reproject(training_warped_path, epgs=32614)

        #training_raster_warped = raster.Data(training_path)

        aspect_file = getattr(SETTINGS, 'ASPECT')
        slope_file = getattr(SETTINGS, 'SLOPE')

        print dem_file, aspect_file, slope_file

        for raster_path in raster_paths:
            scene_bundle = rapideye.Bundle(raster_path)

            raster_mask = scene_bundle.get_raster()

            #example_path = create_filename(temporary_directory, 'mask')
            #create_directory_path(example_path)
            #raster_to_vector_mask(raster_mask, example_path)

            print scene_bundle.get_raster_file()

            basename = get_basename(scene_bundle.get_raster_file())
            all_file = create_filename(temporary_directory,
                                       '%s_all_features.tif' % basename)
            # Do not recalculate if the file is already there.
            if is_file(all_file):
                features_raster = raster.Data(all_file)
            else:
                features_raster = scene_bundle.get_feature_array(all_file)
            new_df = get_dataframe_from_raster(features_raster,
                                               training_raster_warped)
            if new_df is not None:
                if dataframe_features is not None:
                    dataframe_features = pandas.concat([
                        dataframe_features,
                        get_dataframe_from_raster(features_raster,
                                                  training_raster_warped)
                    ])
                else:
                    dataframe_features = get_dataframe_from_raster(
                        features_raster, training_raster_warped)

        features_size = len(list(dataframe_features))

        training_set = dataframe_features.set_index(0).join(
            training_dataframe.set_index('OBJECTID'))

        print training_set

        training_set['target'] = pandas.Categorical.from_array(
            training_set[target_tag]).labels
        categories_array = pandas.Categorical.from_array(
            training_set[target_tag]).categories
        create_categories_file(categories_file, categories_array)
        training_set = training_set[training_set['target'] != -1]
        #features_size includes 0 that is the index of the feature
        training_set_array = numpy.transpose(
            numpy.transpose(training_set.as_matrix([range(1, features_size)])))
        target_set_array = training_set.pop('target')

        print training_set_array.shape
        print target_set_array.shape

        X_train, X_test, y_train, y_test = train_test_split(training_set_array,
                                                            target_set_array,
                                                            train_size=0.8,
                                                            test_size=0.2)
        models_directory = create_filename(temporary_directory, 'models')
        create_directory_path(models_directory)

        for model_name in models:
            start_time = time.time()
            print numpy.unique(y_train)
            train_model(X_train, X_test, y_train, y_test, models_directory,
                        model_name)
            print "--- %s seconds training %s model---" % (
                (time.time() - start_time), model_name)
Example #24
0
    def handle(self, **options):
        '''
        In this example command, the values that come from the user input are
        added up and the result is printed in the screen.
        '''
        output = options['output'][0]
        models = options['modelname']
        model_directory = options['modeldir'][0]
        region = options['region'][0]

        start_time = time.time()

        for path in options['path']:
            print path

            scene_bundle = rapideye.Bundle(path)
            directory = getattr(SETTINGS, 'TEMPORARY')
            directory_helper = create_filename(directory, 'helper')
            create_directory_path(directory_helper)
            categories_file = create_filename(directory, 'categories.json')
            categories_dictionaty = {
                0: "AGRICULTURA DE RIEGO",
                1: "AGRICULTURA DE TEMPORAL",
                2: "AGUA",
                3: "AREAS QUEMADAS",
                4: "ASENTAMIENTOS HUMANOS",
                5: "BOSQUE CULTIVADO",
                6: "BOSQUE DE AYARIN",
                7: "BOSQUE DE ENCINO",
                8: "BOSQUE DE ENCINO-PINO",
                9: "BOSQUE DE GALERIA",
                10: "BOSQUE DE MEZQUITE",
                11: "BOSQUE DE OYAMEL",
                12: "BOSQUE DE PINO",
                13: "BOSQUE DE PINO-ENCINO",
                14: "BOSQUE INDUCIDO",
                15: "BOSQUE MESOFILO DE MONTANA",
                16: "DESPROVISTO DE VEGETACION",
                17: "INDEFINIDO",
                18: "MANGLAR",
                19: "MATORRAL SUBTROPICAL",
                20: "MEZQUITAL",
                21: "NUBES",
                22: "PASTIZAL CULTIVADO",
                23: "PASTIZAL HALOFILO",
                24: "PASTIZAL INDUCIDO",
                25: "PASTIZAL NATURAL",
                26: "PRADERA DE ALTA MONTANA",
                27: "SABANOIDE",
                28: "SELVA ALTA PERENNIFOLIA",
                29: "SELVA ALTA SUBPERENNIFOLIA",
                30: "SELVA BAJA CADUCIFOLIA",
                31: "SELVA BAJA ESPINOSA CADUCIFOLIA",
                32: "SELVA BAJA SUBCADUCIFOLIA",
                33: "SELVA DE GALERIA",
                34: "SELVA MEDIANA CADUCIFOLIA",
                35: "SELVA MEDIANA SUBCADUCIFOLIA",
                36: "SELVA MEDIANA SUBPERENNIFOLIA",
                37: "SIN VEGETACION APARENTE",
                38: "SOMBRAS",
                39: "TULAR",
                40: "VEGETACION DE DUNAS COSTERAS",
                41: "VEGETACION HALOFILA HIDROFILA",
                42: "ZONA URBANA"
            }
            basename = get_basename(scene_bundle.get_raster_file())
            all_file = create_filename(directory_helper,
                                       '%s_all_features.tif' % basename)

            if not is_file(all_file):
                scene_bundle.get_feature_array(all_file)

            filename = get_basename(all_file)

            if not is_file(
                    create_filename(directory_helper, '%s.shp' % filename)):
                shell_string = 'docker run --rm -v %s:/data madmex/segment gdal-segment %s.tif -out helper/%s.shp -algo SLIC -region %s' % (
                    directory, filename, filename, region)
                launcher = LocalProcessLauncher()
                LOGGER.debug('Docker command: %s', shell_string)
                launcher.execute(shell_string)

            data = read_data_table(
                create_filename(directory_helper, '%s.shp' % filename))

            results = {}

            for model_name in models:
                persistence_directory = create_filename(
                    model_directory, model_name)
                print model_name
                model = load_model(model_name)
                model_instance = model.Model(persistence_directory)
                model_instance.load(persistence_directory)
                prediction = model_instance.predict(data)
                results[model_name] = prediction
            print results
            create_directory_path(output)
            write_results(
                create_filename(directory_helper, '%s.shp' % filename),
                create_filename(output,
                                '%s_classification.shp' % filename[0:32]),
                results, categories_dictionaty)

        LOGGER.info("--- %s seconds ---" % (time.time() - start_time))
Example #25
0
    def handle(self, **options):
        '''
        In this example command, the values that come from the user input are
        added up and the result is printed in the screen.
        '''
        random_int = 1
        training_template = options['training'][0]
        target_template = options['target'][0]
        index_template = options['index'][0]
        output = options['output'][0]
        models = options['model']
        features = training_template % random_int
        training = target_template % random_int
        index = index_template % random_int
        features_array = open_handle(features)
        training_array = open_handle(training)
        index_array = open_handle(index)

        labels = numpy.unique(index_array)

        features_flatten = numpy.ravel(features_array).reshape(10, 200 * 200)
        training_flatten = numpy.ravel(training_array).reshape(200 * 200)

        array_aux = []
        for j in range(features_array.shape[0]):
            means = scipy.ndimage.measurements.mean(features_array[j, :, :],
                                                    index_array, labels)
            print means.shape
            array_aux.append(means)
        for j in range(features_array.shape[0]):
            std = scipy.ndimage.measurements.standard_deviation(
                features_array[j, :, :], index_array, labels)
            print std.shape
            array_aux.append(std)

        print len(array_aux)

        features_final = numpy.concatenate([array_aux], axis=0)

        print features_final.shape

        label_object = []
        for id in labels:
            values, counts = numpy.unique(training_array[index_array == id],
                                          return_counts=True)
            label_object.append(values[numpy.argmax(counts)])

        features_total = features_final
        training_total = label_object

        for i in range(1, 53):
            features = training_template % (i + 1)
            training = target_template % (i + 1)
            index = index_template % (i + 1)
            features_array = open_handle(features)
            training_array = open_handle(training)
            index_array = open_handle(index)

            labels = numpy.unique(index_array)

            array_aux = []
            for j in range(features_array.shape[0]):
                array_aux.append(
                    scipy.ndimage.measurements.mean(features_array[j, :, :],
                                                    index_array, labels))
            for j in range(features_array.shape[0]):
                array_aux.append(
                    scipy.ndimage.measurements.standard_deviation(
                        features_array[j, :, :], index_array, labels))
            features_final = numpy.concatenate([array_aux], axis=0)

            print features_final.shape

            label_object = []

            for id in labels:
                values, counts = numpy.unique(
                    training_array[index_array == id], return_counts=True)
                #print '********* ', values[numpy.argmax(counts)], counts[numpy.argmax(counts)]
                label_object.append(values[numpy.argmax(counts)])

            print features_total.shape
            print features_final.shape
            features_total = numpy.concatenate(
                (features_total, features_final), axis=1)
            training_total = numpy.concatenate((training_total, label_object),
                                               axis=1)

            print 'label object', len(label_object)

            #print scipy.ndimage.measurements.mean(training_array, index_array, labels)
            features_flatten = numpy.concatenate(
                (features_flatten, numpy.ravel(features_array).reshape(
                    10, 200 * 200)),
                axis=1)
            training_flatten = numpy.concatenate(
                (training_flatten, numpy.ravel(training_array).reshape(
                    200 * 200)),
                axis=0)
        # Remove the elements that map to None
        #mask = training_flatten!=0
        #features_flatten = features_flatten[:,mask]
        #training_flatten = training_flatten[mask]

        mask = training_total != 0
        features_flatten = features_total[:, mask]
        training_flatten = training_total[mask]

        print features_flatten.shape
        print training_flatten.shape
        X_train, X_test, y_train, y_test = train_test_split(
            numpy.transpose(features_flatten),
            training_flatten,
            train_size=0.8,
            test_size=0.2)

        print X_train[0]
        print y_train[0]

        unsupervised = pca.Model(5)
        unsupervised.fit(X_train)
        '''
        X_train = unsupervised.transform(X_train)
        X_test = unsupervised.transform(X_test) 
        '''

        pca_path = create_filename(output, 'pca')
        create_directory_path(pca_path)
        unsupervised.save(pca_path)

        import time
        for model_name in models:
            start_time = time.time()
            train_model(X_train, X_test, y_train, y_test, output, model_name)
            print "--- %s seconds training %s model---" % (
                (time.time() - start_time), model_name)
Example #26
0
    def handle(self, **options):
        '''
        This is the code that does the ingestion.        
        indexes --path /LUSTRE/MADMEX/staging/2016_tasks/Humedales_for_ctroche/LT/Landsat_2000_2008/L5_021_047_2000_022_sr/ --shape /LUSTRE/MADMEX/staging/2016_tasks/Humedales_for_ctroche/LT/AE_LT_new.shp --output 
        '''
        path = options['path'][0]

        for mask_file in os.listdir(path):
            if mask_file.endswith('_cfmask.tif'):
                print mask_file
                basename = mask_file.replace('_cfmask.tif', '')
                print basename
                cloud_file_name = mask_file

        LOGGER.info('Calculating indexes for Landsat scenes.')

        band_name = basename + '_sr_band%s.tif'

        final_path = options['output'][0]
        create_directory_path(final_path)
        cloud_file = create_filename(path, cloud_file_name)
        cloud_array = open_handle(cloud_file)
        cloud_mask = (cloud_array == 4)

        LOGGER.debug('Recognize sensor depending on file name.')

        if 'L8' in path:
            LOGGER.debug('Landsat 8 scene was detected.')
            blue_file = create_filename(path, band_name % BLUE_L8)
            green_file = create_filename(path, band_name % GREEN_L8)
            red_file = create_filename(path, band_name % RED_L8)
            nir_file = create_filename(path, band_name % NIR_L8)
            swir_file = create_filename(path, band_name % SWIR_L8)
        else:
            LOGGER.debug('Landsat 4, 5 or scene was detected.')
            blue_file = create_filename(path, band_name % BLUE)
            green_file = create_filename(path, band_name % GREEN)
            red_file = create_filename(path, band_name % RED)
            nir_file = create_filename(path, band_name % NIR)
            swir_file = create_filename(path, band_name % SWIR)

        LOGGER.debug('Loading bands of interest.')

        green_array = open_handle(green_file)
        red_array = open_handle(red_file)
        nir_array = open_handle(nir_file)
        swir_array = open_handle(swir_file)

        LOGGER.debug('Calculating indexes.')

        ndvi_array = calculate_index(nir_array, red_array)
        mndwi_array = calculate_index(green_array, swir_array)
        ndwig_array = calculate_index(nir_array, swir_array)
        ndwim_array = calculate_index(green_array, nir_array)

        LOGGER.debug('Setting cloud mask values to -999.')

        ndvi_array[cloud_mask] = -999
        mndwi_array[cloud_mask] = -999
        ndwig_array[cloud_mask] = -999
        ndwim_array[cloud_mask] = -999

        LOGGER.debug('Creating files for indexes.')

        ndvi_final_file = create_filename(final_path, basename + '_ndvi.tif')
        mndwi_final_file = create_filename(final_path, basename + '_mndwi.tif')
        ndwig_final_file = create_filename(final_path, basename + '_ndwig.tif')
        ndwim_final_file = create_filename(final_path, basename + '_ndwim.tif')

        ndvi_clipped_file = create_filename(final_path,
                                            basename + '_ndvi_clipped.tif')
        mndwi_clipped_file = create_filename(final_path,
                                             basename + '_mndwi_clipped.tif')
        ndwig_clipped_file = create_filename(final_path,
                                             basename + '_ndwig_clipped.tif')
        ndwim_clipped_file = create_filename(final_path,
                                             basename + '_ndwim_clipped.tif')

        ndvi_file = create_filename(final_path, basename + '_ndvi_pre.tif')
        mndwi_file = create_filename(final_path, basename + '_mndwi_pre.tif')
        ndwig_file = create_filename(final_path, basename + '_ndwig_pre.tif')
        ndwim_file = create_filename(final_path, basename + '_ndwim_pre.tif')

        files = [ndvi_file, mndwi_file, ndwig_file, ndwim_file]
        clipped_files = [
            ndvi_clipped_file, mndwi_clipped_file, ndwig_clipped_file,
            ndwim_clipped_file
        ]
        final_files = [
            ndvi_final_file, mndwi_final_file, ndwig_final_file,
            ndwim_final_file
        ]

        LOGGER.debug('Writing information to files.')

        create_raster_from_reference(ndvi_file, ndvi_array, green_file)
        create_raster_from_reference(mndwi_file, mndwi_array, green_file)
        create_raster_from_reference(ndwig_file, ndwig_array, green_file)
        create_raster_from_reference(ndwim_file, ndwim_array, green_file)

        LOGGER.debug('Deleting arrays to release memory.')

        del ndvi_array
        del mndwi_array
        del ndwig_array
        del ndwim_array
        del cloud_array

        LOGGER.debug('Reference rgb file creation.')

        rgb_file = create_filename(final_path, basename + '_rgb.tif')
        merge_command = [
            '/Library/Frameworks/GDAL.framework/Programs/gdalbuildvrt',
            '-separate', '-o', rgb_file, red_file, green_file, blue_file
        ]
        call(merge_command)
        shape = options['shape'][0]

        LOGGER.debug('Cookie cut files using the given shape.')

        for i in range(4):
            clip_command = [
                '/Library/Frameworks/GDAL.framework/Programs/gdalwarp',
                '-crop_to_cutline', '-cutline', shape, files[i],
                clipped_files[i]
            ]
            call(clip_command)

        for i in range(4):
            aux_array = open_handle(clipped_files[i])
            aux_array[(aux_array == 0)] = -9999
            create_raster_from_reference(final_files[i], aux_array,
                                         ndvi_clipped_file)

        if not options['debug']:
            LOGGER.info('Remove auxiliary files.')
            os.remove(ndvi_clipped_file)
            os.remove(mndwi_clipped_file)
            os.remove(ndwig_clipped_file)
            os.remove(ndwim_clipped_file)
            os.remove(ndvi_file)
            os.remove(mndwi_file)
            os.remove(ndwig_file)
            os.remove(ndwim_file)

        print 'Done'