Esempio n. 1
0
    def handle(self, **options):

        paths = options['path']
        output = options['output'][0]
        
        print output
        arrays = {}
        
        import time

        for path in paths:
            start_time = time.time()
            arrays[path] = open_handle(path)
            print 'Opened %s' % path
            print("--- %s seconds ---" % (time.time() - start_time))
            
        size = arrays[paths[0]].shape
        

        start_time = time.time()
        result = numpy.full(size, NO_DATA)       
        mask = numpy.equal(arrays[paths[0]],arrays[paths[1]])
        for p in range(2, len(paths)):
            print 'Image %s ' % p
            mask = numpy.logical_and(mask, numpy.equal(arrays[paths[0]],arrays[paths[p]]))
        result[mask] = numpy.array(arrays[paths[0]])[mask]
                    
        for path in paths:
            del arrays[path]
        print("--- %s seconds ---" % (time.time() - start_time))
        
        start_time = time.time()
        create_raster_from_reference(output, result, paths[0], data_type=gdal.GDT_Byte)
        print("--- %s seconds ---" % (time.time() - start_time))            
Esempio n. 2
0
 def method_five(self, path, output):
     '''
     Using numpy utilities, this method mask the desired value.
     '''
     data_array = open_handle(path)
     data_array[data_array != MASK] = 0
     data_array[data_array == MASK] = 1
     create_raster_from_reference(output, data_array, path, gdal.GDT_Byte)
     del data_array
Esempio n. 3
0
 def method_five(self, path, output):
     '''
     Using numpy utilities, this method mask the desired value.
     '''
     data_array = open_handle(path)
     data_array[data_array!=MASK] = 0
     data_array[data_array==MASK] = 1
     create_raster_from_reference(output, data_array, path, gdal.GDT_Byte)
     del data_array
Esempio n. 4
0
 def mask_iterating_values(self, path, output):
     '''
     Iterating over the values in the initial and final arrays this replaces the
     desired values one by one.
     '''
     data_array = open_handle(path)
     my_dictionary = dictionary_from_list(INITIAL_ARRAY, FINAL_ARRAY)
     to_vector_array = replace_in_array(data_array, my_dictionary)
     create_raster_from_reference(output, to_vector_array, path, gdal.GDT_Byte)
Esempio n. 5
0
 def mask_iterating_values(self, path, output, ini_arr, fin_arr):
     '''
     Iterating over the values in the initial and final arrays this replaces the
     desired values one by one.
     '''
     data_array = open_handle(path)
     my_dictionary = dictionary_from_list(ini_arr, fin_arr)
     to_vector_array = replace_in_array(data_array, my_dictionary)
     create_raster_from_reference(output, to_vector_array, path,
                                  gdal.GDT_Byte)
Esempio n. 6
0
def tile_map(image_path, output, x_tile_size, y_tile_size):
    from subprocess import call
    data_array = open_handle(image_path)
    width = data_array.shape[1] 
    height = data_array.shape[0]
    for i in range(0, width, x_tile_size):
        for j in range(0, height, y_tile_size):
            gdaltranString = ['/Library/Frameworks/GDAL.framework/Programs/gdal_translate', '-of', 'GTIFF', '-srcwin', str(i),  str(j) , str(x_tile_size) ,  str(y_tile_size) , image_path, output + 'utm_' + str(i) + '_' + str(j) + '.tif']
            print ' '.join(gdaltranString)    
            call(gdaltranString)
Esempio n. 7
0
    def handle(self, **options):
        '''
        In this example command, the values that come from the user input are
        added up and the result is printed in the screen.
        '''
        output = options['output'][0]
        models = options['modelname']
        model_directory = options['modeldir'][0]

        pca_model = pca.Model(5)
        pca_model.load(create_filename(model_directory, 'pca'))

        for model_name in models:
            persistence_directory = create_filename(model_directory,
                                                    model_name)
            model = load_model(model_name)
            model_instance = model.Model(persistence_directory)
            model_instance.load(persistence_directory)
            block_size = 500
            for path in options['path']:
                image_array = open_handle(path)
                y_size = image_array.shape[1]
                x_size = image_array.shape[2]
                basename = get_basename(path)[:7]
                warnings.filterwarnings('ignore')
                final = numpy.zeros((x_size, y_size))
                import time
                start_time = time.time()
                for i in range(0, y_size, block_size):
                    if i + block_size < y_size:
                        rows = block_size
                    else:
                        rows = y_size - i
                    for j in range(0, x_size, block_size):
                        if j + block_size < x_size:
                            cols = block_size
                        else:
                            cols = x_size - j
                        step = image_array[:, i:i + rows, j:j + cols]
                        step_ravel = step.reshape(10, -1)
                        prediction = model_instance.predict(
                            pca_model.transform(numpy.transpose(step_ravel)))
                        final[i:i + rows, j:j + cols] = prediction.reshape(
                            (rows, cols))
                print("--- %s seconds ---" % (time.time() - start_time))
                create_directory_path(output)
                classification = create_filename(
                    output, '%s-%s.tif' % (basename, model_name))
                create_raster_from_reference(classification,
                                             final.reshape(x_size, y_size),
                                             path,
                                             data_type=gdal.GDT_Byte,
                                             creating_options=['COMPRESS=LZW'])
Esempio n. 8
0
    def handle(self, **options):

        paths = options['path']
        output = options['output'][0]

        print output
        arrays = {}

        image1 = open_handle(paths[0])

        #import time

        #for path in paths:
        #    start_time = time.time()
        #    arrays[path] = open_handle(path)
        #    print 'Opened %s' % path
        #    print("--- %s seconds ---" % (time.time() - start_time))

        size = image1.shape

        #start_time = time.time()
        for p in range(2, len(paths)):
            #print 'Image %s ' % p
            image2 = open_handle(paths[p])
            mask = numpy.logical_and(
                numpy.equal(image1, open_handle(paths[1])),
                numpy.equal(image1, image2))
        image1[~mask] = NO_DATA

        #for path in paths:
        #del arrays[path]
        #print("--- %s seconds ---" % (time.time() - start_time))

        #start_time = time.time()
        create_raster_from_reference(output,
                                     image1y,
                                     paths[0],
                                     data_type=gdal.GDT_Byte)
Esempio n. 9
0
def tile_map(image_path, output, x_tile_size, y_tile_size):
    from subprocess import call

    data_array = open_handle(image_path)
    width = data_array.shape[1]
    height = data_array.shape[0]
    for i in range(0, width, x_tile_size):
        for j in range(0, height, y_tile_size):
            gdaltranString = [
                "/Library/Frameworks/GDAL.framework/Programs/gdal_translate",
                "-of",
                "GTIFF",
                "-srcwin",
                str(i),
                str(j),
                str(x_tile_size),
                str(y_tile_size),
                image_path,
                output + "utm_" + str(i) + "_" + str(j) + ".tif",
            ]
            print " ".join(gdaltranString)
            call(gdaltranString)
Esempio n. 10
0
    def print_unique(self, path):

        data_array = open_handle(path)
        unique, counts = numpy.unique(data_array, return_counts=True)
        print unique, counts
Esempio n. 11
0
    def handle(self, **options):
        import time
        start_time = time.time()

        path = options['path'][0]
        print path
        features_array = []

        bundle = Bundle(path)

        raster_path = bundle.file_dictionary[_IMAGE]

        data_array = open_handle(raster_path)
        data_array = numpy.array(data_array)

        print data_array, len(data_array)

        print len(data_array[data_array != 0])

        for i in range(data_array.shape[0]):
            print 'band: %s' % (i + 1)
            band = data_array[i, :, :].ravel()
            band = band[band != 0]
            features_array.append(numpy.nanpercentile(band, 10))
            features_array.append(numpy.nanpercentile(band, 25))
            features_array.append(numpy.nanpercentile(band, 50))
            features_array.append(numpy.nanpercentile(band, 75))
            features_array.append(numpy.nanpercentile(band, 90))
            features_array.append(numpy.mean(band))
            features_array.append(numpy.min(band) * 1.0)
            features_array.append(numpy.max(band) * 1.0)
        geotransform = get_geotransform(raster_path)

        features_array.append(geotransform[0])
        features_array.append(geotransform[3])

        print features_array

        features_array.append((bundle.get_aquisition_date() -
                               datetime.datetime(1970, 1, 1)).total_seconds())
        tile_id = bundle.get_sensor().get_attribute(TILE_ID)

        features = RapideyeFeatures(band_1_quant_10=features_array[0],
                                    band_1_quant_25=features_array[1],
                                    band_1_quant_50=features_array[2],
                                    band_1_quant_75=features_array[3],
                                    band_1_quant_90=features_array[4],
                                    band_1_mean=features_array[5],
                                    band_1_min=features_array[6],
                                    band_1_max=features_array[7],
                                    band_2_quant_10=features_array[8],
                                    band_2_quant_25=features_array[9],
                                    band_2_quant_50=features_array[10],
                                    band_2_quant_75=features_array[11],
                                    band_2_quant_90=features_array[12],
                                    band_2_mean=features_array[13],
                                    band_2_min=features_array[14],
                                    band_2_max=features_array[15],
                                    band_3_quant_10=features_array[16],
                                    band_3_quant_25=features_array[17],
                                    band_3_quant_50=features_array[18],
                                    band_3_quant_75=features_array[19],
                                    band_3_quant_90=features_array[20],
                                    band_3_mean=features_array[21],
                                    band_3_min=features_array[22],
                                    band_3_max=features_array[23],
                                    band_4_quant_10=features_array[24],
                                    band_4_quant_25=features_array[25],
                                    band_4_quant_50=features_array[26],
                                    band_4_quant_75=features_array[27],
                                    band_4_quant_90=features_array[28],
                                    band_4_mean=features_array[29],
                                    band_4_min=features_array[30],
                                    band_4_max=features_array[31],
                                    band_5_quant_10=features_array[32],
                                    band_5_quant_25=features_array[33],
                                    band_5_quant_50=features_array[34],
                                    band_5_quant_75=features_array[35],
                                    band_5_quant_90=features_array[36],
                                    band_5_mean=features_array[37],
                                    band_5_min=features_array[38],
                                    band_5_max=features_array[39],
                                    top=features_array[40],
                                    left=features_array[41],
                                    time=features_array[42],
                                    footprint=tile_id,
                                    path=raster_path)

        klass = sessionmaker(bind=ENGINE, autoflush=False)
        session = klass()

        session.add(features)
        session.commit()

        print tile_id
        print features_array
        print len(features_array)

        print("--- %s seconds ---" % (time.time() - start_time))
Esempio n. 12
0
    def handle(self, **options):
        '''
        In this example command, the values that come from the user input are
        added up and the result is printed in the screen.
        '''
        shape_name = options['shape'][0]
        destination = options['dest'][0]
        paths = options['path']

        years = []
        for path in paths:
            years.append(self.get_year_from_path(path))

        if os.path.exists(shape_name):
            LOGGER.info('The file %s was found.' % shape_name)
        shape_files = split_shape_into_features(shape_name, destination,
                                                str('id'), str('nombre'), '__')
        process_launcher = LocalProcessLauncher()

        import time
        start_time = time.time()

        cover_array = []
        tth_array = []
        cover_file = 'cover-stats.json'
        tth_name = 'tth-stats.json'
        json_directory = create_filename(destination, 'cover_stats')
        json_file = create_filename(json_directory, cover_file.lower())
        tth_file = create_filename(json_directory, tth_name.lower())

        for shape_file in shape_files:
            shape_name = get_basename(shape_file).split('__')
            anp_id = shape_name[0]
            anp_name = shape_name[1]
            basename = '%s.tif' % anp_name

            dataframe = DataFrame(index=years,
                                  columns=[0, 1, 2, 3, 4, 5, 6, 7, 8])

            create_directory_path(json_directory)

            print shape_file
            for path in paths:
                #pixel_resolution, dataset = pixel_info(path)

                year = self.get_year_from_path(path)
                raster_dir = create_filename(
                    create_filename(destination, 'raster'), year)
                create_directory_path(raster_dir)
                raster_file = create_filename(raster_dir, basename)

                shell_command = 'gdalwarp -ot Byte -co COMPRESS=LZW -cutline %s -crop_to_cutline %s %s' % (
                    shape_file, path, raster_file)
                print shell_command
                print process_launcher.execute(shell_command)
                raster_array = open_handle(raster_file)

                ds = gdal.Open(raster_file)
                geotransform = ds.GetGeoTransform()
                x_resolution = geotransform[1]
                y_resolution = geotransform[5]
                pixel_area = abs(x_resolution * y_resolution)

                unique_values = numpy.unique(raster_array, return_counts=True)

                indexes = unique_values[0]
                counts = unique_values[1]

                for i in range(len(indexes)):
                    key = indexes[i]
                    dataframe.set_value(year, key,
                                        area(int(counts[i]), pixel_area))
            dataframe = dataframe.sort_index()
            columns = list(dataframe.columns.values)
            index = list(dataframe.index.values)

            print dataframe
            cover_array.append(
                self.dataframe_to_json(dataframe, anp_id, anp_name))

            tth_dataframe = DataFrame(columns=columns)

            for i in range(len(index) - 1):
                label = '%s-%s' % (index[i], index[i + 1])
                tth_column = calculate_thh(dataframe.ix[i],
                                           dataframe.ix[i + 1],
                                           int(index[i + 1]) - int(index[i]))
                for j in range(len(tth_column)):
                    tth_dataframe.set_value(label, j, tth_column[j])
            tth_dataframe = tth_dataframe.sort_index()

            print tth_dataframe

            tth_array.append(
                self.dataframe_to_json(tth_dataframe, anp_id, anp_name))

        self.json_to_file(json_file, cover_array)
        self.json_to_file(tth_file, tth_array)

        print("--- %s seconds ---" % (time.time() - start_time))
Esempio n. 13
0
    def handle(self, **options):
        '''
        In this example command, the values that come from the user input are
        added up and the result is printed in the screen.
        '''
        random_int = 1
        training_template = options['training'][0]
        target_template = options['target'][0]
        index_template = options['index'][0]
        output = options['output'][0]
        models = options['model']
        features = training_template % random_int
        training = target_template % random_int
        index = index_template % random_int
        features_array = open_handle(features)
        training_array = open_handle(training)
        index_array = open_handle(index)

        labels = numpy.unique(index_array)

        features_flatten = numpy.ravel(features_array).reshape(10, 200 * 200)
        training_flatten = numpy.ravel(training_array).reshape(200 * 200)

        array_aux = []
        for j in range(features_array.shape[0]):
            means = scipy.ndimage.measurements.mean(features_array[j, :, :],
                                                    index_array, labels)
            print means.shape
            array_aux.append(means)
        for j in range(features_array.shape[0]):
            std = scipy.ndimage.measurements.standard_deviation(
                features_array[j, :, :], index_array, labels)
            print std.shape
            array_aux.append(std)

        print len(array_aux)

        features_final = numpy.concatenate([array_aux], axis=0)

        print features_final.shape

        label_object = []
        for id in labels:
            values, counts = numpy.unique(training_array[index_array == id],
                                          return_counts=True)
            label_object.append(values[numpy.argmax(counts)])

        features_total = features_final
        training_total = label_object

        for i in range(1, 53):
            features = training_template % (i + 1)
            training = target_template % (i + 1)
            index = index_template % (i + 1)
            features_array = open_handle(features)
            training_array = open_handle(training)
            index_array = open_handle(index)

            labels = numpy.unique(index_array)

            array_aux = []
            for j in range(features_array.shape[0]):
                array_aux.append(
                    scipy.ndimage.measurements.mean(features_array[j, :, :],
                                                    index_array, labels))
            for j in range(features_array.shape[0]):
                array_aux.append(
                    scipy.ndimage.measurements.standard_deviation(
                        features_array[j, :, :], index_array, labels))
            features_final = numpy.concatenate([array_aux], axis=0)

            print features_final.shape

            label_object = []

            for id in labels:
                values, counts = numpy.unique(
                    training_array[index_array == id], return_counts=True)
                #print '********* ', values[numpy.argmax(counts)], counts[numpy.argmax(counts)]
                label_object.append(values[numpy.argmax(counts)])

            print features_total.shape
            print features_final.shape
            features_total = numpy.concatenate(
                (features_total, features_final), axis=1)
            training_total = numpy.concatenate((training_total, label_object),
                                               axis=1)

            print 'label object', len(label_object)

            #print scipy.ndimage.measurements.mean(training_array, index_array, labels)
            features_flatten = numpy.concatenate(
                (features_flatten, numpy.ravel(features_array).reshape(
                    10, 200 * 200)),
                axis=1)
            training_flatten = numpy.concatenate(
                (training_flatten, numpy.ravel(training_array).reshape(
                    200 * 200)),
                axis=0)
        # Remove the elements that map to None
        #mask = training_flatten!=0
        #features_flatten = features_flatten[:,mask]
        #training_flatten = training_flatten[mask]

        mask = training_total != 0
        features_flatten = features_total[:, mask]
        training_flatten = training_total[mask]

        print features_flatten.shape
        print training_flatten.shape
        X_train, X_test, y_train, y_test = train_test_split(
            numpy.transpose(features_flatten),
            training_flatten,
            train_size=0.8,
            test_size=0.2)

        print X_train[0]
        print y_train[0]

        unsupervised = pca.Model(5)
        unsupervised.fit(X_train)
        '''
        X_train = unsupervised.transform(X_train)
        X_test = unsupervised.transform(X_test) 
        '''

        pca_path = create_filename(output, 'pca')
        create_directory_path(pca_path)
        unsupervised.save(pca_path)

        import time
        for model_name in models:
            start_time = time.time()
            train_model(X_train, X_test, y_train, y_test, output, model_name)
            print "--- %s seconds training %s model---" % (
                (time.time() - start_time), model_name)
Esempio n. 14
0
    def handle(self, **options):
        import time
        start_time = time.time()
        
        path = options['path'][0]
        print path
        features_array = []
        
        bundle = Bundle(path)
        
        raster_path = bundle.file_dictionary[_IMAGE]
        
        data_array = open_handle(raster_path)
        data_array = numpy.array(data_array)
        
        print data_array, len(data_array)
        
        print len(data_array[data_array!=0])
        
        for i in range(data_array.shape[0]):
            print 'band: %s' % (i + 1)
            band = data_array[i,:,:].ravel()
            band = band[band!=0]
            features_array.append(numpy.nanpercentile(band,10))
            features_array.append(numpy.nanpercentile(band,25))
            features_array.append(numpy.nanpercentile(band,50))
            features_array.append(numpy.nanpercentile(band,75))
            features_array.append(numpy.nanpercentile(band,90))
            features_array.append(numpy.mean(band))
            features_array.append(numpy.min(band) * 1.0)
            features_array.append(numpy.max(band) * 1.0)
        geotransform = get_geotransform(raster_path)
        
        features_array.append(geotransform[0])
        features_array.append(geotransform[3])
        
        
        print features_array
        
        
        features_array.append((bundle.get_aquisition_date() - datetime.datetime(1970, 1, 1)).total_seconds())
        tile_id =  bundle.get_sensor().get_attribute(TILE_ID)
        

        features = RapideyeFeatures(band_1_quant_10=features_array[0],
                                    band_1_quant_25=features_array[1],
                                    band_1_quant_50=features_array[2],
                                    band_1_quant_75=features_array[3],
                                    band_1_quant_90=features_array[4],
                                    band_1_mean=features_array[5],
                                    band_1_min=features_array[6],
                                    band_1_max=features_array[7],
                                    band_2_quant_10=features_array[8],
                                    band_2_quant_25=features_array[9],
                                    band_2_quant_50=features_array[10],
                                    band_2_quant_75=features_array[11],
                                    band_2_quant_90=features_array[12],
                                    band_2_mean=features_array[13],
                                    band_2_min=features_array[14],
                                    band_2_max=features_array[15],
                                    band_3_quant_10=features_array[16],
                                    band_3_quant_25=features_array[17],
                                    band_3_quant_50=features_array[18],
                                    band_3_quant_75=features_array[19],
                                    band_3_quant_90=features_array[20],
                                    band_3_mean=features_array[21],
                                    band_3_min=features_array[22],
                                    band_3_max=features_array[23],
                                    band_4_quant_10=features_array[24],
                                    band_4_quant_25=features_array[25],
                                    band_4_quant_50=features_array[26],
                                    band_4_quant_75=features_array[27],
                                    band_4_quant_90=features_array[28],
                                    band_4_mean=features_array[29],
                                    band_4_min=features_array[30],
                                    band_4_max=features_array[31],
                                    band_5_quant_10=features_array[32],
                                    band_5_quant_25=features_array[33],
                                    band_5_quant_50=features_array[34],
                                    band_5_quant_75=features_array[35],
                                    band_5_quant_90=features_array[36],
                                    band_5_mean=features_array[37],
                                    band_5_min=features_array[38],
                                    band_5_max=features_array[39],
                                    top=features_array[40],
                                    left=features_array[41],
                                    time=features_array[42],
                                    footprint=tile_id,
                                    path=raster_path)
        
        
        klass = sessionmaker(bind=ENGINE, autoflush=False)
        session = klass()
        
        session.add(features)
        session.commit()
        
        print tile_id
        print features_array
        print len(features_array)
        
        print("--- %s seconds ---" % (time.time() - start_time))