Example #1
0
class AGDC2GDF(GDF):
    DEFAULT_CONFIG_FILE = 'agdc2gdf_default.conf' # N.B: Assumed to reside in code root directory
    ARG_DESCRIPTORS = {'xmin': {'short_flag': '-x1', 
                                        'long_flag': '--xmin', 
                                        'default': None, 
                                        'action': 'store',
                                        'const': None, 
                                        'help': 'Minimum X inclusive t (longitude) of spatial range to process'
                                        },
                                'xmax': {'short_flag': '-x2', 
                                        'long_flag': '--xmax', 
                                        'default': None, 
                                        'action': 'store',
                                        'const': None, 
                                        'help': 'Maximum X inclusive t (longitude) of spatial range to process'
                                        },
                                'ymin': {'short_flag': '-y1', 
                                        'long_flag': '--ymin', 
                                        'default': None, 
                                        'action': 'store',
                                        'const': None, 
                                        'help': 'Minimum Y inclusive t (latitude) of spatial range to process'
                                        },
                                'ymax': {'short_flag': '-y2', 
                                        'long_flag': '--ymax', 
                                        'default': None, 
                                        'action': 'store',
                                        'const': None, 
                                        'help': 'Maximum Y inclusive t (latitude) of spatial range to process'
                                        },
                                'tmin': {'short_flag': '-t1', 
                                        'long_flag': '--tmin', 
                                        'default': None, 
                                        'action': 'store',
                                        'const': None, 
                                        'help': 'Minimum t inclusive t (years) of spatial range to process'
                                        },
                                'tmax': {'short_flag': '-t2', 
                                        'long_flag': '--tmax', 
                                        'default': None, 
                                        'action': 'store',
                                        'const': None, 
                                        'help': 'Maximum inclusive t (years) of spatial range to process'
                                        },
                                'storage_type': {'short_flag': '-st', 
                                        'long_flag': '--storage_type', 
                                        'default': None, 
                                        'action': 'store',
                                        'const': None, 
                                        'help': 'GDF storage type to populate'
                                        },
                                'satellite': {'short_flag': '-sa', 
                                        'long_flag': '--satellite', 
                                        'default': None, 
                                        'action': 'store',
                                        'const': None, 
                                        'help': 'AGDC satellite to process'
                                        },
                                'sensors': {'short_flag': '-se', 
                                        'long_flag': '--sensors', 
                                        'default': None, 
                                        'action': 'store',
                                        'const': None, 
                                        'help': 'Comma-separated list of AGDC sensors to process'
                                        },
                                'level': {'short_flag': '-l', 
                                        'long_flag': '--level', 
                                        'default': None, 
                                        'action': 'store',
                                        'const': None, 
                                        'help': 'AGDC processing level to process'
                                        },
                                'temp_dir': {'short_flag': '-t', 
                                        'long_flag': '--temp_dir', 
                                        'default': None, 
                                        'action': 'store',
                                        'const': None, 
                                        'help': 'Temporary directory for AGDC2GDF operation'
                                        },
                                'force': {'short_flag': '-f', 
                                        'long_flag': '--force', 
                                        'default': False, 
                                        'action': 'store_const', 
                                        'const': True,
                                        'help': 'Flag to force replacement of existing files'
                                        },
                                'dryrun': {'short_flag': '-dr', 
                                        'long_flag': '--dryrun', 
                                        'default': False, 
                                        'action': 'store_const', 
                                        'const': True,
                                        'help': 'Flag to skip file writing and SQL query execution'
                                        },
                                }
    
    def __init__(self):
        '''Constructor for class AGDC2GDF
        '''
        self._code_root = os.path.abspath(os.path.dirname(__file__)) # Directory containing module code
        self._gdf_root = os.path.abspath(os.path.dirname(gdf.__file__)) # Directory containing module code
        
        # Create master configuration dict containing both command line and config_file parameters
        self._command_line_params = self._get_command_line_params(AGDC2GDF.ARG_DESCRIPTORS)
        
        self.dryrun = self._command_line_params['dryrun']

        agdc2gdf_config_file = self._command_line_params['config_files'] or os.path.join(self._code_root, AGDC2GDF.DEFAULT_CONFIG_FILE)
        
        agdc2gdf_config_file_object = ConfigFile(agdc2gdf_config_file)
        
        # Comma separated list of GDF config files specified in master config file
        gdf_config_files_string = agdc2gdf_config_file_object.configuration['gdf'].get('config_files') or os.path.join(self._gdf_root, GDF.DEFAULT_CONFIG_FILE)
        
        # Create master GDF configuration dict containing both command line and config_file parameters
        self._configuration = self._get_config(gdf_config_files_string)
                
        self.temp_dir = self._command_line_params.get('temp_dir') or agdc2gdf_config_file_object.configuration['agdc']['temp_dir']
        # Try to create temp & cache directories if they don't exist
        if not directory_writable(self.temp_dir):
            new_temp_dir = os.path.join(os.path.expanduser("~"), 'gdf', 'temp')
            logger.warning('Unable to access temporary directory %s. Using %s instead.', self.temp_dir, new_temp_dir)
            self.temp_dir = new_temp_dir
            if not directory_writable(self.temp_dir):
                raise Exception('Unable to write to temporary directory %s', self.temp_dir)
            

        # Create master GDF database dictorage_config
        self._databases = self._get_dbs()
        
        self.force = self._command_line_params.get('force') or agdc2gdf_config_file_object.configuration['agdc2gdf'].get('force')
        
        logger.debug("self._command_line_params.get('storage_type') = %s", self._command_line_params.get('storage_type'))
        self.storage_type = self._command_line_params.get('storage_type') or agdc2gdf_config_file_object.configuration['gdf']['storage_type']

        self.agdc_satellite = self._command_line_params.get('satellite') or agdc2gdf_config_file_object.configuration['agdc']['satellite']

        self.agdc_sensors = self._command_line_params.get('sensors') or agdc2gdf_config_file_object.configuration['agdc']['sensors']
        self.agdc_sensors = tuple(self.agdc_sensors.split(','))

        self.agdc_level = self._command_line_params.get('level') or agdc2gdf_config_file_object.configuration['agdc']['level']
        
        # Read GDF storage configuration from databases
        self._storage_config = self._get_storage_config()
        self.storage_type_config = self._storage_config[self.storage_type]
        self.database = self._databases[self.storage_type_config['db_ref']]
        
        self.dimensions = self.storage_type_config['dimensions'] # This is used a lot
                
        
        # Set up AGDC stuff now
        agdc_config_dict = gdf_config_files_string = agdc2gdf_config_file_object.configuration['agdc']
        try:
            db_ref = agdc_config_dict['db_ref']
            host = agdc_config_dict['host']
            port = agdc_config_dict['port']
            dbname = agdc_config_dict['dbname']
            user = agdc_config_dict['user']
            password = agdc_config_dict['password']
            
            self.agdc_db = Database(db_ref=db_ref,
                                host=host, 
                                port=port, 
                                dbname=dbname, 
                                user=user, 
                                password=password, 
                                keep_connection=False, # Assume we don't want connections hanging around
                                autocommit=True)
            
            self.agdc_db.submit_query('select 1 as test_field') # Test DB connection
            logger.debug('Connected to database %s:%s for %s', host, dbname, db_ref)
        except Exception, e:
            logger.error('Unable to connect to database for %s: %s', db_ref, e.message)
            raise e
        
       
        # Set self.range_dict from either command line or config file values
        self.range_dict = {}
        for dimension in self.storage_type_config['dimensions']:
            min_value = int(self._command_line_params['%smin' % dimension.lower()] or agdc2gdf_config_file_object.configuration['agdc2gdf']['%smin' % dimension.lower()])
            max_value = int(self._command_line_params['%smax' % dimension.lower()] or agdc2gdf_config_file_object.configuration['agdc2gdf']['%smax' % dimension.lower()])
            self.range_dict[dimension] = (min_value, max_value)

        
        log_multiline(logger.debug, self.__dict__, 'AGDC2GDF.__dict__', '\t')        
Example #2
0
    def __init__(self):
        '''Constructor for class AGDC2GDF
        '''
        self._code_root = os.path.abspath(os.path.dirname(__file__)) # Directory containing module code
        self._gdf_root = os.path.abspath(os.path.dirname(gdf.__file__)) # Directory containing module code
        
        # Create master configuration dict containing both command line and config_file parameters
        self._command_line_params = self._get_command_line_params(AGDC2GDF.ARG_DESCRIPTORS)
        
        self.dryrun = self._command_line_params['dryrun']

        agdc2gdf_config_file = self._command_line_params['config_files'] or os.path.join(self._code_root, AGDC2GDF.DEFAULT_CONFIG_FILE)
        
        agdc2gdf_config_file_object = ConfigFile(agdc2gdf_config_file)
        
        # Comma separated list of GDF config files specified in master config file
        gdf_config_files_string = agdc2gdf_config_file_object.configuration['gdf'].get('config_files') or os.path.join(self._gdf_root, GDF.DEFAULT_CONFIG_FILE)
        
        # Create master GDF configuration dict containing both command line and config_file parameters
        self._configuration = self._get_config(gdf_config_files_string)
                
        self.temp_dir = self._command_line_params.get('temp_dir') or agdc2gdf_config_file_object.configuration['agdc']['temp_dir']
        # Try to create temp & cache directories if they don't exist
        if not directory_writable(self.temp_dir):
            new_temp_dir = os.path.join(os.path.expanduser("~"), 'gdf', 'temp')
            logger.warning('Unable to access temporary directory %s. Using %s instead.', self.temp_dir, new_temp_dir)
            self.temp_dir = new_temp_dir
            if not directory_writable(self.temp_dir):
                raise Exception('Unable to write to temporary directory %s', self.temp_dir)
            

        # Create master GDF database dictorage_config
        self._databases = self._get_dbs()
        
        self.force = self._command_line_params.get('force') or agdc2gdf_config_file_object.configuration['agdc2gdf'].get('force')
        
        logger.debug("self._command_line_params.get('storage_type') = %s", self._command_line_params.get('storage_type'))
        self.storage_type = self._command_line_params.get('storage_type') or agdc2gdf_config_file_object.configuration['gdf']['storage_type']

        self.agdc_satellite = self._command_line_params.get('satellite') or agdc2gdf_config_file_object.configuration['agdc']['satellite']

        self.agdc_sensors = self._command_line_params.get('sensors') or agdc2gdf_config_file_object.configuration['agdc']['sensors']
        self.agdc_sensors = tuple(self.agdc_sensors.split(','))

        self.agdc_level = self._command_line_params.get('level') or agdc2gdf_config_file_object.configuration['agdc']['level']
        
        # Read GDF storage configuration from databases
        self._storage_config = self._get_storage_config()
        self.storage_type_config = self._storage_config[self.storage_type]
        self.database = self._databases[self.storage_type_config['db_ref']]
        
        self.dimensions = self.storage_type_config['dimensions'] # This is used a lot
                
        
        # Set up AGDC stuff now
        agdc_config_dict = gdf_config_files_string = agdc2gdf_config_file_object.configuration['agdc']
        try:
            db_ref = agdc_config_dict['db_ref']
            host = agdc_config_dict['host']
            port = agdc_config_dict['port']
            dbname = agdc_config_dict['dbname']
            user = agdc_config_dict['user']
            password = agdc_config_dict['password']
            
            self.agdc_db = Database(db_ref=db_ref,
                                host=host, 
                                port=port, 
                                dbname=dbname, 
                                user=user, 
                                password=password, 
                                keep_connection=False, # Assume we don't want connections hanging around
                                autocommit=True)
            
            self.agdc_db.submit_query('select 1 as test_field') # Test DB connection
            logger.debug('Connected to database %s:%s for %s', host, dbname, db_ref)
        except Exception, e:
            logger.error('Unable to connect to database for %s: %s', db_ref, e.message)
            raise e
def main():

    database = Database(db_ref='agdc', host=DB_HOST, port=DB_PORT, dbname=DB_NAME, user=DB_USER, password=DB_PASSWORD)
    
    SQL='''-- Find all overlap tiles
SELECT * from ztmp.temp; # Here's one I prepared earlier...
/*
SELECT 
    level_name, 
    t1.tile_type_id, 
    t1.x_index, 
    t1.y_index,
    a1.start_datetime,
    a2.end_datetime,
    a1.acquisition_id AS acquisition_id1,
    d1.dataset_id AS dataset_id1,
    t1.tile_id AS tile_id1,
    a2.acquisition_id AS acquisition_id2,
    d2.dataset_id AS dataset_id2,
    t2.tile_id AS tile_id2,
    regexp_replace(regexp_replace(t1.tile_pathname, '(/(-)*\d{3}_(-)*\d{3}/\d{4}/)(.*)\.\w+$'::text, '\1mosaic_cache/\4.vrt'::text), '(.*_PQA_.*)\.vrt$'::text, '\1.tif'::text) AS tile_pathname,
    a1.x_ref as path, 
    a1.y_ref as row1, 
    a2.y_ref as row2,
    a1.end_datetime as first_end_datetime, 
    a2.start_datetime as second_start_datetime,
    d1.dataset_path as dataset_path1, 
    d2.dataset_path as dataset_path2,
    t1.tile_pathname as tile_pathname1, 
    t2.tile_pathname as tile_pathname2
   FROM acquisition a1
     JOIN dataset d1 ON d1.acquisition_id = a1.acquisition_id
     JOIN tile t1 ON t1.dataset_id = d1.dataset_id AND (t1.tile_class_id = 1 OR t1.tile_class_id = 3)
     JOIN acquisition a2 ON a1.satellite_id = a2.satellite_id AND a1.sensor_id = a2.sensor_id AND a1.x_ref = a2.x_ref AND a1.y_ref = (a2.y_ref - 1) AND a1.end_datetime - a2.start_datetime between interval '-12 seconds' and interval '12 seconds' AND a1.acquisition_id <> a2.acquisition_id
     JOIN dataset d2 ON d2.acquisition_id = a2.acquisition_id AND d1.level_id = d2.level_id
     JOIN tile t2 ON t2.dataset_id = d2.dataset_id AND t1.tile_type_id = t2.tile_type_id AND (t2.tile_class_id = 1 OR t2.tile_class_id = 3) AND t1.x_index = t2.x_index AND t1.y_index = t2.y_index
     JOIN processing_level on d1.level_id = processing_level.level_id
     LEFT JOIN tile mt ON mt.tile_class_id = 4 AND mt.dataset_id = t1.dataset_id AND mt.tile_type_id = t1.tile_type_id AND mt.x_index = t1.x_index AND mt.y_index = t1.y_index
where t1.tile_type_id = 1
and t1.tile_class_id in (1,3) --Non-overlapped & overlapped
and t2.tile_class_id in (1,3) --Non-overlapped & overlapped
and mt.tile_id is null -- Uncatalogued
order by t1.x_index, t1.y_index, level_name, a1.end_datetime;
*/
'''    
    
    uncatalogued_mosaics = database.submit_query(SQL)
    
    for record in uncatalogued_mosaics:
        mosaic_file_ok = False
        mosaic_tile_path = record['tile_pathname']
        
        try:
            if os.path.exists(mosaic_tile_path):
                dataset = gdal.Open(mosaic_tile_path)
                if not dataset:
                    logger.warning('Unable to open dataset %s using gdal', mosaic_tile_path)
                    os.remove(mosaic_tile_path)
                else:
                    dataset.Close()
                    mosaic_file_ok = True
            else:
                logger.warning('Dataset %s does not exist', mosaic_tile_path)
            
            if not mosaic_file_ok:
                create_mosaic_file(record)
            else:
                logger.info('Dataset %s is OK', mosaic_tile_path)
                
            write_mosaic_records(database, record)
                
        except Exception, e:
            logger.warning('Exception raised while processing %s: %s', mosaic_tile_path, e.message)     
Example #4
0
class AGDC2GDF(GDF):
    DEFAULT_CONFIG_FILE = 'agdc2gdf_default.conf'  # N.B: Assumed to reside in code root directory
    ARG_DESCRIPTORS = {
        'xmin': {
            'short_flag': '-x1',
            'long_flag': '--xmin',
            'default': None,
            'action': 'store',
            'const': None,
            'help':
            'Minimum X inclusive t (longitude) of spatial range to process'
        },
        'xmax': {
            'short_flag': '-x2',
            'long_flag': '--xmax',
            'default': None,
            'action': 'store',
            'const': None,
            'help':
            'Maximum X inclusive t (longitude) of spatial range to process'
        },
        'ymin': {
            'short_flag': '-y1',
            'long_flag': '--ymin',
            'default': None,
            'action': 'store',
            'const': None,
            'help':
            'Minimum Y inclusive t (latitude) of spatial range to process'
        },
        'ymax': {
            'short_flag': '-y2',
            'long_flag': '--ymax',
            'default': None,
            'action': 'store',
            'const': None,
            'help':
            'Maximum Y inclusive t (latitude) of spatial range to process'
        },
        'tmin': {
            'short_flag': '-t1',
            'long_flag': '--tmin',
            'default': None,
            'action': 'store',
            'const': None,
            'help': 'Minimum t inclusive t (years) of spatial range to process'
        },
        'tmax': {
            'short_flag': '-t2',
            'long_flag': '--tmax',
            'default': None,
            'action': 'store',
            'const': None,
            'help': 'Maximum inclusive t (years) of spatial range to process'
        },
        'storage_type': {
            'short_flag': '-st',
            'long_flag': '--storage_type',
            'default': None,
            'action': 'store',
            'const': None,
            'help': 'GDF storage type to populate'
        },
        'satellite': {
            'short_flag': '-sa',
            'long_flag': '--satellite',
            'default': None,
            'action': 'store',
            'const': None,
            'help': 'AGDC satellite to process'
        },
        'sensors': {
            'short_flag': '-se',
            'long_flag': '--sensors',
            'default': None,
            'action': 'store',
            'const': None,
            'help': 'Comma-separated list of AGDC sensors to process'
        },
        'level': {
            'short_flag': '-l',
            'long_flag': '--level',
            'default': None,
            'action': 'store',
            'const': None,
            'help': 'AGDC processing level to process'
        },
        'temp_dir': {
            'short_flag': '-t',
            'long_flag': '--temp_dir',
            'default': None,
            'action': 'store',
            'const': None,
            'help': 'Temporary directory for AGDC2GDF operation'
        },
        'force': {
            'short_flag': '-f',
            'long_flag': '--force',
            'default': False,
            'action': 'store_const',
            'const': True,
            'help': 'Flag to force replacement of existing files'
        },
        'dryrun': {
            'short_flag': '-dr',
            'long_flag': '--dryrun',
            'default': False,
            'action': 'store_const',
            'const': True,
            'help': 'Flag to skip file writing and SQL query execution'
        },
    }

    def __init__(self):
        '''Constructor for class AGDC2GDF
        '''
        self._code_root = os.path.abspath(
            os.path.dirname(__file__))  # Directory containing module code
        self._gdf_root = os.path.abspath(os.path.dirname(
            gdf.__file__))  # Directory containing module code

        # Create master configuration dict containing both command line and config_file parameters
        self._command_line_params = self._get_command_line_params(
            AGDC2GDF.ARG_DESCRIPTORS)

        self.dryrun = self._command_line_params['dryrun']

        agdc2gdf_config_file = self._command_line_params[
            'config_files'] or os.path.join(self._code_root,
                                            AGDC2GDF.DEFAULT_CONFIG_FILE)

        agdc2gdf_config_file_object = ConfigFile(agdc2gdf_config_file)

        # Comma separated list of GDF config files specified in master config file
        gdf_config_files_string = agdc2gdf_config_file_object.configuration[
            'gdf'].get('config_files') or os.path.join(self._gdf_root,
                                                       GDF.DEFAULT_CONFIG_FILE)

        # Create master GDF configuration dict containing both command line and config_file parameters
        self._configuration = self._get_config(gdf_config_files_string)

        self.temp_dir = self._command_line_params.get(
            'temp_dir'
        ) or agdc2gdf_config_file_object.configuration['agdc']['temp_dir']
        # Try to create temp & cache directories if they don't exist
        if not directory_writable(self.temp_dir):
            new_temp_dir = os.path.join(os.path.expanduser("~"), 'gdf', 'temp')
            logger.warning(
                'Unable to access temporary directory %s. Using %s instead.',
                self.temp_dir, new_temp_dir)
            self.temp_dir = new_temp_dir
            if not directory_writable(self.temp_dir):
                raise Exception('Unable to write to temporary directory %s',
                                self.temp_dir)

        # Create master GDF database dictorage_config
        self._databases = self._get_dbs()

        self.force = self._command_line_params.get(
            'force'
        ) or agdc2gdf_config_file_object.configuration['agdc2gdf'].get('force')

        logger.debug("self._command_line_params.get('storage_type') = %s",
                     self._command_line_params.get('storage_type'))
        self.storage_type = self._command_line_params.get(
            'storage_type'
        ) or agdc2gdf_config_file_object.configuration['gdf']['storage_type']

        self.agdc_satellite = self._command_line_params.get(
            'satellite'
        ) or agdc2gdf_config_file_object.configuration['agdc']['satellite']

        self.agdc_sensors = self._command_line_params.get(
            'sensors'
        ) or agdc2gdf_config_file_object.configuration['agdc']['sensors']
        self.agdc_sensors = tuple(self.agdc_sensors.split(','))

        self.agdc_level = self._command_line_params.get(
            'level'
        ) or agdc2gdf_config_file_object.configuration['agdc']['level']

        # Read GDF storage configuration from databases
        self._storage_config = self._get_storage_config()
        self.storage_type_config = self._storage_config[self.storage_type]
        self.database = self._databases[self.storage_type_config['db_ref']]

        self.dimensions = self.storage_type_config[
            'dimensions']  # This is used a lot

        # Set up AGDC stuff now
        agdc_config_dict = gdf_config_files_string = agdc2gdf_config_file_object.configuration[
            'agdc']
        try:
            db_ref = agdc_config_dict['db_ref']
            host = agdc_config_dict['host']
            port = agdc_config_dict['port']
            dbname = agdc_config_dict['dbname']
            user = agdc_config_dict['user']
            password = agdc_config_dict['password']

            self.agdc_db = Database(
                db_ref=db_ref,
                host=host,
                port=port,
                dbname=dbname,
                user=user,
                password=password,
                keep_connection=
                False,  # Assume we don't want connections hanging around
                autocommit=True)

            self.agdc_db.submit_query(
                'select 1 as test_field')  # Test DB connection
            logger.debug('Connected to database %s:%s for %s', host, dbname,
                         db_ref)
        except Exception, e:
            logger.error('Unable to connect to database for %s: %s', db_ref,
                         e.message)
            raise e

        # Set self.range_dict from either command line or config file values
        self.range_dict = {}
        for dimension in self.storage_type_config['dimensions']:
            min_value = int(
                self._command_line_params['%smin' % dimension.lower()]
                or agdc2gdf_config_file_object.configuration['agdc2gdf'][
                    '%smin' % dimension.lower()])
            max_value = int(
                self._command_line_params['%smax' % dimension.lower()]
                or agdc2gdf_config_file_object.configuration['agdc2gdf'][
                    '%smax' % dimension.lower()])
            self.range_dict[dimension] = (min_value, max_value)

        log_multiline(logger.debug, self.__dict__, 'AGDC2GDF.__dict__', '\t')
Example #5
0
    def __init__(self):
        '''Constructor for class AGDC2GDF
        '''
        self._code_root = os.path.abspath(
            os.path.dirname(__file__))  # Directory containing module code
        self._gdf_root = os.path.abspath(os.path.dirname(
            gdf.__file__))  # Directory containing module code

        # Create master configuration dict containing both command line and config_file parameters
        self._command_line_params = self._get_command_line_params(
            AGDC2GDF.ARG_DESCRIPTORS)

        self.dryrun = self._command_line_params['dryrun']

        agdc2gdf_config_file = self._command_line_params[
            'config_files'] or os.path.join(self._code_root,
                                            AGDC2GDF.DEFAULT_CONFIG_FILE)

        agdc2gdf_config_file_object = ConfigFile(agdc2gdf_config_file)

        # Comma separated list of GDF config files specified in master config file
        gdf_config_files_string = agdc2gdf_config_file_object.configuration[
            'gdf'].get('config_files') or os.path.join(self._gdf_root,
                                                       GDF.DEFAULT_CONFIG_FILE)

        # Create master GDF configuration dict containing both command line and config_file parameters
        self._configuration = self._get_config(gdf_config_files_string)

        self.temp_dir = self._command_line_params.get(
            'temp_dir'
        ) or agdc2gdf_config_file_object.configuration['agdc']['temp_dir']
        # Try to create temp & cache directories if they don't exist
        if not directory_writable(self.temp_dir):
            new_temp_dir = os.path.join(os.path.expanduser("~"), 'gdf', 'temp')
            logger.warning(
                'Unable to access temporary directory %s. Using %s instead.',
                self.temp_dir, new_temp_dir)
            self.temp_dir = new_temp_dir
            if not directory_writable(self.temp_dir):
                raise Exception('Unable to write to temporary directory %s',
                                self.temp_dir)

        # Create master GDF database dictorage_config
        self._databases = self._get_dbs()

        self.force = self._command_line_params.get(
            'force'
        ) or agdc2gdf_config_file_object.configuration['agdc2gdf'].get('force')

        logger.debug("self._command_line_params.get('storage_type') = %s",
                     self._command_line_params.get('storage_type'))
        self.storage_type = self._command_line_params.get(
            'storage_type'
        ) or agdc2gdf_config_file_object.configuration['gdf']['storage_type']

        self.agdc_satellite = self._command_line_params.get(
            'satellite'
        ) or agdc2gdf_config_file_object.configuration['agdc']['satellite']

        self.agdc_sensors = self._command_line_params.get(
            'sensors'
        ) or agdc2gdf_config_file_object.configuration['agdc']['sensors']
        self.agdc_sensors = tuple(self.agdc_sensors.split(','))

        self.agdc_level = self._command_line_params.get(
            'level'
        ) or agdc2gdf_config_file_object.configuration['agdc']['level']

        # Read GDF storage configuration from databases
        self._storage_config = self._get_storage_config()
        self.storage_type_config = self._storage_config[self.storage_type]
        self.database = self._databases[self.storage_type_config['db_ref']]

        self.dimensions = self.storage_type_config[
            'dimensions']  # This is used a lot

        # Set up AGDC stuff now
        agdc_config_dict = gdf_config_files_string = agdc2gdf_config_file_object.configuration[
            'agdc']
        try:
            db_ref = agdc_config_dict['db_ref']
            host = agdc_config_dict['host']
            port = agdc_config_dict['port']
            dbname = agdc_config_dict['dbname']
            user = agdc_config_dict['user']
            password = agdc_config_dict['password']

            self.agdc_db = Database(
                db_ref=db_ref,
                host=host,
                port=port,
                dbname=dbname,
                user=user,
                password=password,
                keep_connection=
                False,  # Assume we don't want connections hanging around
                autocommit=True)

            self.agdc_db.submit_query(
                'select 1 as test_field')  # Test DB connection
            logger.debug('Connected to database %s:%s for %s', host, dbname,
                         db_ref)
        except Exception, e:
            logger.error('Unable to connect to database for %s: %s', db_ref,
                         e.message)
            raise e
Example #6
0
def main():

    database = Database(db_ref='agdc',
                        host=DB_HOST,
                        port=DB_PORT,
                        dbname=DB_NAME,
                        user=DB_USER,
                        password=DB_PASSWORD)

    SQL = '''-- Find all overlap tiles
SELECT * from ztmp.temp; # Here's one I prepared earlier...
/*
SELECT 
    level_name, 
    t1.tile_type_id, 
    t1.x_index, 
    t1.y_index,
    a1.start_datetime,
    a2.end_datetime,
    a1.acquisition_id AS acquisition_id1,
    d1.dataset_id AS dataset_id1,
    t1.tile_id AS tile_id1,
    a2.acquisition_id AS acquisition_id2,
    d2.dataset_id AS dataset_id2,
    t2.tile_id AS tile_id2,
    regexp_replace(regexp_replace(t1.tile_pathname, '(/(-)*\d{3}_(-)*\d{3}/\d{4}/)(.*)\.\w+$'::text, '\1mosaic_cache/\4.vrt'::text), '(.*_PQA_.*)\.vrt$'::text, '\1.tif'::text) AS tile_pathname,
    a1.x_ref as path, 
    a1.y_ref as row1, 
    a2.y_ref as row2,
    a1.end_datetime as first_end_datetime, 
    a2.start_datetime as second_start_datetime,
    d1.dataset_path as dataset_path1, 
    d2.dataset_path as dataset_path2,
    t1.tile_pathname as tile_pathname1, 
    t2.tile_pathname as tile_pathname2
   FROM acquisition a1
     JOIN dataset d1 ON d1.acquisition_id = a1.acquisition_id
     JOIN tile t1 ON t1.dataset_id = d1.dataset_id AND (t1.tile_class_id = 1 OR t1.tile_class_id = 3)
     JOIN acquisition a2 ON a1.satellite_id = a2.satellite_id AND a1.sensor_id = a2.sensor_id AND a1.x_ref = a2.x_ref AND a1.y_ref = (a2.y_ref - 1) AND a1.end_datetime - a2.start_datetime between interval '-12 seconds' and interval '12 seconds' AND a1.acquisition_id <> a2.acquisition_id
     JOIN dataset d2 ON d2.acquisition_id = a2.acquisition_id AND d1.level_id = d2.level_id
     JOIN tile t2 ON t2.dataset_id = d2.dataset_id AND t1.tile_type_id = t2.tile_type_id AND (t2.tile_class_id = 1 OR t2.tile_class_id = 3) AND t1.x_index = t2.x_index AND t1.y_index = t2.y_index
     JOIN processing_level on d1.level_id = processing_level.level_id
     LEFT JOIN tile mt ON mt.tile_class_id = 4 AND mt.dataset_id = t1.dataset_id AND mt.tile_type_id = t1.tile_type_id AND mt.x_index = t1.x_index AND mt.y_index = t1.y_index
where t1.tile_type_id = 1
and t1.tile_class_id in (1,3) --Non-overlapped & overlapped
and t2.tile_class_id in (1,3) --Non-overlapped & overlapped
and mt.tile_id is null -- Uncatalogued
order by t1.x_index, t1.y_index, level_name, a1.end_datetime;
*/
'''

    uncatalogued_mosaics = database.submit_query(SQL)

    for record in uncatalogued_mosaics:
        mosaic_file_ok = False
        mosaic_tile_path = record['tile_pathname']

        try:
            if os.path.exists(mosaic_tile_path):
                dataset = gdal.Open(mosaic_tile_path)
                if not dataset:
                    logger.warning('Unable to open dataset %s using gdal',
                                   mosaic_tile_path)
                    os.remove(mosaic_tile_path)
                else:
                    dataset.Close()
                    mosaic_file_ok = True
            else:
                logger.warning('Dataset %s does not exist', mosaic_tile_path)

            if not mosaic_file_ok:
                create_mosaic_file(record)
            else:
                logger.info('Dataset %s is OK', mosaic_tile_path)

            write_mosaic_records(database, record)

        except Exception, e:
            logger.warning('Exception raised while processing %s: %s',
                           mosaic_tile_path, e.message)