class AGDC2GDF(GDF):
DEFAULT_CONFIG_FILE = 'agdc2gdf_default.conf' # N.B: Assumed to reside in code root directory
ARG_DESCRIPTORS = {'xmin': {'short_flag': '-x1',
'long_flag': '--xmin',
'default': None,
'action': 'store',
'const': None,
'help': 'Minimum X inclusive t (longitude) of spatial range to process'
},
'xmax': {'short_flag': '-x2',
'long_flag': '--xmax',
'default': None,
'action': 'store',
'const': None,
'help': 'Maximum X inclusive t (longitude) of spatial range to process'
},
'ymin': {'short_flag': '-y1',
'long_flag': '--ymin',
'default': None,
'action': 'store',
'const': None,
'help': 'Minimum Y inclusive t (latitude) of spatial range to process'
},
'ymax': {'short_flag': '-y2',
'long_flag': '--ymax',
'default': None,
'action': 'store',
'const': None,
'help': 'Maximum Y inclusive t (latitude) of spatial range to process'
},
'tmin': {'short_flag': '-t1',
'long_flag': '--tmin',
'default': None,
'action': 'store',
'const': None,
'help': 'Minimum t inclusive t (years) of spatial range to process'
},
'tmax': {'short_flag': '-t2',
'long_flag': '--tmax',
'default': None,
'action': 'store',
'const': None,
'help': 'Maximum inclusive t (years) of spatial range to process'
},
'storage_type': {'short_flag': '-st',
'long_flag': '--storage_type',
'default': None,
'action': 'store',
'const': None,
'help': 'GDF storage type to populate'
},
'satellite': {'short_flag': '-sa',
'long_flag': '--satellite',
'default': None,
'action': 'store',
'const': None,
'help': 'AGDC satellite to process'
},
'sensors': {'short_flag': '-se',
'long_flag': '--sensors',
'default': None,
'action': 'store',
'const': None,
'help': 'Comma-separated list of AGDC sensors to process'
},
'level': {'short_flag': '-l',
'long_flag': '--level',
'default': None,
'action': 'store',
'const': None,
'help': 'AGDC processing level to process'
},
'temp_dir': {'short_flag': '-t',
'long_flag': '--temp_dir',
'default': None,
'action': 'store',
'const': None,
'help': 'Temporary directory for AGDC2GDF operation'
},
'force': {'short_flag': '-f',
'long_flag': '--force',
'default': False,
'action': 'store_const',
'const': True,
'help': 'Flag to force replacement of existing files'
},
'dryrun': {'short_flag': '-dr',
'long_flag': '--dryrun',
'default': False,
'action': 'store_const',
'const': True,
'help': 'Flag to skip file writing and SQL query execution'
},
}
def __init__(self):
'''Constructor for class AGDC2GDF
'''
self._code_root = os.path.abspath(os.path.dirname(__file__)) # Directory containing module code
self._gdf_root = os.path.abspath(os.path.dirname(gdf.__file__)) # Directory containing module code
# Create master configuration dict containing both command line and config_file parameters
self._command_line_params = self._get_command_line_params(AGDC2GDF.ARG_DESCRIPTORS)
self.dryrun = self._command_line_params['dryrun']
agdc2gdf_config_file = self._command_line_params['config_files'] or os.path.join(self._code_root, AGDC2GDF.DEFAULT_CONFIG_FILE)
agdc2gdf_config_file_object = ConfigFile(agdc2gdf_config_file)
# Comma separated list of GDF config files specified in master config file
gdf_config_files_string = agdc2gdf_config_file_object.configuration['gdf'].get('config_files') or os.path.join(self._gdf_root, GDF.DEFAULT_CONFIG_FILE)
# Create master GDF configuration dict containing both command line and config_file parameters
self._configuration = self._get_config(gdf_config_files_string)
self.temp_dir = self._command_line_params.get('temp_dir') or agdc2gdf_config_file_object.configuration['agdc']['temp_dir']
# Try to create temp & cache directories if they don't exist
if not directory_writable(self.temp_dir):
new_temp_dir = os.path.join(os.path.expanduser("~"), 'gdf', 'temp')
logger.warning('Unable to access temporary directory %s. Using %s instead.', self.temp_dir, new_temp_dir)
self.temp_dir = new_temp_dir
if not directory_writable(self.temp_dir):
raise Exception('Unable to write to temporary directory %s', self.temp_dir)
# Create master GDF database dictorage_config
self._databases = self._get_dbs()
self.force = self._command_line_params.get('force') or agdc2gdf_config_file_object.configuration['agdc2gdf'].get('force')
logger.debug("self._command_line_params.get('storage_type') = %s", self._command_line_params.get('storage_type'))
self.storage_type = self._command_line_params.get('storage_type') or agdc2gdf_config_file_object.configuration['gdf']['storage_type']
self.agdc_satellite = self._command_line_params.get('satellite') or agdc2gdf_config_file_object.configuration['agdc']['satellite']
self.agdc_sensors = self._command_line_params.get('sensors') or agdc2gdf_config_file_object.configuration['agdc']['sensors']
self.agdc_sensors = tuple(self.agdc_sensors.split(','))
self.agdc_level = self._command_line_params.get('level') or agdc2gdf_config_file_object.configuration['agdc']['level']
# Read GDF storage configuration from databases
self._storage_config = self._get_storage_config()
self.storage_type_config = self._storage_config[self.storage_type]
self.database = self._databases[self.storage_type_config['db_ref']]
self.dimensions = self.storage_type_config['dimensions'] # This is used a lot
# Set up AGDC stuff now
agdc_config_dict = gdf_config_files_string = agdc2gdf_config_file_object.configuration['agdc']
try:
db_ref = agdc_config_dict['db_ref']
host = agdc_config_dict['host']
port = agdc_config_dict['port']
dbname = agdc_config_dict['dbname']
user = agdc_config_dict['user']
password = agdc_config_dict['password']
self.agdc_db = Database(db_ref=db_ref,
host=host,
port=port,
dbname=dbname,
user=user,
password=password,
keep_connection=False, # Assume we don't want connections hanging around
autocommit=True)
self.agdc_db.submit_query('select 1 as test_field') # Test DB connection
logger.debug('Connected to database %s:%s for %s', host, dbname, db_ref)
except Exception, e:
logger.error('Unable to connect to database for %s: %s', db_ref, e.message)
raise e
# Set self.range_dict from either command line or config file values
self.range_dict = {}
for dimension in self.storage_type_config['dimensions']:
min_value = int(self._command_line_params['%smin' % dimension.lower()] or agdc2gdf_config_file_object.configuration['agdc2gdf']['%smin' % dimension.lower()])
max_value = int(self._command_line_params['%smax' % dimension.lower()] or agdc2gdf_config_file_object.configuration['agdc2gdf']['%smax' % dimension.lower()])
self.range_dict[dimension] = (min_value, max_value)
log_multiline(logger.debug, self.__dict__, 'AGDC2GDF.__dict__', '\t')
def __init__(self):
'''Constructor for class AGDC2GDF
'''
self._code_root = os.path.abspath(os.path.dirname(__file__)) # Directory containing module code
self._gdf_root = os.path.abspath(os.path.dirname(gdf.__file__)) # Directory containing module code
# Create master configuration dict containing both command line and config_file parameters
self._command_line_params = self._get_command_line_params(AGDC2GDF.ARG_DESCRIPTORS)
self.dryrun = self._command_line_params['dryrun']
agdc2gdf_config_file = self._command_line_params['config_files'] or os.path.join(self._code_root, AGDC2GDF.DEFAULT_CONFIG_FILE)
agdc2gdf_config_file_object = ConfigFile(agdc2gdf_config_file)
# Comma separated list of GDF config files specified in master config file
gdf_config_files_string = agdc2gdf_config_file_object.configuration['gdf'].get('config_files') or os.path.join(self._gdf_root, GDF.DEFAULT_CONFIG_FILE)
# Create master GDF configuration dict containing both command line and config_file parameters
self._configuration = self._get_config(gdf_config_files_string)
self.temp_dir = self._command_line_params.get('temp_dir') or agdc2gdf_config_file_object.configuration['agdc']['temp_dir']
# Try to create temp & cache directories if they don't exist
if not directory_writable(self.temp_dir):
new_temp_dir = os.path.join(os.path.expanduser("~"), 'gdf', 'temp')
logger.warning('Unable to access temporary directory %s. Using %s instead.', self.temp_dir, new_temp_dir)
self.temp_dir = new_temp_dir
if not directory_writable(self.temp_dir):
raise Exception('Unable to write to temporary directory %s', self.temp_dir)
# Create master GDF database dictorage_config
self._databases = self._get_dbs()
self.force = self._command_line_params.get('force') or agdc2gdf_config_file_object.configuration['agdc2gdf'].get('force')
logger.debug("self._command_line_params.get('storage_type') = %s", self._command_line_params.get('storage_type'))
self.storage_type = self._command_line_params.get('storage_type') or agdc2gdf_config_file_object.configuration['gdf']['storage_type']
self.agdc_satellite = self._command_line_params.get('satellite') or agdc2gdf_config_file_object.configuration['agdc']['satellite']
self.agdc_sensors = self._command_line_params.get('sensors') or agdc2gdf_config_file_object.configuration['agdc']['sensors']
self.agdc_sensors = tuple(self.agdc_sensors.split(','))
self.agdc_level = self._command_line_params.get('level') or agdc2gdf_config_file_object.configuration['agdc']['level']
# Read GDF storage configuration from databases
self._storage_config = self._get_storage_config()
self.storage_type_config = self._storage_config[self.storage_type]
self.database = self._databases[self.storage_type_config['db_ref']]
self.dimensions = self.storage_type_config['dimensions'] # This is used a lot
# Set up AGDC stuff now
agdc_config_dict = gdf_config_files_string = agdc2gdf_config_file_object.configuration['agdc']
try:
db_ref = agdc_config_dict['db_ref']
host = agdc_config_dict['host']
port = agdc_config_dict['port']
dbname = agdc_config_dict['dbname']
user = agdc_config_dict['user']
password = agdc_config_dict['password']
self.agdc_db = Database(db_ref=db_ref,
host=host,
port=port,
dbname=dbname,
user=user,
password=password,
keep_connection=False, # Assume we don't want connections hanging around
autocommit=True)
self.agdc_db.submit_query('select 1 as test_field') # Test DB connection
logger.debug('Connected to database %s:%s for %s', host, dbname, db_ref)
except Exception, e:
logger.error('Unable to connect to database for %s: %s', db_ref, e.message)
raise e
def main():
database = Database(db_ref='agdc', host=DB_HOST, port=DB_PORT, dbname=DB_NAME, user=DB_USER, password=DB_PASSWORD)
SQL='''-- Find all overlap tiles
SELECT * from ztmp.temp; # Here's one I prepared earlier...
/*
SELECT
level_name,
t1.tile_type_id,
t1.x_index,
t1.y_index,
a1.start_datetime,
a2.end_datetime,
a1.acquisition_id AS acquisition_id1,
d1.dataset_id AS dataset_id1,
t1.tile_id AS tile_id1,
a2.acquisition_id AS acquisition_id2,
d2.dataset_id AS dataset_id2,
t2.tile_id AS tile_id2,
regexp_replace(regexp_replace(t1.tile_pathname, '(/(-)*\d{3}_(-)*\d{3}/\d{4}/)(.*)\.\w+$'::text, '\1mosaic_cache/\4.vrt'::text), '(.*_PQA_.*)\.vrt$'::text, '\1.tif'::text) AS tile_pathname,
a1.x_ref as path,
a1.y_ref as row1,
a2.y_ref as row2,
a1.end_datetime as first_end_datetime,
a2.start_datetime as second_start_datetime,
d1.dataset_path as dataset_path1,
d2.dataset_path as dataset_path2,
t1.tile_pathname as tile_pathname1,
t2.tile_pathname as tile_pathname2
FROM acquisition a1
JOIN dataset d1 ON d1.acquisition_id = a1.acquisition_id
JOIN tile t1 ON t1.dataset_id = d1.dataset_id AND (t1.tile_class_id = 1 OR t1.tile_class_id = 3)
JOIN acquisition a2 ON a1.satellite_id = a2.satellite_id AND a1.sensor_id = a2.sensor_id AND a1.x_ref = a2.x_ref AND a1.y_ref = (a2.y_ref - 1) AND a1.end_datetime - a2.start_datetime between interval '-12 seconds' and interval '12 seconds' AND a1.acquisition_id <> a2.acquisition_id
JOIN dataset d2 ON d2.acquisition_id = a2.acquisition_id AND d1.level_id = d2.level_id
JOIN tile t2 ON t2.dataset_id = d2.dataset_id AND t1.tile_type_id = t2.tile_type_id AND (t2.tile_class_id = 1 OR t2.tile_class_id = 3) AND t1.x_index = t2.x_index AND t1.y_index = t2.y_index
JOIN processing_level on d1.level_id = processing_level.level_id
LEFT JOIN tile mt ON mt.tile_class_id = 4 AND mt.dataset_id = t1.dataset_id AND mt.tile_type_id = t1.tile_type_id AND mt.x_index = t1.x_index AND mt.y_index = t1.y_index
where t1.tile_type_id = 1
and t1.tile_class_id in (1,3) --Non-overlapped & overlapped
and t2.tile_class_id in (1,3) --Non-overlapped & overlapped
and mt.tile_id is null -- Uncatalogued
order by t1.x_index, t1.y_index, level_name, a1.end_datetime;
*/
'''
uncatalogued_mosaics = database.submit_query(SQL)
for record in uncatalogued_mosaics:
mosaic_file_ok = False
mosaic_tile_path = record['tile_pathname']
try:
if os.path.exists(mosaic_tile_path):
dataset = gdal.Open(mosaic_tile_path)
if not dataset:
logger.warning('Unable to open dataset %s using gdal', mosaic_tile_path)
os.remove(mosaic_tile_path)
else:
dataset.Close()
mosaic_file_ok = True
else:
logger.warning('Dataset %s does not exist', mosaic_tile_path)
if not mosaic_file_ok:
create_mosaic_file(record)
else:
logger.info('Dataset %s is OK', mosaic_tile_path)
write_mosaic_records(database, record)
except Exception, e:
logger.warning('Exception raised while processing %s: %s', mosaic_tile_path, e.message)
class AGDC2GDF(GDF):
DEFAULT_CONFIG_FILE = 'agdc2gdf_default.conf' # N.B: Assumed to reside in code root directory
ARG_DESCRIPTORS = {
'xmin': {
'short_flag': '-x1',
'long_flag': '--xmin',
'default': None,
'action': 'store',
'const': None,
'help':
'Minimum X inclusive t (longitude) of spatial range to process'
},
'xmax': {
'short_flag': '-x2',
'long_flag': '--xmax',
'default': None,
'action': 'store',
'const': None,
'help':
'Maximum X inclusive t (longitude) of spatial range to process'
},
'ymin': {
'short_flag': '-y1',
'long_flag': '--ymin',
'default': None,
'action': 'store',
'const': None,
'help':
'Minimum Y inclusive t (latitude) of spatial range to process'
},
'ymax': {
'short_flag': '-y2',
'long_flag': '--ymax',
'default': None,
'action': 'store',
'const': None,
'help':
'Maximum Y inclusive t (latitude) of spatial range to process'
},
'tmin': {
'short_flag': '-t1',
'long_flag': '--tmin',
'default': None,
'action': 'store',
'const': None,
'help': 'Minimum t inclusive t (years) of spatial range to process'
},
'tmax': {
'short_flag': '-t2',
'long_flag': '--tmax',
'default': None,
'action': 'store',
'const': None,
'help': 'Maximum inclusive t (years) of spatial range to process'
},
'storage_type': {
'short_flag': '-st',
'long_flag': '--storage_type',
'default': None,
'action': 'store',
'const': None,
'help': 'GDF storage type to populate'
},
'satellite': {
'short_flag': '-sa',
'long_flag': '--satellite',
'default': None,
'action': 'store',
'const': None,
'help': 'AGDC satellite to process'
},
'sensors': {
'short_flag': '-se',
'long_flag': '--sensors',
'default': None,
'action': 'store',
'const': None,
'help': 'Comma-separated list of AGDC sensors to process'
},
'level': {
'short_flag': '-l',
'long_flag': '--level',
'default': None,
'action': 'store',
'const': None,
'help': 'AGDC processing level to process'
},
'temp_dir': {
'short_flag': '-t',
'long_flag': '--temp_dir',
'default': None,
'action': 'store',
'const': None,
'help': 'Temporary directory for AGDC2GDF operation'
},
'force': {
'short_flag': '-f',
'long_flag': '--force',
'default': False,
'action': 'store_const',
'const': True,
'help': 'Flag to force replacement of existing files'
},
'dryrun': {
'short_flag': '-dr',
'long_flag': '--dryrun',
'default': False,
'action': 'store_const',
'const': True,
'help': 'Flag to skip file writing and SQL query execution'
},
}
def __init__(self):
'''Constructor for class AGDC2GDF
'''
self._code_root = os.path.abspath(
os.path.dirname(__file__)) # Directory containing module code
self._gdf_root = os.path.abspath(os.path.dirname(
gdf.__file__)) # Directory containing module code
# Create master configuration dict containing both command line and config_file parameters
self._command_line_params = self._get_command_line_params(
AGDC2GDF.ARG_DESCRIPTORS)
self.dryrun = self._command_line_params['dryrun']
agdc2gdf_config_file = self._command_line_params[
'config_files'] or os.path.join(self._code_root,
AGDC2GDF.DEFAULT_CONFIG_FILE)
agdc2gdf_config_file_object = ConfigFile(agdc2gdf_config_file)
# Comma separated list of GDF config files specified in master config file
gdf_config_files_string = agdc2gdf_config_file_object.configuration[
'gdf'].get('config_files') or os.path.join(self._gdf_root,
GDF.DEFAULT_CONFIG_FILE)
# Create master GDF configuration dict containing both command line and config_file parameters
self._configuration = self._get_config(gdf_config_files_string)
self.temp_dir = self._command_line_params.get(
'temp_dir'
) or agdc2gdf_config_file_object.configuration['agdc']['temp_dir']
# Try to create temp & cache directories if they don't exist
if not directory_writable(self.temp_dir):
new_temp_dir = os.path.join(os.path.expanduser("~"), 'gdf', 'temp')
logger.warning(
'Unable to access temporary directory %s. Using %s instead.',
self.temp_dir, new_temp_dir)
self.temp_dir = new_temp_dir
if not directory_writable(self.temp_dir):
raise Exception('Unable to write to temporary directory %s',
self.temp_dir)
# Create master GDF database dictorage_config
self._databases = self._get_dbs()
self.force = self._command_line_params.get(
'force'
) or agdc2gdf_config_file_object.configuration['agdc2gdf'].get('force')
logger.debug("self._command_line_params.get('storage_type') = %s",
self._command_line_params.get('storage_type'))
self.storage_type = self._command_line_params.get(
'storage_type'
) or agdc2gdf_config_file_object.configuration['gdf']['storage_type']
self.agdc_satellite = self._command_line_params.get(
'satellite'
) or agdc2gdf_config_file_object.configuration['agdc']['satellite']
self.agdc_sensors = self._command_line_params.get(
'sensors'
) or agdc2gdf_config_file_object.configuration['agdc']['sensors']
self.agdc_sensors = tuple(self.agdc_sensors.split(','))
self.agdc_level = self._command_line_params.get(
'level'
) or agdc2gdf_config_file_object.configuration['agdc']['level']
# Read GDF storage configuration from databases
self._storage_config = self._get_storage_config()
self.storage_type_config = self._storage_config[self.storage_type]
self.database = self._databases[self.storage_type_config['db_ref']]
self.dimensions = self.storage_type_config[
'dimensions'] # This is used a lot
# Set up AGDC stuff now
agdc_config_dict = gdf_config_files_string = agdc2gdf_config_file_object.configuration[
'agdc']
try:
db_ref = agdc_config_dict['db_ref']
host = agdc_config_dict['host']
port = agdc_config_dict['port']
dbname = agdc_config_dict['dbname']
user = agdc_config_dict['user']
password = agdc_config_dict['password']
self.agdc_db = Database(
db_ref=db_ref,
host=host,
port=port,
dbname=dbname,
user=user,
password=password,
keep_connection=
False, # Assume we don't want connections hanging around
autocommit=True)
self.agdc_db.submit_query(
'select 1 as test_field') # Test DB connection
logger.debug('Connected to database %s:%s for %s', host, dbname,
db_ref)
except Exception, e:
logger.error('Unable to connect to database for %s: %s', db_ref,
e.message)
raise e
# Set self.range_dict from either command line or config file values
self.range_dict = {}
for dimension in self.storage_type_config['dimensions']:
min_value = int(
self._command_line_params['%smin' % dimension.lower()]
or agdc2gdf_config_file_object.configuration['agdc2gdf'][
'%smin' % dimension.lower()])
max_value = int(
self._command_line_params['%smax' % dimension.lower()]
or agdc2gdf_config_file_object.configuration['agdc2gdf'][
'%smax' % dimension.lower()])
self.range_dict[dimension] = (min_value, max_value)
log_multiline(logger.debug, self.__dict__, 'AGDC2GDF.__dict__', '\t')
def __init__(self):
'''Constructor for class AGDC2GDF
'''
self._code_root = os.path.abspath(
os.path.dirname(__file__)) # Directory containing module code
self._gdf_root = os.path.abspath(os.path.dirname(
gdf.__file__)) # Directory containing module code
# Create master configuration dict containing both command line and config_file parameters
self._command_line_params = self._get_command_line_params(
AGDC2GDF.ARG_DESCRIPTORS)
self.dryrun = self._command_line_params['dryrun']
agdc2gdf_config_file = self._command_line_params[
'config_files'] or os.path.join(self._code_root,
AGDC2GDF.DEFAULT_CONFIG_FILE)
agdc2gdf_config_file_object = ConfigFile(agdc2gdf_config_file)
# Comma separated list of GDF config files specified in master config file
gdf_config_files_string = agdc2gdf_config_file_object.configuration[
'gdf'].get('config_files') or os.path.join(self._gdf_root,
GDF.DEFAULT_CONFIG_FILE)
# Create master GDF configuration dict containing both command line and config_file parameters
self._configuration = self._get_config(gdf_config_files_string)
self.temp_dir = self._command_line_params.get(
'temp_dir'
) or agdc2gdf_config_file_object.configuration['agdc']['temp_dir']
# Try to create temp & cache directories if they don't exist
if not directory_writable(self.temp_dir):
new_temp_dir = os.path.join(os.path.expanduser("~"), 'gdf', 'temp')
logger.warning(
'Unable to access temporary directory %s. Using %s instead.',
self.temp_dir, new_temp_dir)
self.temp_dir = new_temp_dir
if not directory_writable(self.temp_dir):
raise Exception('Unable to write to temporary directory %s',
self.temp_dir)
# Create master GDF database dictorage_config
self._databases = self._get_dbs()
self.force = self._command_line_params.get(
'force'
) or agdc2gdf_config_file_object.configuration['agdc2gdf'].get('force')
logger.debug("self._command_line_params.get('storage_type') = %s",
self._command_line_params.get('storage_type'))
self.storage_type = self._command_line_params.get(
'storage_type'
) or agdc2gdf_config_file_object.configuration['gdf']['storage_type']
self.agdc_satellite = self._command_line_params.get(
'satellite'
) or agdc2gdf_config_file_object.configuration['agdc']['satellite']
self.agdc_sensors = self._command_line_params.get(
'sensors'
) or agdc2gdf_config_file_object.configuration['agdc']['sensors']
self.agdc_sensors = tuple(self.agdc_sensors.split(','))
self.agdc_level = self._command_line_params.get(
'level'
) or agdc2gdf_config_file_object.configuration['agdc']['level']
# Read GDF storage configuration from databases
self._storage_config = self._get_storage_config()
self.storage_type_config = self._storage_config[self.storage_type]
self.database = self._databases[self.storage_type_config['db_ref']]
self.dimensions = self.storage_type_config[
'dimensions'] # This is used a lot
# Set up AGDC stuff now
agdc_config_dict = gdf_config_files_string = agdc2gdf_config_file_object.configuration[
'agdc']
try:
db_ref = agdc_config_dict['db_ref']
host = agdc_config_dict['host']
port = agdc_config_dict['port']
dbname = agdc_config_dict['dbname']
user = agdc_config_dict['user']
password = agdc_config_dict['password']
self.agdc_db = Database(
db_ref=db_ref,
host=host,
port=port,
dbname=dbname,
user=user,
password=password,
keep_connection=
False, # Assume we don't want connections hanging around
autocommit=True)
self.agdc_db.submit_query(
'select 1 as test_field') # Test DB connection
logger.debug('Connected to database %s:%s for %s', host, dbname,
db_ref)
except Exception, e:
logger.error('Unable to connect to database for %s: %s', db_ref,
e.message)
raise e
def main():
database = Database(db_ref='agdc',
host=DB_HOST,
port=DB_PORT,
dbname=DB_NAME,
user=DB_USER,
password=DB_PASSWORD)
SQL = '''-- Find all overlap tiles
SELECT * from ztmp.temp; # Here's one I prepared earlier...
/*
SELECT
level_name,
t1.tile_type_id,
t1.x_index,
t1.y_index,
a1.start_datetime,
a2.end_datetime,
a1.acquisition_id AS acquisition_id1,
d1.dataset_id AS dataset_id1,
t1.tile_id AS tile_id1,
a2.acquisition_id AS acquisition_id2,
d2.dataset_id AS dataset_id2,
t2.tile_id AS tile_id2,
regexp_replace(regexp_replace(t1.tile_pathname, '(/(-)*\d{3}_(-)*\d{3}/\d{4}/)(.*)\.\w+$'::text, '\1mosaic_cache/\4.vrt'::text), '(.*_PQA_.*)\.vrt$'::text, '\1.tif'::text) AS tile_pathname,
a1.x_ref as path,
a1.y_ref as row1,
a2.y_ref as row2,
a1.end_datetime as first_end_datetime,
a2.start_datetime as second_start_datetime,
d1.dataset_path as dataset_path1,
d2.dataset_path as dataset_path2,
t1.tile_pathname as tile_pathname1,
t2.tile_pathname as tile_pathname2
FROM acquisition a1
JOIN dataset d1 ON d1.acquisition_id = a1.acquisition_id
JOIN tile t1 ON t1.dataset_id = d1.dataset_id AND (t1.tile_class_id = 1 OR t1.tile_class_id = 3)
JOIN acquisition a2 ON a1.satellite_id = a2.satellite_id AND a1.sensor_id = a2.sensor_id AND a1.x_ref = a2.x_ref AND a1.y_ref = (a2.y_ref - 1) AND a1.end_datetime - a2.start_datetime between interval '-12 seconds' and interval '12 seconds' AND a1.acquisition_id <> a2.acquisition_id
JOIN dataset d2 ON d2.acquisition_id = a2.acquisition_id AND d1.level_id = d2.level_id
JOIN tile t2 ON t2.dataset_id = d2.dataset_id AND t1.tile_type_id = t2.tile_type_id AND (t2.tile_class_id = 1 OR t2.tile_class_id = 3) AND t1.x_index = t2.x_index AND t1.y_index = t2.y_index
JOIN processing_level on d1.level_id = processing_level.level_id
LEFT JOIN tile mt ON mt.tile_class_id = 4 AND mt.dataset_id = t1.dataset_id AND mt.tile_type_id = t1.tile_type_id AND mt.x_index = t1.x_index AND mt.y_index = t1.y_index
where t1.tile_type_id = 1
and t1.tile_class_id in (1,3) --Non-overlapped & overlapped
and t2.tile_class_id in (1,3) --Non-overlapped & overlapped
and mt.tile_id is null -- Uncatalogued
order by t1.x_index, t1.y_index, level_name, a1.end_datetime;
*/
'''
uncatalogued_mosaics = database.submit_query(SQL)
for record in uncatalogued_mosaics:
mosaic_file_ok = False
mosaic_tile_path = record['tile_pathname']
try:
if os.path.exists(mosaic_tile_path):
dataset = gdal.Open(mosaic_tile_path)
if not dataset:
logger.warning('Unable to open dataset %s using gdal',
mosaic_tile_path)
os.remove(mosaic_tile_path)
else:
dataset.Close()
mosaic_file_ok = True
else:
logger.warning('Dataset %s does not exist', mosaic_tile_path)
if not mosaic_file_ok:
create_mosaic_file(record)
else:
logger.info('Dataset %s is OK', mosaic_tile_path)
write_mosaic_records(database, record)
except Exception, e:
logger.warning('Exception raised while processing %s: %s',
mosaic_tile_path, e.message)