def __init__(self, db_ref, host, port, dbname, user, password, keep_connection=True, autocommit=True):
'''
Constructor for class Database.
Parameters:
host: PostgreSQL database host
port: PostgreSQL database port
dbname: PostgreSQL database database name
user: PostgreSQL database user
password: PostgreSQL database password for user
'''
self._db_ref = db_ref
self._host = host
self._port = port
self._dbname = dbname
self._user = user
self._password = password
self._keep_connection = keep_connection
self._autocommit = autocommit
self._default_connection = None
self._default_cursor = None
self._setup_default_cursor()
log_multiline(logger.debug, self.__dict__, 'Database.__dict__', '\t')
def set_dimension(dimension, dimension_config, index, dimension_index_vector=None):
"""
Parameters:
dimension: Dimension tag (e.g. X, Y, T, etc.)
dimension_config: Nested dict containing storage configuration from GDF.storage_config['<storage_type>']
index: index for storage unit
dimension_index_vector: Numpy array of index values for irregular dimension (e.g. time) or None for unlimited irregular dimension
"""
logger.debug("dimension = %s", dimension)
logger.debug("dimension_config = %s", dimension_config)
logger.debug("index = %s", index)
logger.debug("dimension_index_vector = %s", dimension_index_vector)
if dimension_config["indexing_type"] == "regular" and not dimension_index_vector:
element_size = dimension_config["dimension_element_size"]
dimension_min = (
index * dimension_config["dimension_extent"]
+ dimension_config["dimension_origin"]
+ element_size / 2.0
) # Half pixel to account for netCDF centre of pixel reference
dimension_max = dimension_min + dimension_config["dimension_extent"]
dimension_index_vector = np.around(
np.arange(dimension_min, dimension_max, element_size), self.decimal_places
)
# Cater for reversed index (e.g. positive Y index tends Southwards when image origin is in UL/NW corner)
if dimension_config["reverse_index"]:
dimension_index_vector = dimension_index_vector[::-1]
# TODO: Implement fixed indexing type
log_multiline(logger.debug, dimension_index_vector, "dimension_index_vector for %s" % dimension, "\t")
if dimension_index_vector is not None:
dimension_index_shape = dimension_index_vector.shape
assert len(dimension_index_shape) == 1, "Invalid dimension_index_vector shape. Must be 1D"
assert dimension_index_shape[0] <= dimension_config["dimension_elements"], (
"dimension_index_vector must have %d elements or fewer" % dimension_config["dimension_elements"]
)
dimension_size = len(dimension_index_vector)
# TODO: Do range checks to ensure indices are within storage unit boundaries
else:
dimension_size = 0 # Unlimited dimension
dimension_name = dimension_config["dimension_name"]
# Dimensions can be renamed with the 'renameDimension' method of the file
self.netcdf_object.createDimension(dimension_name, dimension_size)
variable = self.netcdf_object.createVariable(dimension_name, "f8", (dimension_name,))
for property_name, property_value in dimension_config["properties"].items():
logger.debug("property_name = %s, property_value = %s", property_name, property_value)
variable.__setattr__(property_name, property_value)
variable[:] = dimension_index_vector
def __init__(self, path):
'''Constructor for class ConfigFile
Parameters:
path: Path to valid config file (required)
'''
log_multiline(logger.debug, path, 'path', '\t')
self._path = os.path.abspath(path)
assert os.path.exists(self._path), "%s does not exist" % self._path
self._configuration = self._parse_config_file()
log_multiline(logger.debug, self.__dict__, 'ConfigFile.__dict__', '\t')
def __init__(self, path):
'''Constructor for class ConfigFile
Parameters:
path: Path to valid config file (required)
'''
log_multiline(logger.debug, path, 'path', '\t')
self._path = os.path.abspath(path)
assert os.path.exists(self._path), "%s does not exist" % self._path
self._configuration = self._parse_config_file()
log_multiline(logger.debug, self.__dict__, 'ConfigFile.__dict__', '\t')
def set_dimension(dimension, dimension_config, index, dimension_index_vector=None):
'''
Parameters:
dimension: Dimension tag (e.g. X, Y, T, etc.)
dimension_config: Nested dict containing storage configuration from GDF.storage_config['<storage_type>']
index: index for storage unit
dimension_index_vector: Numpy array of index values for irregular dimension (e.g. time) or None for unlimited irregular dimension
'''
logger.debug('dimension = %s', dimension)
logger.debug('dimension_config = %s', dimension_config)
logger.debug('index = %s', index)
logger.debug('dimension_index_vector = %s', dimension_index_vector)
if dimension_config['indexing_type'] == 'regular' and not dimension_index_vector:
element_size = dimension_config['dimension_element_size']
dimension_min = index * dimension_config['dimension_extent'] + dimension_config['dimension_origin'] + element_size / 2.0 # Half pixel to account for netCDF centre of pixel reference
dimension_max = dimension_min + dimension_config['dimension_extent']
dimension_index_vector = np.around(np.arange(dimension_min, dimension_max, element_size), GDFNetCDF.DECIMAL_PLACES)
# Cater for reversed index (e.g. positive Y index tends Southwards when image origin is in UL/NW corner)
if dimension_config['reverse_index']:
dimension_index_vector = dimension_index_vector[::-1]
#TODO: Implement fixed indexing type
log_multiline(logger.debug, dimension_index_vector, 'dimension_index_vector for %s' % dimension, '\t')
if dimension_index_vector is not None:
dimension_index_shape = dimension_index_vector.shape
assert len(dimension_index_shape) == 1, 'Invalid dimension_index_vector shape. Must be 1D'
assert dimension_index_shape[0] <= dimension_config['dimension_elements'], 'dimension_index_vector must have %d elements or fewer' % dimension_config['dimension_elements']
dimension_size = len(dimension_index_vector)
#TODO: Do range checks to ensure indices are within storage unit boundaries
else:
dimension_size = 0 # Unlimited dimension
dimension_name = dimension_config['dimension_name']
# Dimensions can be renamed with the 'renameDimension' method of the file
self.netcdf_object.createDimension(dimension_name, dimension_size)
variable = self.netcdf_object.createVariable(dimension_name,'f8',(dimension_name,))
for property_name, property_value in dimension_config['properties'].items():
logger.debug('property_name = %s, property_value = %s', property_name, property_value)
variable.__setattr__(property_name, property_value)
variable[:] = dimension_index_vector
def execSQL(self, SQL, params=None, cursor=None):
'''
Function to return cursor with query results for specified SQL and parameters
Parameters:
SQL: Query text
params: Dict containing query parameters (optional)
cursor: cursor to return results. Defaults to self._default_cursor
'''
cursor = cursor or self._default_cursor or self.create_connection().cursor()
log_multiline(logger.debug, cursor.mogrify(SQL, params), 'SQL', '\t')
cursor.execute(SQL, params)
return cursor
def submit_query(self, SQL, params=None, connection=None):
'''
Function to return CachedResultSet object to manage an in-memory cache of query results for specified SQL and parameters
Parameters:
SQL: Query text
params: Dict containing query parameters (optional)
connection: DB connection to query. Defaults to self._default_connection
'''
connection = connection or self._default_connection or self.create_connection()
log_multiline(logger.debug, SQL, 'SQL', '\t')
log_multiline(logger.debug, params, 'params', '\t')
# Use local cursor to return results
return CachedResultSet(self.execSQL(SQL, params, cursor=connection.cursor()))
def __init__(self, arg_descriptors={}):
'''Constructor for class CommandLineArgs
Parameters:
arg_descriptors: dict keyed by dest variable name containing sub-dicts as follows:
'short_flag': '-d',
'long_flag': '--debug',
'default': <Boolean>,
'action': 'store_const',
'const': <Boolean>,
'help': <help string>
'''
log_multiline(logger.debug, arg_descriptors, 'arg_descriptors', '\t')
# # Replace defaults with supplied dict
# arg_descriptors = arg_descriptors or CommandLineArgs.DEFAULT_ARG_DESCRIPTORS
# Merge defaults with supplied dict (overwriting defaults)
temp_arg_descriptors = CommandLineArgs.DEFAULT_ARG_DESCRIPTORS.copy()
temp_arg_descriptors.update(arg_descriptors)
arg_descriptors = temp_arg_descriptors
log_multiline(logger.debug, arg_descriptors, 'arg_descriptors', '\t')
self._arguments = self._parse_args(arg_descriptors)
log_multiline(logger.debug, self.__dict__, 'CommandLineArgs.__dict__', '\t')
def _parse_config_file(self):
'''
Function to return a nested dict of config file entries
Returns:
dict {<section_name>: {<key>: <value>,... },... }
'''
logger.debug('Opening config file %s', self._path)
config_parser = ConfigParser.SafeConfigParser(allow_no_value=True)
config_parser.read(self._path)
config_dict = collections.OrderedDict() # Need to preserve order of sections
for section_name in config_parser.sections():
section_dict = {}
config_dict[section_name.lower()] = section_dict
for attribute_name in config_parser.options(section_name):
attribute_value = config_parser.get(section_name, attribute_name)
section_dict[attribute_name.lower()] = attribute_value
log_multiline(logger.debug, config_dict, 'config_dict', '\t')
return config_dict
def _parse_config_file(self):
'''
Function to return a nested dict of config file entries
Returns:
dict {<section_name>: {<key>: <value>,... },... }
'''
logger.debug('Opening config file %s', self._path)
config_parser = ConfigParser.SafeConfigParser(allow_no_value=True)
config_parser.read(self._path)
config_dict = collections.OrderedDict(
) # Need to preserve order of sections
for section_name in config_parser.sections():
section_dict = {}
config_dict[section_name.lower()] = section_dict
for attribute_name in config_parser.options(section_name):
attribute_value = config_parser.get(section_name,
attribute_name)
section_dict[attribute_name.lower()] = attribute_value
log_multiline(logger.debug, config_dict, 'config_dict', '\t')
return config_dict
def _parse_args(self, arg_descriptors):
"""Virtual function to parse command line arguments.
Parameters:
arg_descriptors: dict keyed by dest variable name containing sub-dicts as follows:
'short_flag': '-d',
'long_flag': '--debug',
'default': <Boolean>,
'action': 'store_const',
'const': <Boolean>,
'help': <help string>
Returns:
argparse namespace object
"""
logger.debug('Calling _parse_args()')
log_multiline(logger.debug, arg_descriptors, 'arg_descriptors', '\t')
_arg_parser = argparse.ArgumentParser(description=os.path.basename(sys.argv[0]))
for arg_dest in sorted(arg_descriptors.keys()):
arg_descriptor = arg_descriptors[arg_dest]
log_multiline(logger.debug, arg_descriptor, 'arg_descriptor for %s' % arg_dest, '\t')
_arg_parser.add_argument(arg_descriptor['short_flag'],
arg_descriptor['long_flag'],
dest=arg_dest,
default=arg_descriptor['default'],
action=arg_descriptor['action'],
const=arg_descriptor['const'],
help=arg_descriptor['help']
)
args, _unknown_args = _arg_parser.parse_known_args()
return args.__dict__
def set_dimension(dimension,
dimension_config,
index,
dimension_index_vector=None):
'''
Parameters:
dimension: Dimension tag (e.g. X, Y, T, etc.)
dimension_config: Nested dict containing storage configuration from GDF.storage_config['<storage_type>']
index: index for storage unit
dimension_index_vector: Numpy array of index values for irregular dimension (e.g. time) or None for unlimited irregular dimension
'''
logger.debug('dimension = %s', dimension)
logger.debug('dimension_config = %s', dimension_config)
logger.debug('index = %s', index)
logger.debug('dimension_index_vector = %s', dimension_index_vector)
if dimension_config[
'indexing_type'] == 'regular' and not dimension_index_vector:
element_size = dimension_config['dimension_element_size']
dimension_min = index * dimension_config[
'dimension_extent'] + dimension_config[
'dimension_origin'] + element_size / 2.0 # Half pixel to account for netCDF centre of pixel reference
dimension_max = dimension_min + dimension_config[
'dimension_extent']
dimension_index_vector = np.around(
np.arange(dimension_min, dimension_max, element_size),
GDFNetCDF.DECIMAL_PLACES)
# Cater for reversed index (e.g. positive Y index tends Southwards when image origin is in UL/NW corner)
if dimension_config['reverse_index']:
dimension_index_vector = dimension_index_vector[::-1]
#TODO: Implement fixed indexing type
log_multiline(logger.debug, dimension_index_vector,
'dimension_index_vector for %s' % dimension, '\t')
if dimension_index_vector is not None:
dimension_index_shape = dimension_index_vector.shape
assert len(
dimension_index_shape
) == 1, 'Invalid dimension_index_vector shape. Must be 1D'
assert dimension_index_shape[0] <= dimension_config[
'dimension_elements'], 'dimension_index_vector must have %d elements or fewer' % dimension_config[
'dimension_elements']
dimension_size = len(dimension_index_vector)
#TODO: Do range checks to ensure indices are within storage unit boundaries
else:
dimension_size = 0 # Unlimited dimension
dimension_name = dimension_config['dimension_name']
# Dimensions can be renamed with the 'renameDimension' method of the file
self.netcdf_object.createDimension(dimension_name, dimension_size)
variable = self.netcdf_object.createVariable(
dimension_name, 'f8', (dimension_name, ))
for property_name, property_value in dimension_config[
'properties'].items():
logger.debug('property_name = %s, property_value = %s',
property_name, property_value)
variable.__setattr__(property_name, property_value)
variable[:] = dimension_index_vector
