def __init__(self, db_ref, host, port, dbname, user, password, keep_connection=True, autocommit=True): ''' Constructor for class Database. Parameters: host: PostgreSQL database host port: PostgreSQL database port dbname: PostgreSQL database database name user: PostgreSQL database user password: PostgreSQL database password for user ''' self._db_ref = db_ref self._host = host self._port = port self._dbname = dbname self._user = user self._password = password self._keep_connection = keep_connection self._autocommit = autocommit self._default_connection = None self._default_cursor = None self._setup_default_cursor() log_multiline(logger.debug, self.__dict__, 'Database.__dict__', '\t')
def set_dimension(dimension, dimension_config, index, dimension_index_vector=None): """ Parameters: dimension: Dimension tag (e.g. X, Y, T, etc.) dimension_config: Nested dict containing storage configuration from GDF.storage_config['<storage_type>'] index: index for storage unit dimension_index_vector: Numpy array of index values for irregular dimension (e.g. time) or None for unlimited irregular dimension """ logger.debug("dimension = %s", dimension) logger.debug("dimension_config = %s", dimension_config) logger.debug("index = %s", index) logger.debug("dimension_index_vector = %s", dimension_index_vector) if dimension_config["indexing_type"] == "regular" and not dimension_index_vector: element_size = dimension_config["dimension_element_size"] dimension_min = ( index * dimension_config["dimension_extent"] + dimension_config["dimension_origin"] + element_size / 2.0 ) # Half pixel to account for netCDF centre of pixel reference dimension_max = dimension_min + dimension_config["dimension_extent"] dimension_index_vector = np.around( np.arange(dimension_min, dimension_max, element_size), self.decimal_places ) # Cater for reversed index (e.g. positive Y index tends Southwards when image origin is in UL/NW corner) if dimension_config["reverse_index"]: dimension_index_vector = dimension_index_vector[::-1] # TODO: Implement fixed indexing type log_multiline(logger.debug, dimension_index_vector, "dimension_index_vector for %s" % dimension, "\t") if dimension_index_vector is not None: dimension_index_shape = dimension_index_vector.shape assert len(dimension_index_shape) == 1, "Invalid dimension_index_vector shape. Must be 1D" assert dimension_index_shape[0] <= dimension_config["dimension_elements"], ( "dimension_index_vector must have %d elements or fewer" % dimension_config["dimension_elements"] ) dimension_size = len(dimension_index_vector) # TODO: Do range checks to ensure indices are within storage unit boundaries else: dimension_size = 0 # Unlimited dimension dimension_name = dimension_config["dimension_name"] # Dimensions can be renamed with the 'renameDimension' method of the file self.netcdf_object.createDimension(dimension_name, dimension_size) variable = self.netcdf_object.createVariable(dimension_name, "f8", (dimension_name,)) for property_name, property_value in dimension_config["properties"].items(): logger.debug("property_name = %s, property_value = %s", property_name, property_value) variable.__setattr__(property_name, property_value) variable[:] = dimension_index_vector
def __init__(self, path): '''Constructor for class ConfigFile Parameters: path: Path to valid config file (required) ''' log_multiline(logger.debug, path, 'path', '\t') self._path = os.path.abspath(path) assert os.path.exists(self._path), "%s does not exist" % self._path self._configuration = self._parse_config_file() log_multiline(logger.debug, self.__dict__, 'ConfigFile.__dict__', '\t')
def __init__(self, path): '''Constructor for class ConfigFile Parameters: path: Path to valid config file (required) ''' log_multiline(logger.debug, path, 'path', '\t') self._path = os.path.abspath(path) assert os.path.exists(self._path), "%s does not exist" % self._path self._configuration = self._parse_config_file() log_multiline(logger.debug, self.__dict__, 'ConfigFile.__dict__', '\t')
def set_dimension(dimension, dimension_config, index, dimension_index_vector=None): ''' Parameters: dimension: Dimension tag (e.g. X, Y, T, etc.) dimension_config: Nested dict containing storage configuration from GDF.storage_config['<storage_type>'] index: index for storage unit dimension_index_vector: Numpy array of index values for irregular dimension (e.g. time) or None for unlimited irregular dimension ''' logger.debug('dimension = %s', dimension) logger.debug('dimension_config = %s', dimension_config) logger.debug('index = %s', index) logger.debug('dimension_index_vector = %s', dimension_index_vector) if dimension_config['indexing_type'] == 'regular' and not dimension_index_vector: element_size = dimension_config['dimension_element_size'] dimension_min = index * dimension_config['dimension_extent'] + dimension_config['dimension_origin'] + element_size / 2.0 # Half pixel to account for netCDF centre of pixel reference dimension_max = dimension_min + dimension_config['dimension_extent'] dimension_index_vector = np.around(np.arange(dimension_min, dimension_max, element_size), GDFNetCDF.DECIMAL_PLACES) # Cater for reversed index (e.g. positive Y index tends Southwards when image origin is in UL/NW corner) if dimension_config['reverse_index']: dimension_index_vector = dimension_index_vector[::-1] #TODO: Implement fixed indexing type log_multiline(logger.debug, dimension_index_vector, 'dimension_index_vector for %s' % dimension, '\t') if dimension_index_vector is not None: dimension_index_shape = dimension_index_vector.shape assert len(dimension_index_shape) == 1, 'Invalid dimension_index_vector shape. Must be 1D' assert dimension_index_shape[0] <= dimension_config['dimension_elements'], 'dimension_index_vector must have %d elements or fewer' % dimension_config['dimension_elements'] dimension_size = len(dimension_index_vector) #TODO: Do range checks to ensure indices are within storage unit boundaries else: dimension_size = 0 # Unlimited dimension dimension_name = dimension_config['dimension_name'] # Dimensions can be renamed with the 'renameDimension' method of the file self.netcdf_object.createDimension(dimension_name, dimension_size) variable = self.netcdf_object.createVariable(dimension_name,'f8',(dimension_name,)) for property_name, property_value in dimension_config['properties'].items(): logger.debug('property_name = %s, property_value = %s', property_name, property_value) variable.__setattr__(property_name, property_value) variable[:] = dimension_index_vector
def execSQL(self, SQL, params=None, cursor=None): ''' Function to return cursor with query results for specified SQL and parameters Parameters: SQL: Query text params: Dict containing query parameters (optional) cursor: cursor to return results. Defaults to self._default_cursor ''' cursor = cursor or self._default_cursor or self.create_connection().cursor() log_multiline(logger.debug, cursor.mogrify(SQL, params), 'SQL', '\t') cursor.execute(SQL, params) return cursor
def submit_query(self, SQL, params=None, connection=None): ''' Function to return CachedResultSet object to manage an in-memory cache of query results for specified SQL and parameters Parameters: SQL: Query text params: Dict containing query parameters (optional) connection: DB connection to query. Defaults to self._default_connection ''' connection = connection or self._default_connection or self.create_connection() log_multiline(logger.debug, SQL, 'SQL', '\t') log_multiline(logger.debug, params, 'params', '\t') # Use local cursor to return results return CachedResultSet(self.execSQL(SQL, params, cursor=connection.cursor()))
def __init__(self, arg_descriptors={}): '''Constructor for class CommandLineArgs Parameters: arg_descriptors: dict keyed by dest variable name containing sub-dicts as follows: 'short_flag': '-d', 'long_flag': '--debug', 'default': <Boolean>, 'action': 'store_const', 'const': <Boolean>, 'help': <help string> ''' log_multiline(logger.debug, arg_descriptors, 'arg_descriptors', '\t') # # Replace defaults with supplied dict # arg_descriptors = arg_descriptors or CommandLineArgs.DEFAULT_ARG_DESCRIPTORS # Merge defaults with supplied dict (overwriting defaults) temp_arg_descriptors = CommandLineArgs.DEFAULT_ARG_DESCRIPTORS.copy() temp_arg_descriptors.update(arg_descriptors) arg_descriptors = temp_arg_descriptors log_multiline(logger.debug, arg_descriptors, 'arg_descriptors', '\t') self._arguments = self._parse_args(arg_descriptors) log_multiline(logger.debug, self.__dict__, 'CommandLineArgs.__dict__', '\t')
def _parse_config_file(self): ''' Function to return a nested dict of config file entries Returns: dict {<section_name>: {<key>: <value>,... },... } ''' logger.debug('Opening config file %s', self._path) config_parser = ConfigParser.SafeConfigParser(allow_no_value=True) config_parser.read(self._path) config_dict = collections.OrderedDict() # Need to preserve order of sections for section_name in config_parser.sections(): section_dict = {} config_dict[section_name.lower()] = section_dict for attribute_name in config_parser.options(section_name): attribute_value = config_parser.get(section_name, attribute_name) section_dict[attribute_name.lower()] = attribute_value log_multiline(logger.debug, config_dict, 'config_dict', '\t') return config_dict
def _parse_config_file(self): ''' Function to return a nested dict of config file entries Returns: dict {<section_name>: {<key>: <value>,... },... } ''' logger.debug('Opening config file %s', self._path) config_parser = ConfigParser.SafeConfigParser(allow_no_value=True) config_parser.read(self._path) config_dict = collections.OrderedDict( ) # Need to preserve order of sections for section_name in config_parser.sections(): section_dict = {} config_dict[section_name.lower()] = section_dict for attribute_name in config_parser.options(section_name): attribute_value = config_parser.get(section_name, attribute_name) section_dict[attribute_name.lower()] = attribute_value log_multiline(logger.debug, config_dict, 'config_dict', '\t') return config_dict
def _parse_args(self, arg_descriptors): """Virtual function to parse command line arguments. Parameters: arg_descriptors: dict keyed by dest variable name containing sub-dicts as follows: 'short_flag': '-d', 'long_flag': '--debug', 'default': <Boolean>, 'action': 'store_const', 'const': <Boolean>, 'help': <help string> Returns: argparse namespace object """ logger.debug('Calling _parse_args()') log_multiline(logger.debug, arg_descriptors, 'arg_descriptors', '\t') _arg_parser = argparse.ArgumentParser(description=os.path.basename(sys.argv[0])) for arg_dest in sorted(arg_descriptors.keys()): arg_descriptor = arg_descriptors[arg_dest] log_multiline(logger.debug, arg_descriptor, 'arg_descriptor for %s' % arg_dest, '\t') _arg_parser.add_argument(arg_descriptor['short_flag'], arg_descriptor['long_flag'], dest=arg_dest, default=arg_descriptor['default'], action=arg_descriptor['action'], const=arg_descriptor['const'], help=arg_descriptor['help'] ) args, _unknown_args = _arg_parser.parse_known_args() return args.__dict__
def set_dimension(dimension, dimension_config, index, dimension_index_vector=None): ''' Parameters: dimension: Dimension tag (e.g. X, Y, T, etc.) dimension_config: Nested dict containing storage configuration from GDF.storage_config['<storage_type>'] index: index for storage unit dimension_index_vector: Numpy array of index values for irregular dimension (e.g. time) or None for unlimited irregular dimension ''' logger.debug('dimension = %s', dimension) logger.debug('dimension_config = %s', dimension_config) logger.debug('index = %s', index) logger.debug('dimension_index_vector = %s', dimension_index_vector) if dimension_config[ 'indexing_type'] == 'regular' and not dimension_index_vector: element_size = dimension_config['dimension_element_size'] dimension_min = index * dimension_config[ 'dimension_extent'] + dimension_config[ 'dimension_origin'] + element_size / 2.0 # Half pixel to account for netCDF centre of pixel reference dimension_max = dimension_min + dimension_config[ 'dimension_extent'] dimension_index_vector = np.around( np.arange(dimension_min, dimension_max, element_size), GDFNetCDF.DECIMAL_PLACES) # Cater for reversed index (e.g. positive Y index tends Southwards when image origin is in UL/NW corner) if dimension_config['reverse_index']: dimension_index_vector = dimension_index_vector[::-1] #TODO: Implement fixed indexing type log_multiline(logger.debug, dimension_index_vector, 'dimension_index_vector for %s' % dimension, '\t') if dimension_index_vector is not None: dimension_index_shape = dimension_index_vector.shape assert len( dimension_index_shape ) == 1, 'Invalid dimension_index_vector shape. Must be 1D' assert dimension_index_shape[0] <= dimension_config[ 'dimension_elements'], 'dimension_index_vector must have %d elements or fewer' % dimension_config[ 'dimension_elements'] dimension_size = len(dimension_index_vector) #TODO: Do range checks to ensure indices are within storage unit boundaries else: dimension_size = 0 # Unlimited dimension dimension_name = dimension_config['dimension_name'] # Dimensions can be renamed with the 'renameDimension' method of the file self.netcdf_object.createDimension(dimension_name, dimension_size) variable = self.netcdf_object.createVariable( dimension_name, 'f8', (dimension_name, )) for property_name, property_value in dimension_config[ 'properties'].items(): logger.debug('property_name = %s, property_value = %s', property_name, property_value) variable.__setattr__(property_name, property_value) variable[:] = dimension_index_vector