def db(pg_conn_main, pg_conn_test, mocker): dsn_params = pg_conn_main.info.dsn_parameters mocker.patch("dodoo.configs.db.DbConfig.resolve_dsn", lambda *args: make_dsn(**dsn_params)) # also avoid @ensure_framework decorator: mocker.patch("dodoo.utils.db._dsn_resolver", lambda _: make_dsn(**dsn_params)) yield pg_conn_test.info.dbname
def test_url_is_cool(self): url = 'postgresql://*****:*****@/test?application_name=wat' dsn = ext.make_dsn(url) self.assertEqual(dsn, url) dsn = ext.make_dsn(url, application_name='woot') self.assertDsnEqual(dsn, 'dbname=test user=tester password=secret application_name=woot') self.assertRaises(psycopg2.ProgrammingError, ext.make_dsn, 'postgresql://*****:*****@/test?nosuch=param') self.assertRaises(psycopg2.ProgrammingError, ext.make_dsn, url, nosuch="param")
def test_escape(self): dsn = ext.make_dsn(dbname='hello world') self.assertEqual(dsn, "dbname='hello world'") dsn = ext.make_dsn(dbname=r'back\slash') self.assertEqual(dsn, r"dbname=back\\slash") dsn = ext.make_dsn(dbname="quo'te") self.assertEqual(dsn, r"dbname=quo\'te") dsn = ext.make_dsn(dbname="with\ttab") self.assertEqual(dsn, "dbname='with\ttab'") dsn = ext.make_dsn(dbname=r"\every thing'") self.assertEqual(dsn, r"dbname='\\every thing\''")
def wrapper(dsn, connection_factory=None, *args, **kwargs): config = parse_dsn(dsn) host = config.get("host") if host is None: host = kwargs.get("host") if host is None: # Host may be left out to use localhost or # possibly set using environment variables, nothing # we can do in either case. logger.error( "'host' parameter is not present in call to psycopg2.connect, " "DNS resolution might not work properly" ) return fn(dsn, connection_factory, *args, **kwargs) if re.search(host_regex, host): logger.debug("Host %s matched SRV regex, resolving", host) host, port = resolve_srv_record(host, srv_resolver) config["host"] = host config["port"] = port dsn = make_dsn(**config) return fn(dsn, connection_factory, *args, **kwargs)
def connect(self, database=None, user=None, password=None, host=None, port=None, dsn=None, **kwargs): conn_params = self._conn_params.copy() new_params = { 'database': database, 'user': user, 'password': password, 'host': host, 'port': port, 'dsn': dsn, } new_params.update(kwargs) conn_params.update({ k: unicode2utf8(v) for k, v in new_params.items() if v is not None }) if 'password' in conn_params and 'dsn' in conn_params: conn_params['dsn'] = make_dsn( conn_params['dsn'], password=conn_params.pop('password')) conn = psycopg2.connect(**conn_params) cursor = conn.cursor() conn.set_client_encoding('utf8') self._conn_params = conn_params if self.conn: self.conn.close() self.conn = conn self.conn.autocommit = True # When we connect using a DSN, we don't really know what db, # user, etc. we connected to. Let's read it. # Note: moved this after setting autocommit because of #664. # TODO: use actual connection info from psycopg2.extensions.Connection.info as psycopg>2.8 is available and required dependency # noqa dsn_parameters = conn.get_dsn_parameters() self.dbname = dsn_parameters['dbname'] self.user = dsn_parameters['user'] self.password = password self.host = dsn_parameters['host'] self.port = dsn_parameters['port'] self.extra_args = kwargs if not self.host: self.host = self.get_socket_directory() cursor.execute("SHOW ALL") db_parameters = dict(name_val_desc[:2] for name_val_desc in cursor.fetchall()) pid = self._select_one(cursor, 'select pg_backend_pid()')[0] self.pid = pid self.superuser = db_parameters.get('is_superuser') == '1' self.server_version = self.get_server_version(cursor) register_date_typecasters(conn) register_json_typecasters(self.conn, self._json_typecaster) register_hstore_typecaster(self.conn)
def create_database_for_package( modname: str, conninfo: str, template: str = "template1" ): """Create the database pointed at with ``conninfo``, and initialize it using -superuser- SQL files found in the package ``modname``. Args: modname: Name of the module of which we're loading the files conninfo: connection info string or plain database name for the SQL database template: the name of the database to connect to and use as template to create the new database """ # Use the given conninfo string, but with dbname replaced by the template dbname # for the database creation step creation_dsn = parse_dsn_or_dbname(conninfo) dbname = creation_dsn["dbname"] creation_dsn["dbname"] = template logger.debug("db_create dbname=%s (from %s)", dbname, template) subprocess.check_call( [ "psql", "--quiet", "--no-psqlrc", "-v", "ON_ERROR_STOP=1", "-d", make_dsn(**creation_dsn), "-c", f'CREATE DATABASE "{dbname}"', ] ) init_admin_extensions(modname, conninfo)
def _create_connection(config): conn_id = 'pgmigrate-{id}'.format(id=str(uuid.uuid4())) conn = _create_raw_connection( make_dsn(config.conn, application_name=conn_id)) if config.terminator_instance: config.terminator_instance.add_conn(conn) return conn
def connect(dsn=None, connection_factory=None, cursor_factory=None, **kwargs): """ Create a new database connection. The connection parameters can be specified as a string: conn = psycopg2.connect("dbname=test user=postgres password=secret") or using a set of keyword arguments: conn = psycopg2.connect(database="test", user="******", password="******") Or as a mix of both. The basic connection parameters are: - *dbname*: the database name - *database*: the database name (only as keyword argument) - *user*: user name used to authenticate - *password*: password used to authenticate - *host*: database host address (defaults to UNIX socket if not provided) - *port*: connection port number (defaults to 5432 if not provided) Using the *connection_factory* parameter a different class or connections factory can be specified. It should be a callable object taking a dsn argument. Using the *cursor_factory* parameter, a new default cursor factory will be used by cursor(). Using *async*=True an asynchronous connection will be created. *async_* is a valid alias (for Python versions where ``async`` is a keyword). Any other keyword parameter will be passed to the underlying client library: the list of supported parameters depends on the library version. """ kwasync = {} if 'async' in kwargs: kwasync['async'] = kwargs.pop('async') if 'async_' in kwargs: kwasync['async_'] = kwargs.pop('async_') if dsn is None and not kwargs: raise TypeError('missing dsn and no parameters') dsn = _ext.make_dsn(dsn, **kwargs) print dsn #exit(0) conn = _connect(dsn, connection_factory=connection_factory, **kwasync) if cursor_factory is not None: conn.cursor_factory = cursor_factory return conn
def __init__(self): self.pg_dsn = make_dsn( host=os.environ["SYNAPSE_DB_HOST"], # port is optional and make_dsn ignores None values, so we use .get() port=os.environ.get("SYNAPSE_DB_PORT"), user=os.environ["SYNAPSE_DB_USERNAME"], password=os.environ["SYNAPSE_DB_PASSWORD"], dbname=os.environ["SYNAPSE_DB_DATABASE"], options=os.environ["SYNAPSE_DB_OPTIONS"], ) self.STATS_DB_HOST = os.environ["STATS_DB_HOST"] self.STATS_DB_USERNAME = os.environ["STATS_DB_USERNAME"] self.STATS_DB_PASSWORD = os.environ["STATS_DB_PASSWORD"] self.STATS_DB_DATABASE = os.environ["STATS_DB_DATABASE"]
def connect(dsn=None, connection_factory=None, cursor_factory=None, **kwargs): """ Create a new database connection. The connection parameters can be specified as a string: conn = psycopg2.connect("dbname=test user=postgres password=secret") or using a set of keyword arguments: conn = psycopg2.connect(database="test", user="******", password="******") Or as a mix of both. The basic connection parameters are: - *dbname*: the database name - *database*: the database name (only as keyword argument) - *user*: user name used to authenticate - *password*: password used to authenticate - *host*: database host address (defaults to UNIX socket if not provided) - *port*: connection port number (defaults to 5432 if not provided) Using the *connection_factory* parameter a different class or connections factory can be specified. It should be a callable object taking a dsn argument. Using the *cursor_factory* parameter, a new default cursor factory will be used by cursor(). Using *async*=True an asynchronous connection will be created. *async_* is a valid alias (for Python versions where ``async`` is a keyword). Any other keyword parameter will be passed to the underlying client library: the list of supported parameters depends on the library version. """ kwasync = {} if 'async' in kwargs: kwasync['async'] = kwargs.pop('async') if 'async_' in kwargs: kwasync['async_'] = kwargs.pop('async_') if dsn is None and not kwargs: raise TypeError('missing dsn and no parameters') dsn = _ext.make_dsn(dsn, **kwargs) conn = _connect(dsn, connection_factory=connection_factory, **kwasync) if cursor_factory is not None: conn.cursor_factory = cursor_factory return conn
def init(): global pool try: pool = ThreadSafeConnectionPool(minconn=0, maxconn=5000, idle_timeout=300, dsn=make_dsn( host = config.database_host, dbname = config.database_dbname, user = config.database_user, password = config.database_password, port = 5432 ) ) except Exception as e: print(f'Failed to connect to the database: {e}') return pool
def connect(self, dbname=None, user=None, password=None, host=None, port=None, attempts=ATTEMPTS): """TODO.""" connection_kwargs = { 'connect_timeout': self.TIMEOUT, 'dbname': dbname, 'host': host, 'password': password, 'port': port, 'user': user } connectionString = make_dsn(**connection_kwargs) try: self.logInfo('Connecting to ' + self.getConnectionString(**connection_kwargs) + ' (' + connectionString + ')...') with connect(connectionString, BetterLoggingConnection) as connection: if hasattr(self, 'connection'): self.connection.close() self.connection = connection self.logInfo('...connected; configuring connection...') connection.initialize(self.logger) connection.notices = deque(connection.notices) connection.set_client_encoding(self.ENCODING) self.logInfo('...connection configured...') return self except OperationalError as error: self.logDebug(str(error)) self.logDebug(str(error.pgcode)) self.logError('...' + str(error.pgerror).strip() + '...') self.logError('...couldn\'t connect.') self.logDebug(connection_kwargs) raise ExitException()
def parse_value(value, proto): if '://' in value: proto, value = value.split('://', 1) display_value = '{}://{}'.format(proto, value) if proto == 'tcp': if ':' not in value: raise argparse.ArgumentTypeError( 'Invalid service spec {!r}. Must have ":". Where\'s the port?'. format(display_value)) host, port = value.strip('/').split(':', 1) if not port.isdigit(): raise argparse.ArgumentTypeError( 'Invalid service spec {!r}. Port must be a number not {!r}.'. format(display_value, port)) port = int(port) return TcpCheck(host, port) elif proto in ('pg', 'postgresql', 'postgres'): if psycopg2 is None: raise argparse.ArgumentTypeError( 'Protocol {} unusable. Install holdup[pg].'.format(proto)) uri = 'postgresql://{}'.format(value) try: connection_uri = make_dsn(uri) except Exception as exc: raise argparse.ArgumentTypeError( 'Failed to parse %r: %s. Must be a valid connection URI.' % (display_value, exc)) return PgCheck(connection_uri) elif proto == 'unix': return UnixCheck(value) elif proto == 'path': return PathCheck(value) elif proto in ('http', 'https', 'https+insecure'): return HttpCheck('%s://%s' % (proto, value)) elif proto == 'eval': return EvalCheck(value) else: raise argparse.ArgumentTypeError( 'Unknown protocol {!r} in {!r}. Must be "tcp", "path", "unix" or "pg".' .format(proto, display_value))
def parse_value(value, proto): if '://' in value: proto, value = value.split('://', 1) if proto == 'tcp': if ':' not in value: raise argparse.ArgumentTypeError( 'Invalid service spec %r. Must have ":". Where\'s the port?' % value) host, port = value.strip('/').split(':', 1) if not port.isdigit(): raise argparse.ArgumentTypeError( 'Invalid service spec %r. Port must be a number.' % value) port = int(port) return TcpCheck(host, port) elif proto in ('pg', 'postgresql', 'postgres'): if not psycopg2_imported: raise ImportError( "You need to install psycopg2 in order to use postgresql protocol" ) try: connection_uri = make_dsn('postgresql://{}'.format(value)) except Exception as exc: raise ArgumentTypeError('Failed to parse %s : %s. Expected format is'\ '"postgresql://*****:*****@host:port/dbname"' % (value, exc)) return PgCheck(connection_uri) elif proto == 'unix': return UnixCheck(value) elif proto == 'path': return PathCheck(value) elif proto in ('http', 'https', 'https+insecure'): return HttpCheck('%s://%s' % (proto, value)) elif proto == 'eval': return EvalCheck(value) else: raise argparse.ArgumentTypeError( 'Unknown protocol %r in %r. Must be "tcp", "path" or "unix".' % (proto, value))
def test_no_dsn_munging(self): dsnin = 'dbname=a host=b user=c password=d' dsn = ext.make_dsn(dsnin) self.assertEqual(dsn, dsnin)
def test_params_merging(self): dsn = ext.make_dsn('dbname=foo host=bar', host='baz') self.assertDsnEqual(dsn, 'dbname=foo host=baz') dsn = ext.make_dsn('dbname=foo', user='******') self.assertDsnEqual(dsn, 'dbname=foo user=postgres')
def test_database_is_a_keyword(self): self.assertEqual(ext.make_dsn(database='sigh'), "dbname=sigh")
- *dbname*: the database name - *database*: the database name (only as keyword argument) - *user*: user name used to authenticate - *password*: password used to authenticate - *host*: database host address (defaults to UNIX socket if not provided) - *port*: connection port number (defaults to 5432 if not provided) Using the *connection_factory* parameter a different class or connections factory can be specified. It should be a callable object taking a dsn argument. Using the *cursor_factory* parameter, a new default cursor factory will be used by cursor(). Using *async*=True an asynchronous connection will be created. Any other keyword parameter will be passed to the underlying client library: the list of supported parameters depends on the library version. """ if dsn is None and not kwargs: raise TypeError('missing dsn and no parameters') dsn = _ext.make_dsn(dsn, **kwargs) conn = _connect(dsn, connection_factory=connection_factory, async=async) if cursor_factory is not None: conn.cursor_factory = cursor_factory return conn
def test_empty_param(self): dsn = ext.make_dsn(dbname='sony', password='') self.assertDsnEqual(dsn, "dbname=sony password=''")
def test_empty_string(self): dsn = ext.make_dsn('') self.assertEqual(dsn, '')
def test_empty_arguments(self): self.assertEqual(ext.make_dsn(), '')
def connection_info_for(self, dbname): # Already reloaded by _patch_odoo_db_connect # reloaded = self.DbConfig.reload() dsn = self.DbConfig.resolve_dsn(dbname) return dbname, parse_dsn(make_dsn(dsn, dbname=dbname, application_name="odoo"))
def _ensure_dsn_with_passfile(dsn): dsn_dict = parse_dsn( dsn) # Validates here as (desireable) side effect if not dsn_dict.get("passfile"): dsn_dict["passfile"] = passfile return make_dsn(**dsn_dict)
def connect(self, database=None, user=None, password=None, host=None, port=None, dsn=None, **kwargs): conn_params = self._conn_params.copy() new_params = { "database": database, "user": user, "password": password, "host": host, "port": port, "dsn": dsn, } new_params.update(kwargs) if new_params["dsn"]: new_params = { "dsn": new_params["dsn"], "password": new_params["password"] } if new_params["password"]: new_params["dsn"] = make_dsn( new_params["dsn"], password=new_params.pop("password")) conn_params.update( {k: unicode2utf8(v) for k, v in new_params.items() if v}) conn = psycopg2.connect(**conn_params) cursor = conn.cursor() conn.set_client_encoding("utf8") self._conn_params = conn_params if self.conn: self.conn.close() self.conn = conn self.conn.autocommit = True # When we connect using a DSN, we don't really know what db, # user, etc. we connected to. Let's read it. # Note: moved this after setting autocommit because of #664. # TODO: use actual connection info from psycopg2.extensions.Connection.info as psycopg>2.8 is available and required dependency # noqa dsn_parameters = conn.get_dsn_parameters() self.dbname = dsn_parameters.get("dbname") self.user = dsn_parameters.get("user") self.password = password self.host = dsn_parameters.get("host") self.port = dsn_parameters.get("port") self.extra_args = kwargs if not self.host: self.host = self.get_socket_directory() cursor.execute("SHOW ALL") db_parameters = dict(name_val_desc[:2] for name_val_desc in cursor.fetchall()) pid = self._select_one(cursor, "select pg_backend_pid()")[0] self.pid = pid self.superuser = db_parameters.get("is_superuser") == "1" self.server_version = self.get_server_version(cursor) register_date_typecasters(conn) register_json_typecasters(self.conn, self._json_typecaster) register_hstore_typecaster(self.conn)
def connect( self, database=None, user=None, password=None, host=None, port=None, dsn=None, **kwargs, ): conn_params = self._conn_params.copy() new_params = { "database": database, "user": user, "password": password, "host": host, "port": port, "dsn": dsn, } new_params.update(kwargs) if new_params["dsn"]: new_params = { "dsn": new_params["dsn"], "password": new_params["password"] } if new_params["password"]: new_params["dsn"] = make_dsn( new_params["dsn"], password=new_params.pop("password")) conn_params.update({k: v for k, v in new_params.items() if v}) conn = psycopg2.connect(**conn_params) cursor = conn.cursor() conn.set_client_encoding("utf8") self._conn_params = conn_params if self.conn: self.conn.close() self.conn = conn self.conn.autocommit = True # When we connect using a DSN, we don't really know what db, # user, etc. we connected to. Let's read it. # Note: moved this after setting autocommit because of #664. libpq_version = psycopg2.__libpq_version__ dsn_parameters = {} if libpq_version >= 93000: # use actual connection info from psycopg2.extensions.Connection.info # as libpq_version > 9.3 is available and required dependency dsn_parameters = conn.info.dsn_parameters else: try: dsn_parameters = conn.get_dsn_parameters() except Exception as x: # https://github.com/dbcli/pgcli/issues/1110 # PQconninfo not available in libpq < 9.3 _logger.info("Exception in get_dsn_parameters: %r", x) if dsn_parameters: self.dbname = dsn_parameters.get("dbname") self.user = dsn_parameters.get("user") self.host = dsn_parameters.get("host") self.port = dsn_parameters.get("port") else: self.dbname = conn_params.get("database") self.user = conn_params.get("user") self.host = conn_params.get("host") self.port = conn_params.get("port") self.password = password self.extra_args = kwargs if not self.host: self.host = self.get_socket_directory() pid = self._select_one(cursor, "select pg_backend_pid()")[0] self.pid = pid self.superuser = conn.get_parameter_status("is_superuser") in ("on", "1") self.server_version = conn.get_parameter_status("server_version") register_date_typecasters(conn) register_json_typecasters(self.conn, self._json_typecaster) register_hstore_typecaster(self.conn)
def db(pg_conn_main, pg_conn_test, mocker): dsn_params = pg_conn_main.info.dsn_parameters mocker.patch("dodoo.utils.db._dsn_resolver", lambda _: make_dsn(**dsn_params)) yield pg_conn_test.info.dbname