def db(pg_conn_main, pg_conn_test, mocker):
dsn_params = pg_conn_main.info.dsn_parameters
mocker.patch("dodoo.configs.db.DbConfig.resolve_dsn",
lambda *args: make_dsn(**dsn_params))
# also avoid @ensure_framework decorator:
mocker.patch("dodoo.utils.db._dsn_resolver",
lambda _: make_dsn(**dsn_params))
yield pg_conn_test.info.dbname
def test_url_is_cool(self):
url = 'postgresql://*****:*****@/test?application_name=wat'
dsn = ext.make_dsn(url)
self.assertEqual(dsn, url)
dsn = ext.make_dsn(url, application_name='woot')
self.assertDsnEqual(dsn,
'dbname=test user=tester password=secret application_name=woot')
self.assertRaises(psycopg2.ProgrammingError,
ext.make_dsn, 'postgresql://*****:*****@/test?nosuch=param')
self.assertRaises(psycopg2.ProgrammingError,
ext.make_dsn, url, nosuch="param")
def test_escape(self):
dsn = ext.make_dsn(dbname='hello world')
self.assertEqual(dsn, "dbname='hello world'")
dsn = ext.make_dsn(dbname=r'back\slash')
self.assertEqual(dsn, r"dbname=back\\slash")
dsn = ext.make_dsn(dbname="quo'te")
self.assertEqual(dsn, r"dbname=quo\'te")
dsn = ext.make_dsn(dbname="with\ttab")
self.assertEqual(dsn, "dbname='with\ttab'")
dsn = ext.make_dsn(dbname=r"\every thing'")
self.assertEqual(dsn, r"dbname='\\every thing\''")
def wrapper(dsn, connection_factory=None, *args, **kwargs):
config = parse_dsn(dsn)
host = config.get("host")
if host is None:
host = kwargs.get("host")
if host is None:
# Host may be left out to use localhost or
# possibly set using environment variables, nothing
# we can do in either case.
logger.error(
"'host' parameter is not present in call to psycopg2.connect, "
"DNS resolution might not work properly"
)
return fn(dsn, connection_factory, *args, **kwargs)
if re.search(host_regex, host):
logger.debug("Host %s matched SRV regex, resolving", host)
host, port = resolve_srv_record(host, srv_resolver)
config["host"] = host
config["port"] = port
dsn = make_dsn(**config)
return fn(dsn, connection_factory, *args, **kwargs)
def connect(self, database=None, user=None, password=None, host=None,
port=None, dsn=None, **kwargs):
conn_params = self._conn_params.copy()
new_params = {
'database': database,
'user': user,
'password': password,
'host': host,
'port': port,
'dsn': dsn,
}
new_params.update(kwargs)
conn_params.update({
k: unicode2utf8(v) for k, v in new_params.items() if v is not None
})
if 'password' in conn_params and 'dsn' in conn_params:
conn_params['dsn'] = make_dsn(
conn_params['dsn'], password=conn_params.pop('password'))
conn = psycopg2.connect(**conn_params)
cursor = conn.cursor()
conn.set_client_encoding('utf8')
self._conn_params = conn_params
if self.conn:
self.conn.close()
self.conn = conn
self.conn.autocommit = True
# When we connect using a DSN, we don't really know what db,
# user, etc. we connected to. Let's read it.
# Note: moved this after setting autocommit because of #664.
# TODO: use actual connection info from psycopg2.extensions.Connection.info as psycopg>2.8 is available and required dependency # noqa
dsn_parameters = conn.get_dsn_parameters()
self.dbname = dsn_parameters['dbname']
self.user = dsn_parameters['user']
self.password = password
self.host = dsn_parameters['host']
self.port = dsn_parameters['port']
self.extra_args = kwargs
if not self.host:
self.host = self.get_socket_directory()
cursor.execute("SHOW ALL")
db_parameters = dict(name_val_desc[:2] for name_val_desc in cursor.fetchall())
pid = self._select_one(cursor, 'select pg_backend_pid()')[0]
self.pid = pid
self.superuser = db_parameters.get('is_superuser') == '1'
self.server_version = self.get_server_version(cursor)
register_date_typecasters(conn)
register_json_typecasters(self.conn, self._json_typecaster)
register_hstore_typecaster(self.conn)
def create_database_for_package(
modname: str, conninfo: str, template: str = "template1"
):
"""Create the database pointed at with ``conninfo``, and initialize it using
-superuser- SQL files found in the package ``modname``.
Args:
modname: Name of the module of which we're loading the files
conninfo: connection info string or plain database name for the SQL database
template: the name of the database to connect to and use as template to create
the new database
"""
# Use the given conninfo string, but with dbname replaced by the template dbname
# for the database creation step
creation_dsn = parse_dsn_or_dbname(conninfo)
dbname = creation_dsn["dbname"]
creation_dsn["dbname"] = template
logger.debug("db_create dbname=%s (from %s)", dbname, template)
subprocess.check_call(
[
"psql",
"--quiet",
"--no-psqlrc",
"-v",
"ON_ERROR_STOP=1",
"-d",
make_dsn(**creation_dsn),
"-c",
f'CREATE DATABASE "{dbname}"',
]
)
init_admin_extensions(modname, conninfo)
def _create_connection(config):
conn_id = 'pgmigrate-{id}'.format(id=str(uuid.uuid4()))
conn = _create_raw_connection(
make_dsn(config.conn, application_name=conn_id))
if config.terminator_instance:
config.terminator_instance.add_conn(conn)
return conn
def connect(dsn=None, connection_factory=None, cursor_factory=None, **kwargs):
"""
Create a new database connection.
The connection parameters can be specified as a string:
conn = psycopg2.connect("dbname=test user=postgres password=secret")
or using a set of keyword arguments:
conn = psycopg2.connect(database="test", user="******", password="******")
Or as a mix of both. The basic connection parameters are:
- *dbname*: the database name
- *database*: the database name (only as keyword argument)
- *user*: user name used to authenticate
- *password*: password used to authenticate
- *host*: database host address (defaults to UNIX socket if not provided)
- *port*: connection port number (defaults to 5432 if not provided)
Using the *connection_factory* parameter a different class or connections
factory can be specified. It should be a callable object taking a dsn
argument.
Using the *cursor_factory* parameter, a new default cursor factory will be
used by cursor().
Using *async*=True an asynchronous connection will be created. *async_* is
a valid alias (for Python versions where ``async`` is a keyword).
Any other keyword parameter will be passed to the underlying client
library: the list of supported parameters depends on the library version.
"""
kwasync = {}
if 'async' in kwargs:
kwasync['async'] = kwargs.pop('async')
if 'async_' in kwargs:
kwasync['async_'] = kwargs.pop('async_')
if dsn is None and not kwargs:
raise TypeError('missing dsn and no parameters')
dsn = _ext.make_dsn(dsn, **kwargs)
print dsn
#exit(0)
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
if cursor_factory is not None:
conn.cursor_factory = cursor_factory
return conn
def __init__(self):
self.pg_dsn = make_dsn(
host=os.environ["SYNAPSE_DB_HOST"],
# port is optional and make_dsn ignores None values, so we use .get()
port=os.environ.get("SYNAPSE_DB_PORT"),
user=os.environ["SYNAPSE_DB_USERNAME"],
password=os.environ["SYNAPSE_DB_PASSWORD"],
dbname=os.environ["SYNAPSE_DB_DATABASE"],
options=os.environ["SYNAPSE_DB_OPTIONS"],
)
self.STATS_DB_HOST = os.environ["STATS_DB_HOST"]
self.STATS_DB_USERNAME = os.environ["STATS_DB_USERNAME"]
self.STATS_DB_PASSWORD = os.environ["STATS_DB_PASSWORD"]
self.STATS_DB_DATABASE = os.environ["STATS_DB_DATABASE"]
def connect(dsn=None, connection_factory=None, cursor_factory=None, **kwargs):
"""
Create a new database connection.
The connection parameters can be specified as a string:
conn = psycopg2.connect("dbname=test user=postgres password=secret")
or using a set of keyword arguments:
conn = psycopg2.connect(database="test", user="******", password="******")
Or as a mix of both. The basic connection parameters are:
- *dbname*: the database name
- *database*: the database name (only as keyword argument)
- *user*: user name used to authenticate
- *password*: password used to authenticate
- *host*: database host address (defaults to UNIX socket if not provided)
- *port*: connection port number (defaults to 5432 if not provided)
Using the *connection_factory* parameter a different class or connections
factory can be specified. It should be a callable object taking a dsn
argument.
Using the *cursor_factory* parameter, a new default cursor factory will be
used by cursor().
Using *async*=True an asynchronous connection will be created. *async_* is
a valid alias (for Python versions where ``async`` is a keyword).
Any other keyword parameter will be passed to the underlying client
library: the list of supported parameters depends on the library version.
"""
kwasync = {}
if 'async' in kwargs:
kwasync['async'] = kwargs.pop('async')
if 'async_' in kwargs:
kwasync['async_'] = kwargs.pop('async_')
if dsn is None and not kwargs:
raise TypeError('missing dsn and no parameters')
dsn = _ext.make_dsn(dsn, **kwargs)
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
if cursor_factory is not None:
conn.cursor_factory = cursor_factory
return conn
def init():
global pool
try:
pool = ThreadSafeConnectionPool(minconn=0, maxconn=5000, idle_timeout=300,
dsn=make_dsn(
host = config.database_host,
dbname = config.database_dbname,
user = config.database_user,
password = config.database_password,
port = 5432
)
)
except Exception as e:
print(f'Failed to connect to the database: {e}')
return pool
def connect(self,
dbname=None,
user=None,
password=None,
host=None,
port=None,
attempts=ATTEMPTS):
"""TODO."""
connection_kwargs = {
'connect_timeout': self.TIMEOUT,
'dbname': dbname,
'host': host,
'password': password,
'port': port,
'user': user
}
connectionString = make_dsn(**connection_kwargs)
try:
self.logInfo('Connecting to ' +
self.getConnectionString(**connection_kwargs) + ' (' +
connectionString + ')...')
with connect(connectionString,
BetterLoggingConnection) as connection:
if hasattr(self, 'connection'):
self.connection.close()
self.connection = connection
self.logInfo('...connected; configuring connection...')
connection.initialize(self.logger)
connection.notices = deque(connection.notices)
connection.set_client_encoding(self.ENCODING)
self.logInfo('...connection configured...')
return self
except OperationalError as error:
self.logDebug(str(error))
self.logDebug(str(error.pgcode))
self.logError('...' + str(error.pgerror).strip() + '...')
self.logError('...couldn\'t connect.')
self.logDebug(connection_kwargs)
raise ExitException()
def parse_value(value, proto):
if '://' in value:
proto, value = value.split('://', 1)
display_value = '{}://{}'.format(proto, value)
if proto == 'tcp':
if ':' not in value:
raise argparse.ArgumentTypeError(
'Invalid service spec {!r}. Must have ":". Where\'s the port?'.
format(display_value))
host, port = value.strip('/').split(':', 1)
if not port.isdigit():
raise argparse.ArgumentTypeError(
'Invalid service spec {!r}. Port must be a number not {!r}.'.
format(display_value, port))
port = int(port)
return TcpCheck(host, port)
elif proto in ('pg', 'postgresql', 'postgres'):
if psycopg2 is None:
raise argparse.ArgumentTypeError(
'Protocol {} unusable. Install holdup[pg].'.format(proto))
uri = 'postgresql://{}'.format(value)
try:
connection_uri = make_dsn(uri)
except Exception as exc:
raise argparse.ArgumentTypeError(
'Failed to parse %r: %s. Must be a valid connection URI.' %
(display_value, exc))
return PgCheck(connection_uri)
elif proto == 'unix':
return UnixCheck(value)
elif proto == 'path':
return PathCheck(value)
elif proto in ('http', 'https', 'https+insecure'):
return HttpCheck('%s://%s' % (proto, value))
elif proto == 'eval':
return EvalCheck(value)
else:
raise argparse.ArgumentTypeError(
'Unknown protocol {!r} in {!r}. Must be "tcp", "path", "unix" or "pg".'
.format(proto, display_value))
def parse_value(value, proto):
if '://' in value:
proto, value = value.split('://', 1)
if proto == 'tcp':
if ':' not in value:
raise argparse.ArgumentTypeError(
'Invalid service spec %r. Must have ":". Where\'s the port?' %
value)
host, port = value.strip('/').split(':', 1)
if not port.isdigit():
raise argparse.ArgumentTypeError(
'Invalid service spec %r. Port must be a number.' % value)
port = int(port)
return TcpCheck(host, port)
elif proto in ('pg', 'postgresql', 'postgres'):
if not psycopg2_imported:
raise ImportError(
"You need to install psycopg2 in order to use postgresql protocol"
)
try:
connection_uri = make_dsn('postgresql://{}'.format(value))
except Exception as exc:
raise ArgumentTypeError('Failed to parse %s : %s. Expected format is'\
'"postgresql://*****:*****@host:port/dbname"' % (value, exc))
return PgCheck(connection_uri)
elif proto == 'unix':
return UnixCheck(value)
elif proto == 'path':
return PathCheck(value)
elif proto in ('http', 'https', 'https+insecure'):
return HttpCheck('%s://%s' % (proto, value))
elif proto == 'eval':
return EvalCheck(value)
else:
raise argparse.ArgumentTypeError(
'Unknown protocol %r in %r. Must be "tcp", "path" or "unix".' %
(proto, value))
def test_no_dsn_munging(self):
dsnin = 'dbname=a host=b user=c password=d'
dsn = ext.make_dsn(dsnin)
self.assertEqual(dsn, dsnin)
def test_params_merging(self):
dsn = ext.make_dsn('dbname=foo host=bar', host='baz')
self.assertDsnEqual(dsn, 'dbname=foo host=baz')
dsn = ext.make_dsn('dbname=foo', user='******')
self.assertDsnEqual(dsn, 'dbname=foo user=postgres')
def test_database_is_a_keyword(self):
self.assertEqual(ext.make_dsn(database='sigh'), "dbname=sigh")
- *dbname*: the database name
- *database*: the database name (only as keyword argument)
- *user*: user name used to authenticate
- *password*: password used to authenticate
- *host*: database host address (defaults to UNIX socket if not provided)
- *port*: connection port number (defaults to 5432 if not provided)
Using the *connection_factory* parameter a different class or connections
factory can be specified. It should be a callable object taking a dsn
argument.
Using the *cursor_factory* parameter, a new default cursor factory will be
used by cursor().
Using *async*=True an asynchronous connection will be created.
Any other keyword parameter will be passed to the underlying client
library: the list of supported parameters depends on the library version.
"""
if dsn is None and not kwargs:
raise TypeError('missing dsn and no parameters')
dsn = _ext.make_dsn(dsn, **kwargs)
conn = _connect(dsn, connection_factory=connection_factory, async=async)
if cursor_factory is not None:
conn.cursor_factory = cursor_factory
return conn
def test_empty_param(self):
dsn = ext.make_dsn(dbname='sony', password='')
self.assertDsnEqual(dsn, "dbname=sony password=''")
def test_empty_string(self):
dsn = ext.make_dsn('')
self.assertEqual(dsn, '')
def test_empty_arguments(self):
self.assertEqual(ext.make_dsn(), '')
- *dbname*: the database name
- *database*: the database name (only as keyword argument)
- *user*: user name used to authenticate
- *password*: password used to authenticate
- *host*: database host address (defaults to UNIX socket if not provided)
- *port*: connection port number (defaults to 5432 if not provided)
Using the *connection_factory* parameter a different class or connections
factory can be specified. It should be a callable object taking a dsn
argument.
Using the *cursor_factory* parameter, a new default cursor factory will be
used by cursor().
Using *async*=True an asynchronous connection will be created.
Any other keyword parameter will be passed to the underlying client
library: the list of supported parameters depends on the library version.
"""
if dsn is None and not kwargs:
raise TypeError('missing dsn and no parameters')
dsn = _ext.make_dsn(dsn, **kwargs)
conn = _connect(dsn, connection_factory=connection_factory, async=async)
if cursor_factory is not None:
conn.cursor_factory = cursor_factory
return conn
def connection_info_for(self, dbname):
# Already reloaded by _patch_odoo_db_connect
# reloaded = self.DbConfig.reload()
dsn = self.DbConfig.resolve_dsn(dbname)
return dbname, parse_dsn(make_dsn(dsn, dbname=dbname, application_name="odoo"))
def _ensure_dsn_with_passfile(dsn):
dsn_dict = parse_dsn(
dsn) # Validates here as (desireable) side effect
if not dsn_dict.get("passfile"):
dsn_dict["passfile"] = passfile
return make_dsn(**dsn_dict)
def connect(self,
database=None,
user=None,
password=None,
host=None,
port=None,
dsn=None,
**kwargs):
conn_params = self._conn_params.copy()
new_params = {
"database": database,
"user": user,
"password": password,
"host": host,
"port": port,
"dsn": dsn,
}
new_params.update(kwargs)
if new_params["dsn"]:
new_params = {
"dsn": new_params["dsn"],
"password": new_params["password"]
}
if new_params["password"]:
new_params["dsn"] = make_dsn(
new_params["dsn"], password=new_params.pop("password"))
conn_params.update(
{k: unicode2utf8(v)
for k, v in new_params.items() if v})
conn = psycopg2.connect(**conn_params)
cursor = conn.cursor()
conn.set_client_encoding("utf8")
self._conn_params = conn_params
if self.conn:
self.conn.close()
self.conn = conn
self.conn.autocommit = True
# When we connect using a DSN, we don't really know what db,
# user, etc. we connected to. Let's read it.
# Note: moved this after setting autocommit because of #664.
# TODO: use actual connection info from psycopg2.extensions.Connection.info as psycopg>2.8 is available and required dependency # noqa
dsn_parameters = conn.get_dsn_parameters()
self.dbname = dsn_parameters.get("dbname")
self.user = dsn_parameters.get("user")
self.password = password
self.host = dsn_parameters.get("host")
self.port = dsn_parameters.get("port")
self.extra_args = kwargs
if not self.host:
self.host = self.get_socket_directory()
cursor.execute("SHOW ALL")
db_parameters = dict(name_val_desc[:2]
for name_val_desc in cursor.fetchall())
pid = self._select_one(cursor, "select pg_backend_pid()")[0]
self.pid = pid
self.superuser = db_parameters.get("is_superuser") == "1"
self.server_version = self.get_server_version(cursor)
register_date_typecasters(conn)
register_json_typecasters(self.conn, self._json_typecaster)
register_hstore_typecaster(self.conn)
def connect(
self,
database=None,
user=None,
password=None,
host=None,
port=None,
dsn=None,
**kwargs,
):
conn_params = self._conn_params.copy()
new_params = {
"database": database,
"user": user,
"password": password,
"host": host,
"port": port,
"dsn": dsn,
}
new_params.update(kwargs)
if new_params["dsn"]:
new_params = {
"dsn": new_params["dsn"],
"password": new_params["password"]
}
if new_params["password"]:
new_params["dsn"] = make_dsn(
new_params["dsn"], password=new_params.pop("password"))
conn_params.update({k: v for k, v in new_params.items() if v})
conn = psycopg2.connect(**conn_params)
cursor = conn.cursor()
conn.set_client_encoding("utf8")
self._conn_params = conn_params
if self.conn:
self.conn.close()
self.conn = conn
self.conn.autocommit = True
# When we connect using a DSN, we don't really know what db,
# user, etc. we connected to. Let's read it.
# Note: moved this after setting autocommit because of #664.
libpq_version = psycopg2.__libpq_version__
dsn_parameters = {}
if libpq_version >= 93000:
# use actual connection info from psycopg2.extensions.Connection.info
# as libpq_version > 9.3 is available and required dependency
dsn_parameters = conn.info.dsn_parameters
else:
try:
dsn_parameters = conn.get_dsn_parameters()
except Exception as x:
# https://github.com/dbcli/pgcli/issues/1110
# PQconninfo not available in libpq < 9.3
_logger.info("Exception in get_dsn_parameters: %r", x)
if dsn_parameters:
self.dbname = dsn_parameters.get("dbname")
self.user = dsn_parameters.get("user")
self.host = dsn_parameters.get("host")
self.port = dsn_parameters.get("port")
else:
self.dbname = conn_params.get("database")
self.user = conn_params.get("user")
self.host = conn_params.get("host")
self.port = conn_params.get("port")
self.password = password
self.extra_args = kwargs
if not self.host:
self.host = self.get_socket_directory()
pid = self._select_one(cursor, "select pg_backend_pid()")[0]
self.pid = pid
self.superuser = conn.get_parameter_status("is_superuser") in ("on",
"1")
self.server_version = conn.get_parameter_status("server_version")
register_date_typecasters(conn)
register_json_typecasters(self.conn, self._json_typecaster)
register_hstore_typecaster(self.conn)
def db(pg_conn_main, pg_conn_test, mocker):
dsn_params = pg_conn_main.info.dsn_parameters
mocker.patch("dodoo.utils.db._dsn_resolver",
lambda _: make_dsn(**dsn_params))
yield pg_conn_test.info.dbname
