def dump(database_name):
from trytond.tools import exec_command_pipe
cmd = ['pg_dump', '--format=c', '--no-owner']
env = {}
uri = parse_uri(config.get('database', 'uri'))
if uri.username:
cmd.append('--username='******'--host=' + uri.hostname)
if uri.port:
cmd.append('--port=' + str(uri.port))
if uri.password:
# if db_password is set in configuration we should pass
# an environment variable PGPASSWORD to our subprocess
# see libpg documentation
env['PGPASSWORD'] = uri.password
cmd.append(database_name)
pipe = exec_command_pipe(*tuple(cmd), env=env)
pipe.stdin.close()
data = pipe.stdout.read()
res = pipe.wait()
if res:
raise Exception('Couldn\'t dump database!')
return data
def get_connection(self, autocommit=False, readonly=False):
conv = MySQLdb.converters.conversions.copy()
conv[float] = lambda value, _: repr(value)
conv[MySQLdb.constants.FIELD_TYPE.TIME] = MySQLdb.times.Time_or_None
args = {
'db': self.name,
'sql_mode': 'traditional,postgresql',
'use_unicode': True,
'charset': 'utf8',
'conv': conv,
}
uri = parse_uri(config.get('database', 'uri'))
assert uri.scheme == 'mysql'
if uri.hostname:
args['host'] = uri.hostname
if uri.port:
args['port'] = uri.port
if uri.username:
args['user'] = uri.username
if uri.password:
args['passwd'] = urllib.unquote_plus(uri.password)
conn = MySQLdb.connect(**args)
cursor = conn.cursor()
cursor.execute('SET time_zone = "+00:00"')
return conn
def cursor(self, autocommit=False, readonly=False):
conv = MySQLdb.converters.conversions.copy()
conv[float] = lambda value, _: repr(value)
conv[MySQLdb.constants.FIELD_TYPE.TIME] = MySQLdb.times.Time_or_None
args = {
'db': self.database_name,
'sql_mode': 'traditional,postgresql',
'use_unicode': True,
'charset': 'utf8',
'conv': conv,
}
uri = parse_uri(config.get('database', 'uri'))
assert uri.scheme == 'mysql'
if uri.hostname:
args['host'] = uri.hostname
if uri.port:
args['port'] = uri.port
if uri.username:
args['user'] = uri.username
if uri.password:
args['passwd'] = urllib.unquote_plus(uri.password)
conn = MySQLdb.connect(**args)
cursor = Cursor(conn, self.database_name)
cursor.execute('SET time_zone = `UTC`')
return cursor
def dump(database_name):
from trytond.tools import exec_command_pipe
cmd = ['pg_dump', '--format=c', '--no-owner']
env = {}
uri = parse_uri(config.get('database', 'uri'))
if uri.username:
cmd.append('--username='******'--host=' + uri.hostname)
if uri.port:
cmd.append('--port=' + str(uri.port))
if uri.password:
# if db_password is set in configuration we should pass
# an environment variable PGPASSWORD to our subprocess
# see libpg documentation
env['PGPASSWORD'] = uri.password
cmd.append(database_name)
pipe = exec_command_pipe(*tuple(cmd), env=env)
pipe.stdin.close()
data = pipe.stdout.read()
res = pipe.wait()
if res:
raise Exception('Couldn\'t dump database!')
return data
def get_smtp_server(uri=None, strict=False):
if uri is None:
uri = config.get('email', 'uri')
ini_uri = uri
uri = parse_uri(uri)
extra = {}
if uri.query:
cast = {'timeout': int}
for key, value in parse_qs(uri.query, strict_parsing=True).items():
extra[key] = cast.get(key, lambda a: a)(value[0])
if uri.scheme.startswith('smtps'):
connector = smtplib.SMTP_SSL
else:
connector = smtplib.SMTP
try:
server = connector(uri.hostname, uri.port, **extra)
except Exception:
if strict:
raise
logger.error('fail to connect to %s', uri, exc_info=True)
return
if 'tls' in uri.scheme:
server.starttls()
if uri.username and uri.password:
server.login(
unquote_plus(uri.username),
unquote_plus(uri.password))
server.uri = ini_uri
return server
def _connection_params(cls, name):
uri = parse_uri(config.get('database', 'uri'))
if uri.path and uri.path != '/':
warnings.warn("The path specified in the URI will be overridden")
params = {
'dsn': uri._replace(path=name).geturl(),
}
return params
def dsn(cls, name):
uri = parse_uri(config.get('database', 'uri'))
host = uri.hostname and "host=%s" % uri.hostname or ''
port = uri.port and "port=%s" % uri.port or ''
name = "dbname=%s" % name
user = uri.username and "user=%s" % uri.username or ''
password = ("password=%s" %
urllib.unquote_plus(uri.password) if uri.password else '')
return '%s %s %s %s %s' % (host, port, name, user, password)
def dsn(cls, name):
uri = parse_uri(config.get('database', 'uri'))
assert uri.scheme == 'postgresql'
host = uri.hostname and "host=%s" % uri.hostname or ''
port = uri.port and "port=%s" % uri.port or ''
name = "dbname=%s" % name
user = uri.username and "user=%s" % uri.username or ''
password = ("password=%s" % urllib.unquote_plus(uri.password)
if uri.password else '')
return '%s %s %s %s %s' % (host, port, name, user, password)
def parse_ldap_url(uri):
unquote = urlparse.unquote
uri = parse_uri(uri)
dn = unquote(uri.path)[1:]
attributes, scope, filter_, extensions = (uri.query.split('?') +
[''] * 4)[:4]
if not scope:
scope = 'base'
extensions = urlparse.parse_qs(extensions)
return (uri, dn, unquote(attributes), unquote(scope), unquote(filter_),
extensions)
def parse_ldap_url(uri):
unquote = urlparse.unquote
uri = parse_uri(uri)
dn = unquote(uri.path)[1:]
attributes, scope, filter_, extensions = (
uri.query.split('?') + [''] * 4)[:4]
if not scope:
scope = 'base'
extensions = urlparse.parse_qs(extensions)
return (uri, dn, unquote(attributes), unquote(scope), unquote(filter_),
extensions)
def _pg_options():
uri = parse_uri(config.get('database', 'uri'))
options = []
env = os.environ.copy()
if uri.hostname:
options.extend(['-h', uri.hostname])
if uri.port:
options.extend(['-p', str(uri.port)])
if uri.username:
options.extend(['-U', uri.username])
if uri.password:
env['PGPASSWORD'] = uri.password
return options, env
def restore(database_name, data):
from trytond.tools import exec_command_pipe
database = Database().connect()
cursor = database.cursor(autocommit=True)
database.create(cursor, database_name)
cursor.commit()
cursor.close()
database.close()
cmd = ['pg_restore', '--no-owner']
env = {}
uri = parse_uri(config.get('database', 'uri'))
if uri.username:
cmd.append('--username='******'--host=' + uri.hostname)
if uri.port:
cmd.append('--port=' + str(uri.port))
if uri.password:
env['PGPASSWORD'] = uri.password
cmd.append('--dbname=' + database_name)
args2 = tuple(cmd)
if os.name == "nt":
tmpfile = (os.environ['TMP'] or 'C:\\') + os.tmpnam()
with open(tmpfile, 'wb') as fp:
fp.write(data)
args2 = list(args2)
args2.append(' ' + tmpfile)
args2 = tuple(args2)
pipe = exec_command_pipe(*args2, env=env)
if not os.name == "nt":
pipe.stdin.write(data)
pipe.stdin.close()
res = pipe.wait()
if res:
raise Exception('Couldn\'t restore database')
database = Database(database_name).connect()
cursor = database.cursor()
if not cursor.test():
cursor.close()
database.close()
raise Exception('Couldn\'t restore database!')
cursor.close()
database.close()
Database._list_cache = None
return True
def _connection_params(cls, name):
uri = parse_uri(config.get('database', 'uri'))
params = {
'dbname': name,
}
if uri.username:
params['user'] = uri.username
if uri.password:
params['password'] = urllib.parse.unquote_plus(uri.password)
if uri.hostname:
params['host'] = uri.hostname
if uri.port:
params['port'] = uri.port
return params
def restore(database_name, data):
from trytond.tools import exec_command_pipe
database = Database().connect()
cursor = database.cursor(autocommit=True)
database.create(cursor, database_name)
cursor.commit()
cursor.close()
database.close()
cmd = ['pg_restore', '--no-owner']
env = {}
uri = parse_uri(config.get('database', 'uri'))
if uri.username:
cmd.append('--username='******'--host=' + uri.hostname)
if uri.port:
cmd.append('--port=' + str(uri.port))
if uri.password:
env['PGPASSWORD'] = uri.password
cmd.append('--dbname=' + database_name)
args2 = tuple(cmd)
if os.name == "nt":
tmpfile = (os.environ['TMP'] or 'C:\\') + os.tmpnam()
with open(tmpfile, 'wb') as fp:
fp.write(data)
args2 = list(args2)
args2.append(' ' + tmpfile)
args2 = tuple(args2)
pipe = exec_command_pipe(*args2, env=env)
if not os.name == "nt":
pipe.stdin.write(data)
pipe.stdin.close()
res = pipe.wait()
if res:
raise Exception('Couldn\'t restore database')
database = Database(database_name).connect()
cursor = database.cursor()
if not cursor.test():
cursor.close()
database.close()
raise Exception('Couldn\'t restore database!')
cursor.close()
database.close()
Database._list_cache = None
return True
def connect(self):
if self._connpool is not None:
return self
logger.info('connect to "%s"', self.database_name)
uri = parse_uri(config.get('database', 'uri'))
assert uri.scheme == 'postgresql'
host = uri.hostname and "host=%s" % uri.hostname or ''
port = uri.port and "port=%s" % uri.port or ''
name = "dbname=%s" % self.database_name
user = uri.username and "user=%s" % uri.username or ''
password = uri.password and "password=%s" % uri.password or ''
minconn = config.getint('database', 'minconn', default=1)
maxconn = config.getint('database', 'maxconn', default=64)
dsn = '%s %s %s %s %s' % (host, port, name, user, password)
self._connpool = ThreadedConnectionPool(minconn, maxconn, dsn)
return self
def restore(database_name, data):
from trytond.tools import exec_command_pipe
database = Database().connect()
cursor = database.cursor(autocommit=True)
database.create(cursor, database_name)
cursor.commit()
cursor.close()
cmd = ['mysql']
uri = parse_uri(config.get('database', 'uri'))
if uri.username:
cmd.append('--user='******'--host=' + uri.hostname)
if uri.port:
cmd.append('--port=' + str(uri.port))
if uri.password:
cmd.append('--password='******'wb+') as fd:
fd.write(data)
cmd.append('<')
cmd.append(file_name)
args2 = tuple(cmd)
pipe = exec_command_pipe(*args2)
pipe.stdin.close()
res = pipe.wait()
os.remove(file_name)
if res:
raise Exception('Couldn\'t restore database')
database = Database(database_name).connect()
cursor = database.cursor()
if not cursor.test():
cursor.close()
database.close()
raise Exception('Couldn\'t restore database!')
cursor.close()
database.close()
Database._list_cache = None
return True
def restore(database_name, data):
from trytond.tools import exec_command_pipe
database = Database().connect()
cursor = database.cursor(autocommit=True)
database.create(cursor, database_name)
cursor.commit()
cursor.close()
cmd = ['mysql']
uri = parse_uri(config.get('database', 'uri'))
if uri.username:
cmd.append('--user='******'--host=' + uri.hostname)
if uri.port:
cmd.append('--port=' + str(uri.port))
if uri.password:
cmd.append('--password='******'wb+') as fd:
fd.write(data)
cmd.append('<')
cmd.append(file_name)
args2 = tuple(cmd)
pipe = exec_command_pipe(*args2)
pipe.stdin.close()
res = pipe.wait()
os.remove(file_name)
if res:
raise Exception('Couldn\'t restore database')
database = Database(database_name).connect()
cursor = database.cursor()
if not cursor.test():
cursor.close()
database.close()
raise Exception('Couldn\'t restore database!')
cursor.close()
database.close()
Database._list_cache = None
return True
def connect(self):
if self._connpool is not None:
return self
logger.info('connect to "%s"', self.database_name)
uri = parse_uri(config.get('database', 'uri'))
assert uri.scheme == 'postgresql'
host = uri.hostname and "host=%s" % uri.hostname or ''
port = uri.port and "port=%s" % uri.port or ''
name = "dbname=%s" % self.database_name
user = uri.username and "user=%s" % uri.username or ''
password = ("password=%s" % urllib.unquote_plus(uri.password)
if uri.password else '')
minconn = config.getint('database', 'minconn', default=1)
maxconn = config.getint('database', 'maxconn', default=64)
dsn = '%s %s %s %s %s' % (host, port, name, user, password)
self._connpool = ThreadedConnectionPool(minconn, maxconn, dsn)
return self
def _connection_params(cls, name):
uri = parse_uri(config.get('database', 'uri'))
params = {
'dbname': name,
}
if uri.username:
params['user'] = uri.username
if uri.password:
params['password'] = urllib.parse.unquote_plus(uri.password)
if uri.hostname:
params['host'] = uri.hostname
if uri.port:
params['port'] = uri.port
query_params = urllib.parse.parse_qs(uri.query)
if query_params.get('application_name'):
params['application_name'] = "-".join(
query_params['application_name'])
return params
def list(cursor):
now = time.time()
timeout = config.getint('session', 'timeout')
res = Database._list_cache
if res and abs(Database._list_cache_timestamp - now) < timeout:
return res
uri = parse_uri(config.get('database', 'uri'))
db_user = uri.username or os.environ.get('PGUSER')
if not db_user and os.name == 'posix':
db_user = pwd.getpwuid(os.getuid())[0]
if db_user:
cursor.execute(
"SELECT datname "
"FROM pg_database "
"WHERE datdba = ("
"SELECT usesysid "
"FROM pg_user "
"WHERE usename=%s) "
"AND datname not in "
"('template0', 'template1', 'postgres') "
"ORDER BY datname", (db_user, ))
else:
cursor.execute("SELECT datname "
"FROM pg_database "
"WHERE datname not in "
"('template0', 'template1','postgres') "
"ORDER BY datname")
res = []
for db_name, in cursor.fetchall():
db_name = db_name.encode('utf-8')
try:
database = Database(db_name).connect()
except Exception:
continue
cursor2 = database.cursor()
if cursor2.test():
res.append(db_name)
cursor2.close(close=True)
else:
cursor2.close(close=True)
database.close()
Database._list_cache = res
Database._list_cache_timestamp = now
return res
def get_smtp_server():
"""
Instanciate, configure and return a SMTP or SMTP_SSL instance from
smtplib.
:return: A SMTP instance. The quit() method must be call when all
the calls to sendmail() have been made.
"""
uri = parse_uri(config.get('email', 'uri'))
if uri.scheme.startswith('smtps'):
smtp_server = smtplib.SMTP_SSL(uri.hostname, uri.port)
else:
smtp_server = smtplib.SMTP(uri.hostname, uri.port)
if 'tls' in uri.scheme:
smtp_server.starttls()
if uri.username and uri.password:
smtp_server.login(uri.username, uri.password)
return smtp_server
def dump(database_name):
from trytond.tools import exec_command_pipe
cmd = ['mysqldump', '--no-create-db']
uri = parse_uri(config.get('database', 'uri'))
if uri.username:
cmd.append('--user='******'--host=' + uri.hostname)
if uri.port:
cmd.append('--port=' + str(uri.port))
if uri.password:
cmd.append('--password='******'Couldn\'t dump database!')
return data
def dump(database_name):
from trytond.tools import exec_command_pipe
cmd = ['mysqldump', '--no-create-db']
uri = parse_uri(config.get('database', 'uri'))
if uri.username:
cmd.append('--user='******'--host=' + uri.hostname)
if uri.port:
cmd.append('--port=' + str(uri.port))
if uri.password:
cmd.append('--password='******'Couldn\'t dump database!')
return data
def get_smtp_server():
"""
Instanciate, configure and return a SMTP or SMTP_SSL instance from
smtplib.
:return: A SMTP instance. The quit() method must be call when all
the calls to sendmail() have been made.
"""
uri = parse_uri(config.get('email', 'uri'))
if uri.scheme.startswith('smtps'):
smtp_server = smtplib.SMTP_SSL(uri.hostname, uri.port)
else:
smtp_server = smtplib.SMTP(uri.hostname, uri.port)
if 'tls' in uri.scheme:
smtp_server.starttls()
if uri.username and uri.password:
smtp_server.login(
urllib.unquote_plus(uri.username),
urllib.unquote_plus(uri.password))
return smtp_server
def _make_uri(self):
uri = config.get('database', 'uri')
base_uri = parse_uri(uri)
if base_uri.path and base_uri.path != '/':
warnings.warn("The path specified in the URI will be overridden")
if self.name == ':memory:':
query_string = urllib.parse.parse_qs(base_uri.query)
query_string['mode'] = 'memory'
query = urllib.parse.urlencode(query_string, doseq=True)
db_uri = base_uri._replace(netloc='', path='/', query=query)
else:
db_path = safe_join(config.get('database', 'path'),
self.name + '.sqlite')
if not os.path.isfile(db_path):
raise IOError("Database '%s' doesn't exist!" % db_path)
db_uri = base_uri._replace(path=db_path)
# Use unparse before replacing sqlite with file because SQLite accepts
# a relative path URI like file:db/test.sqlite which doesn't conform to
# RFC8089 which urllib follows and enforces when the scheme is 'file'
db_uri = urllib.parse.urlunparse(db_uri)
return db_uri.replace('sqlite', 'file', 1)
def aggregate_function():
uri = parse_uri(config.get('database', 'uri'))
if uri.scheme == 'postgresql':
return "array_agg(r.id)"
return "group_concat(r.id, ',')"
def wrapper(request, *args, **kwargs):
uri = config.get('database', 'uri')
database_name = parse_uri(uri).path[1:]
return with_pool(func)(request, database_name, *args, **kwargs)
