def get_uri(self):
"""Return URI instance with a defined host (and port, for TCP)."""
if not self.environment_variable:
raise RuntimeError("Define at least %s.environment_variable" %
type(self).__name__)
uri_str = os.environ.get(self.host_environment_variable)
if uri_str:
uri = URI(uri_str)
if not uri.host:
raise RuntimeError("The URI in %s must include a host." %
self.host_environment_variable)
if not uri.host.startswith("/") and not uri.port:
if not self.default_port:
raise RuntimeError("Define at least %s.default_port" %
type(self).__name)
uri.port = self.default_port
return uri
else:
uri_str = os.environ.get(self.environment_variable)
if uri_str:
uri = URI(uri_str)
if uri.host:
if not uri.host.startswith("/") and not uri.port:
if not self.default_port:
raise RuntimeError(
"Define at least %s.default_port" %
type(self).__name)
uri.port = self.default_port
return uri
return None
def get_uri(self):
"""Return an URI instance using the uri config for this model
"""
if self.uri is not None:
uri = URI(self.uri)
else:
uri = URI(config.Database().uri)
return uri
def get_uri(self):
"""Return an URI instance using the uri config for this model
"""
if self.uri is not None:
uri = URI(self.uri)
else:
config_uri = config.Database().uri
if type(config_uri) is dict:
uri = URI(config_uri[self.mamba_database()])
else:
uri = URI(config_uri)
return uri
def getCreationSQL(store):
connType = store._connection.__class__.__name__
return {
'SQLiteConnection': {
'tableExists':
lambda s, t: bool(
s.execute(
"SELECT count(*) FROM sqlite_master where name = '%s'" % t)
.get_one()[0]),
'creations': [
('warp_avatar', """
CREATE TABLE warp_avatar (
id INTEGER NOT NULL PRIMARY KEY,
email VARCHAR,
password VARCHAR,
UNIQUE(email))"""),
('warp_session', """
CREATE TABLE warp_session (
uid BYTEA NOT NULL PRIMARY KEY,
avatar_id INTEGER REFERENCES warp_avatar(id) ON DELETE CASCADE)"""
),
('warp_avatar_role', """
CREATE TABLE warp_avatar_role (
id INTEGER NOT NULL PRIMARY KEY,
avatar_id INTEGER NOT NULL REFERENCES warp_avatar(id) ON DELETE CASCADE,
role_name BYTEA NOT NULL,
position INTEGER NOT NULL DEFAULT 0)"""),
],
},
'MySQLConnection': {
'tableExists':
lambda s, t: bool(
s.execute(
"""
SELECT count(*) FROM information_schema.tables
WHERE table_schema = ? AND table_name=?""",
(URI(config['db']).database, t)).get_one()[0]),
'creations': [
('warp_avatar', """
CREATE TABLE warp_avatar (
id INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY,
email VARCHAR(64),
password VARCHAR(32),
UNIQUE(email)
) engine=InnoDB, charset=utf8"""),
('warp_session', """
CREATE TABLE warp_session (
uid VARBINARY(32) NOT NULL PRIMARY KEY,
avatar_id INTEGER REFERENCES warp_avatar(id) ON DELETE CASCADE
) engine=InnoDB, charset=utf8"""),
('warp_avatar_role', """
CREATE TABLE warp_avatar_role (
id INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY,
avatar_id INTEGER NOT NULL REFERENCES warp_avatar(id) ON DELETE CASCADE,
role_name VARBINARY(32) NOT NULL,
position INTEGER NOT NULL
) engine=InnoDB, charset=utf8"""),
],
},
}.get(connType)
def test_parse_username_password_host_empty_port(self):
uri = URI("scheme://user%6eame:pass%77ord@ho%73t:")
self.assertEquals(uri.scheme, "scheme")
self.assertEquals(uri.username, "username")
self.assertEquals(uri.password, "password")
self.assertEquals(uri.host, "host")
self.assertEquals(uri.port, None)
def test_charset_option(self):
uri = URI(os.environ["STORM_MYSQL_URI"])
uri.options["charset"] = "ascii"
database = create_database(uri)
connection = database.connect()
result = connection.execute("SELECT @@character_set_client")
self.assertEquals(result.get_one(), ("ascii", ))
def __init__(self, uri):
if MySQLdb is dummy:
raise DatabaseModuleError("'MySQLdb' module not found")
self._connect_kwargs = {}
if isinstance(uri, six.string_types):
uri = URI(uri)
if uri.database is not None:
self._connect_kwargs["db"] = uri.database
if uri.host is not None:
self._connect_kwargs["host"] = uri.host
if uri.port is not None:
self._connect_kwargs["port"] = uri.port
if uri.username is not None:
self._connect_kwargs["user"] = uri.username
if uri.password is not None:
self._connect_kwargs["passwd"] = uri.password
for option in ["unix_socket"]:
if option in uri.options:
self._connect_kwargs[option] = uri.options.get(option)
if self._converters is None:
# MySQLdb returns a timedelta by default on TIME fields.
converters = MySQLdb.converters.conversions.copy()
converters[MySQLdb.converters.FIELD_TYPE.TIME] = _convert_time
self.__class__._converters = converters
self._connect_kwargs["conv"] = self._converters
self._connect_kwargs["use_unicode"] = True
self._connect_kwargs["charset"] = uri.options.get("charset", "utf8")
def test_parse_username_password_host_port():
uri = URI("scheme://user%6eame:pass%77ord@ho%73t:1234")
assert uri.scheme == "scheme"
assert uri.username == "username"
assert uri.password == "password"
assert uri.host == "host"
assert uri.port == 1234
def create_proxy_and_uri(uri):
"""Create a TCP proxy to a Unix-domain database identified by `uri`."""
proxy = ProxyTCPServer(os.path.join(uri.host, ".s.PGSQL.5432"))
proxy_host, proxy_port = proxy.server_address
proxy_uri = URI(
urlunsplit(("postgres", "%s:%s" % (proxy_host, proxy_port),
"/storm_test", "", "")))
return proxy, proxy_uri
def test_synchronous(self):
synchronous_values = {"OFF": 0, "NORMAL": 1, "FULL": 2}
for value in synchronous_values:
database = SQLite(
URI("sqlite:%s?synchronous=%s" % (self.get_path(), value)))
connection = database.connect()
result = connection.execute("PRAGMA synchronous")
self.assertEqual(result.get_one()[0], synchronous_values[value])
def test_parse_username_password_database(self):
uri = URI("scheme://user%6eame:pass%77ord@/d%61tabase")
self.assertEquals(uri.scheme, "scheme")
self.assertEquals(uri.username, "username")
self.assertEquals(uri.password, "password")
self.assertEquals(uri.host, None)
self.assertEquals(uri.port, None)
self.assertEquals(uri.database, "database")
def test_parse_username_password_host_port_database(self):
uri = URI("scheme://user%6eame:pass%77ord@ho%73t:1234/d%61tabase")
self.assertEquals(uri.scheme, "scheme")
self.assertEquals(uri.username, "username")
self.assertEquals(uri.password, "password")
self.assertEquals(uri.host, "host")
self.assertEquals(uri.port, 1234)
self.assertEquals(uri.database, "database")
def test_parse_username_password_database():
uri = URI("scheme://user%6eame:pass%77ord@/d%61tabase")
assert uri.scheme == "scheme"
assert uri.username == "username"
assert uri.password == "password"
assert uri.host == None
assert uri.port == None
assert uri.database == "database"
def test_parse_username_password_host_port_database():
uri = URI("scheme://user%6eame:pass%77ord@ho%73t:1234/d%61tabase")
assert uri.scheme == "scheme"
assert uri.username == "username"
assert uri.password == "password"
assert uri.host == "host"
assert uri.port == 1234
assert uri.database == "database"
def test_foreign_keys(self):
foreign_keys_values = {"ON": 1, "OFF": 0}
for value in foreign_keys_values:
database = SQLite(
URI("sqlite:%s?foreign_keys=%s" % (self.get_path(), value)))
connection = database.connect()
result = connection.execute("PRAGMA foreign_keys").get_one()[0]
self.assertEqual(result, foreign_keys_values[value])
def test_parse_defaults(self):
uri = URI("scheme:")
self.assertEquals(uri.scheme, "scheme")
self.assertEquals(uri.options, {})
self.assertEquals(uri.username, None)
self.assertEquals(uri.password, None)
self.assertEquals(uri.host, None)
self.assertEquals(uri.port, None)
self.assertEquals(uri.database, None)
def test_parse_defaults():
uri = URI("scheme:")
assert uri.scheme == "scheme"
assert uri.options == {}
assert uri.username == None
assert uri.password == None
assert uri.host == None
assert uri.port == None
assert uri.database == None
class Generator(object):
def __init__(self, verbose):
self._verbose = verbose
self._tables = []
self._sqlType = ''
self._dbconfig = None
def generate_database(self):
try:
db = Database()
except Exception, e:
print '\n' + red(e[0])
print '\nAborting'
exit(-1)
if self._verbose: print bold('Fixing null values on schema...')
(success, msg) = db.get_schema().fix_tables()
if not success:
raise DatabaseException(msg)
if self._verbose:
print green('Schema fixed!')
print bold('Retrieving database configuration...')
self._dbconfig = db.get_schema().get_properties()
uri = URI(self._dbconfig['uri'])
if self._verbose:
if uri.scheme == 'sqlite':
print bold('Trying to connect to {0} database'.format(
uri.database))
else:
print bold('Trying to connect to {0} server on {1} port {2} ' \
'database {3} with user {4} and passsword {5}...'.format(
uri.scheme.capitalize(), uri.host, uri.port,
uri.database, uri.username, uri.password))
self._sqlType = uri.scheme
try:
for table, columns in db.get_schema().get_tables().iteritems():
self._createTable(table, columns)
except SchemaException:
print 'WARNING: No tables are defined in your schema\n'+\
'Only Goliat Users table will be generated.'
for relation in db.get_schema().many2many():
table = relation['table']
cols = {}
for key in relation['keys']:
cols[key if type(key)==str else key.keys()[0]]=\
{ 'type' : 'integer', 'required' : True, 'primaryKey' : True }
if relation.get('fields') != None:
for field in relation['fields']:
for fname, fvalue in field.iteritems():
cols[fname] = fvalue
self._createTable(table, cols, True)
# Goliat User Table
from goliat.session.user import user_sql_data
self._createTable(user_sql_data.keys()[0], user_sql_data.values()[0])
def test_foreign_keys_persistency_to_rollback(self):
foreign_keys_values = {"ON": 1, "OFF": 0}
for value in foreign_keys_values:
database = SQLite(
URI("sqlite:%s?foreign_keys=%s" % (self.get_path(), value)))
connection = database.connect()
connection.execute("CREATE TABLE test (id INTEGER PRIMARY KEY)")
connection.rollback()
result = connection.execute("PRAGMA foreign_keys").get_one()[0]
self.assertEqual(result, foreign_keys_values[value])
def setUp(self, dsn=conf.LITE_DSN):
"""should load the dataset"""
from storm.uri import URI
from storm.locals import create_database, Store
from storm.tracer import debug
#debug(1)
self.store = Store(create_database(URI(dsn)))
self.fixture.store = self.store
setup_db(self.store)
def test_block_access_with_multiple_stores(self):
"""
If multiple L{Store}s are passed to L{block_access} they will all be
blocked until the managed context is left.
"""
database = SQLite(URI("sqlite:%s" % self.make_path()))
store = Store(database)
with block_access(self.store, store):
self.assertRaises(ConnectionBlockedError, self.store.execute,
"SELECT 1")
self.assertRaises(ConnectionBlockedError, store.execute,
"SELECT 1")
def _handle_shell_command(self, mgr=None):
"""This runs the shell for each of the databases supported.
"""
mamba_services, db = self._prepare_model_db()
uri = URI(mamba_services.config.Database().uri)
if db.backend == 'mysql':
_shell.mysql(uri)
elif db.backend == 'postgres':
_shell.postgres(uri)
elif db.backend == 'sqlite':
_shell.sqlite(uri)
def _create_uri(self, dbname=None):
# postgres is a special database which is always present,
# it was added in 8.1 which is thus our requirement'
dbname = dbname or 'postgres'
# Do not output the password in the logs
if not self.first:
log.info('connecting to %s' %
self._build_dsn(dbname, filter_password=True))
self.first = False
dsn = self._build_dsn(dbname, filter_password=False)
uri = URI(dsn)
uri.options['isolation'] = 'read-committed'
return uri
def test_journal(self):
journal_values = {
"DELETE": u'delete',
"TRUNCATE": u'truncate',
"PERSIST": u'persist',
"MEMORY": u'memory',
"WAL": u'wal',
"OFF": u'off'
}
for value in journal_values:
database = SQLite(
URI("sqlite:%s?journal_mode=%s" % (self.get_path(), value)))
connection = database.connect()
result = connection.execute("PRAGMA journal_mode").get_one()[0]
self.assertEqual(result, journal_values[value])
def setUp(self):
super(PostgresDisconnectionTestWithPGBouncerBase, self).setUp()
database_uri = URI(os.environ["STORM_POSTGRES_HOST_URI"])
database_user = database_uri.username or os.environ['USER']
database_dsn = make_dsn(database_uri)
# Create a pgbouncer fixture.
self.pgbouncer = pgbouncer.fixture.PGBouncerFixture()
self.pgbouncer.databases[database_uri.database] = database_dsn
self.pgbouncer.users[database_user] = "trusted"
self.pgbouncer.admin_users = [database_user]
self.useFixture(self.pgbouncer)
# Create a Database that uses pgbouncer.
pgbouncer_uri = database_uri.copy()
pgbouncer_uri.host = self.pgbouncer.host
pgbouncer_uri.port = self.pgbouncer.port
self.database = create_database(pgbouncer_uri)
def test_exception_when_unsupported(self):
# Install a directory in front of the search path.
module_dir = self.make_path()
os.mkdir(module_dir)
sys.path.insert(0, module_dir)
# Copy the real module over to a new place, since the old one is
# already using the real module, if it's available.
db_module = __import__("storm.databases."+self.db_module_name,
None, None, [""])
db_module_filename = db_module.__file__
if db_module_filename.endswith(".pyc"):
db_module_filename = db_module_filename[:-1]
shutil.copyfile(db_module_filename,
os.path.join(module_dir, "_fake_.py"))
dbapi_modules = {}
for dbapi_module_name in self.dbapi_module_names:
# If the real module is available, remove it from sys.modules.
dbapi_module = sys.modules.pop(dbapi_module_name, None)
if dbapi_module is not None:
dbapi_modules[dbapi_module_name] = dbapi_module
# Create a module which raises ImportError when imported, to fake
# a missing module.
dirname = self.make_path(path=os.path.join(module_dir,
dbapi_module_name))
os.mkdir(dirname)
self.make_path("raise ImportError",
os.path.join(module_dir, dbapi_module_name,
"__init__.py"))
# Finally, test it.
import _fake_
uri = URI("_fake_://db")
try:
with pytest.raises(DatabaseModuleError):
_fake_.create_from_uri(uri)
finally:
# Unhack the environment.
del sys.path[0]
del sys.modules["_fake_"]
sys.modules.update(dbapi_modules)
def test_journal_persistency_to_rollback(self):
journal_values = {
"DELETE": u'delete',
"TRUNCATE": u'truncate',
"PERSIST": u'persist',
"MEMORY": u'memory',
"WAL": u'wal',
"OFF": u'off'
}
for value in journal_values:
database = SQLite(
URI("sqlite:%s?journal_mode=%s" % (self.get_path(), value)))
connection = database.connect()
connection.execute("CREATE TABLE test (id INTEGER PRIMARY KEY)")
connection.rollback()
result = connection.execute("PRAGMA journal_mode").get_one()[0]
assert result == journal_values[value]
def create_database(uri):
"""Create a database instance.
@param uri: An URI instance, or a string describing the URI. Some examples:
- "sqlite:" An in memory sqlite database.
- "sqlite:example.db" A SQLite database called example.db
- "postgres:test" The database 'test' from the local postgres server.
- "postgres://*****:*****@host/test" The database test on machine host
with supplied user credentials, using postgres.
- "anything:..." Where 'anything' has previously been registered
with L{register_scheme}.
"""
if isinstance(uri, six.string_types):
uri = URI(uri)
if uri.scheme in _database_schemes:
factory = _database_schemes[uri.scheme]
else:
module = __import__("%s.databases.%s" % (storm.__name__, uri.scheme),
None, None, [""])
factory = module.create_from_uri
return factory(uri)
def _create_uri(self, dbname=None):
# postgres is a special database which is always present,
# it was added in 8.1 which is thus our requirement'
dbname = dbname or 'postgres'
# Do not output the password in the logs
if not self.first:
log.info('connecting to %s' % self._build_dsn(
dbname, filter_password=True))
self.first = False
dsn = self._build_dsn(dbname, filter_password=False)
uri = URI(dsn)
uri.options['isolation'] = 'read-committed'
if uri.host == "":
pair = test_local_database()
if pair is None:
raise DatabaseError(
_("Could not find a database server on this computer"))
uri.host = pair[0]
uri.port = int(pair[1])
return uri
def test_get_uri(self):
dummy = DummyModel()
self.assertEqual(dummy.get_uri().scheme, URI('sqlite:').scheme)
