def all_to_intermediary(filename_or_input, schema=None):
""" Dispatch the filename_or_input to the different function to produce the intermediary syntax.
All the supported classes names are in `swich_input_class_to_method`.
The input can also be a list of strings in markdown format or a filename finishing by '.er' containing markdown
format.
"""
# Try to convert from the name of the class
input_class_name = filename_or_input.__class__.__name__
try:
this_to_intermediary = switch_input_class_to_method[input_class_name]
tables, relationships = this_to_intermediary(filename_or_input)
return tables, relationships
except KeyError:
pass
# try to read markdown file.
if isinstance(filename_or_input, basestring):
if filename_or_input.split('.')[-1] == 'er':
return markdown_file_to_intermediary(filename_or_input)
# try to read a markdown in a string
if not isinstance(filename_or_input, basestring):
if all(isinstance(e, basestring) for e in filename_or_input):
return line_iterator_to_intermediary(filename_or_input)
# try to read DB URI.
try:
make_url(filename_or_input)
return database_to_intermediary(filename_or_input, schema=schema)
except ArgumentError:
pass
msg = 'Cannot process filename_or_input {}'.format(input_class_name)
raise ValueError(msg)
def test_comparison(self):
components = (
"drivername",
"username",
"password",
"host",
"database",
"query",
"port",
)
common_url = (
"dbtype://*****:*****@[2001:da8:2004:1000:202:116:160:90]:80/database?foo=bar"
)
other_url = "dbtype://*****:*****@host/"
url1 = url.make_url(common_url)
url2 = url.make_url(common_url)
url3 = url.make_url(other_url)
is_true(url1 == url2)
is_false(url1 != url2)
is_true(url1 != url3)
is_false(url1 == url3)
for curr_component in components:
setattr(url2, curr_component, "new_changed_value")
is_true(url1 != url2)
is_false(url1 == url2)
setattr(url2, curr_component, getattr(url1, curr_component))
def normalize_url(raw):
"""
Build a URL from a string according to obvious rules.
Converts *.sqlite paths to sqlite:/// URLs, etc.
>>> str(normalize_url("/tmp/foo.sqlite"))
'sqlite:////tmp/foo.sqlite'
>>> str(normalize_url("postgresql://user@host/database"))
'postgresql://user@host/database'
"""
from sqlalchemy.engine.url import make_url
from sqlalchemy.exceptions import ArgumentError
try:
return make_url(raw)
except ArgumentError:
from os.path import (
abspath,
splitext,
)
(root, extension) = splitext(raw)
if extension == ".sqlite":
return make_url("sqlite:///%s" % abspath(raw))
else:
raise ArgumentError("could not parse supposed URL \"%s\"" % raw)
def _create_connection(db):
if isinstance(db, str):
url = make_url(connections.databases[db]['ENGINE'])
elif isinstance(db, dict):
url = make_url(db['ENGINE'])
else:
url = db
db_engine = url.drivername.split('+')[0]
database = url.database
kwargs = {'autocommit': True}
if db_engine == 'sqlite':
return
elif db_engine == 'postgresql':
database = 'postgres'
elif db_engine == 'mssql':
kwargs['isolation_level'] = 'AUTOCOMMIT'
database = 'master'
elif db_engine == 'mysql':
database = None
elif db_engine == 'oracle':
database = connections.databases[DEFAULT_DB_ALIAS].get('SYSTEM_DB', 'SYSTEM')
url = URL(url.drivername, url.username, url.password, url.host, url.port, database, url.query)
return create_engine(url).connect().execution_options(**kwargs)
def test_rfc1738_password(self):
u = url.make_url("dbtype://*****:*****@host/dbname")
eq_(u.password, "pass word + other:words")
eq_(str(u), "dbtype://*****:*****@host/dbname")
u = url.make_url(
"dbtype://*****:*****@hostspec/database"
)
eq_(u.password, "apples/oranges")
eq_(str(u), "dbtype://*****:*****@hostspec/database")
u = url.make_url(
"dbtype://*****:*****@hostspec/database"
)
eq_(u.password, "apples@oranges@@")
eq_(
str(u),
"dbtype://*****:*****@hostspec/database",
)
u = url.make_url("dbtype://username%40:@hostspec/database")
eq_(u.password, "")
eq_(u.username, "username@")
eq_(str(u), "dbtype://username%40:@hostspec/database")
u = url.make_url("dbtype://*****:*****@hostspec/database")
eq_(u.password, "pass/word")
eq_(str(u), "dbtype://*****:*****@hostspec/database")
def __init__(self):
try:
super(SQLRepository, self).__init__()
self.conf = cfg.CONF
url = None
if self.conf.mysql.database_name is not None:
settings_db = (self.conf.mysql.username,
self.conf.mysql.password,
self.conf.mysql.hostname,
self.conf.mysql.database_name)
url = make_url("mysql+pymysql://%s:%s@%s/%s" % settings_db)
else:
if self.conf.database.url is not None:
url = make_url(self.conf.database.url)
else:
database_conf = dict(self.conf.database)
if 'url' in database_conf:
del database_conf['url']
url = URL(**database_conf)
from sqlalchemy import create_engine
self._db_engine = create_engine(url, pool_recycle=3600)
self.metadata = MetaData()
except Exception as ex:
LOG.exception(ex)
raise exceptions.RepositoryException(ex)
def configure(cls, connection=None, url=None, dialect_name=None, opts={}):
"""Create a new :class:`.MigrationContext`.
This is a factory method usually called
by :meth:`.EnvironmentContext.configure`.
:param connection: a :class:`~sqlalchemy.engine.Connection`
to use for SQL execution in "online" mode. When present,
is also used to determine the type of dialect in use.
:param url: a string database url, or a
:class:`sqlalchemy.engine.url.URL` object.
The type of dialect to be used will be derived from this if
``connection`` is not passed.
:param dialect_name: string name of a dialect, such as
"postgresql", "mssql", etc. The type of dialect to be used will be
derived from this if ``connection`` and ``url`` are not passed.
:param opts: dictionary of options. Most other options
accepted by :meth:`.EnvironmentContext.configure` are passed via
this dictionary.
"""
if connection:
dialect = connection.dialect
elif url:
url = sqla_url.make_url(url)
dialect = url.get_dialect()()
elif dialect_name:
url = sqla_url.make_url("%s://" % dialect_name)
dialect = url.get_dialect()()
else:
raise Exception("Connection, url, or dialect_name is required.")
return MigrationContext(dialect, connection, opts)
def test_pymssql_port_setting(self):
dialect = pymssql.dialect()
u = url.make_url("mssql+pymssql://scott:tiger@somehost/test")
connection = dialect.create_connect_args(u)
eq_(
[
[],
{
"host": "somehost",
"password": "******",
"user": "******",
"database": "test",
},
],
connection,
)
u = url.make_url("mssql+pymssql://scott:tiger@somehost:5000/test")
connection = dialect.create_connect_args(u)
eq_(
[
[],
{
"host": "somehost:5000",
"password": "******",
"user": "******",
"database": "test",
},
],
connection,
)
def test_database_schema_hive(self):
sqlalchemy_uri = 'hive://[email protected]:10000/default?auth=NOSASL'
model = Database(sqlalchemy_uri=sqlalchemy_uri)
db = make_url(model.get_sqla_engine().url).database
self.assertEquals('default', db)
db = make_url(model.get_sqla_engine(schema='core_db').url).database
self.assertEquals('core_db', db)
def test_database_schema_postgres(self):
sqlalchemy_uri = 'postgresql+psycopg2://postgres.airbnb.io:5439/prod'
model = Database(sqlalchemy_uri=sqlalchemy_uri)
db = make_url(model.get_sqla_engine().url).database
self.assertEquals('prod', db)
db = make_url(model.get_sqla_engine(schema='foo').url).database
self.assertEquals('prod', db)
def test_database_schema_mysql(self):
sqlalchemy_uri = 'mysql://root@localhost/superset'
model = Database(sqlalchemy_uri=sqlalchemy_uri)
db = make_url(model.get_sqla_engine().url).database
self.assertEquals('superset', db)
db = make_url(model.get_sqla_engine(schema='staging').url).database
self.assertEquals('staging', db)
def get_absolute_uri_for_session(session):
engine = session.get_bind()
url_ = engine.url
if url_.drivername.startswith('sqlite'):
database_path = url_.database
abs_database_path = os.path.abspath(database_path)
abs_database_path = '/'.join(abs_database_path.split(os.sep))
return make_url("sqlite:///%s" % abs_database_path)
else:
return make_url(url_)
def test_database_impersonate_user(self):
uri = 'mysql://root@localhost'
example_user = '******'
model = Database(sqlalchemy_uri=uri)
model.impersonate_user = True
user_name = make_url(model.get_sqla_engine(user_name=example_user).url).username
self.assertEquals(example_user, user_name)
model.impersonate_user = False
user_name = make_url(model.get_sqla_engine(user_name=example_user).url).username
self.assertNotEquals(example_user, user_name)
def make_connection_manager(self, path):
if isinstance(path, ConnectionManager):
return path
try:
path = str(path)
if path == "":
return SQLiteConnectionManager("sqlite://")
# Ensure that the path is in fact a URI
make_url(path)
return ConnectionManager(path)
except:
return self.connection_manager_type(path)
def test_plugin_multiple_url_registration(self):
from sqlalchemy.dialects import sqlite
global MyEnginePlugin1
global MyEnginePlugin2
def side_effect_1(url, kw):
eq_(kw, {"logging_name": "foob"})
kw["logging_name"] = "bar"
url.query.pop("myplugin1_arg", None)
return MyEnginePlugin1
def side_effect_2(url, kw):
url.query.pop("myplugin2_arg", None)
return MyEnginePlugin2
MyEnginePlugin1 = Mock(side_effect=side_effect_1)
MyEnginePlugin2 = Mock(side_effect=side_effect_2)
plugins.register("engineplugin1", __name__, "MyEnginePlugin1")
plugins.register("engineplugin2", __name__, "MyEnginePlugin2")
e = create_engine(
"sqlite:///?plugin=engineplugin1&foo=bar&myplugin1_arg=bat"
"&plugin=engineplugin2&myplugin2_arg=hoho",
logging_name="foob",
)
eq_(e.dialect.name, "sqlite")
eq_(e.logging_name, "bar")
# plugin args are removed from URL.
eq_(e.url.query, {"foo": "bar"})
assert isinstance(e.dialect, sqlite.dialect)
eq_(
MyEnginePlugin1.mock_calls,
[
call(url.make_url("sqlite:///?foo=bar"), {}),
call.handle_dialect_kwargs(sqlite.dialect, mock.ANY),
call.handle_pool_kwargs(mock.ANY, {"dialect": e.dialect}),
call.engine_created(e),
],
)
eq_(
MyEnginePlugin2.mock_calls,
[
call(url.make_url("sqlite:///?foo=bar"), {}),
call.handle_dialect_kwargs(sqlite.dialect, mock.ANY),
call.handle_pool_kwargs(mock.ANY, {"dialect": e.dialect}),
call.engine_created(e),
],
)
def _assert_db_exists(engine):
db_url = make_url(engine.url)
db_name = db_url.database
try:
logger.debug('try to connect to database "%s": %s', db_name, db_url)
connection = engine.connect()
connection.close()
except sqlalchemy.exc.OperationalError as err:
db_url = make_url(engine.url)
db_name = db_url.database
logger.error('could not connect to database "%s": %s', db_name, str(err))
raise ValueError('Cannot connect to database "%s".' % db_name)
def test_query_string(self):
u = url.make_url("dialect://*****:*****@host/db?arg1=param1&arg2=param2")
eq_(u.query, {"arg1": "param1", "arg2": "param2"})
eq_(str(u), "dialect://*****:*****@host/db?arg1=param1&arg2=param2")
u = url.make_url(
"dialect://*****:*****@host/db?arg1=param1&arg2=param2&arg2=param3"
)
eq_(u.query, {"arg1": "param1", "arg2": ["param2", "param3"]})
eq_(
str(u),
"dialect://*****:*****@host/db?arg1=param1&arg2=param2&arg2=param3",
)
def check_db_url(db_url):
"""Check for the correct format of the DB URL. Return none if check
passed. Raise exception otherwise.
:param db_url: The DB URL to be checked
:raise exc.ArPrDataBadDBURLFormatException:
:rtype : None
"""
try:
make_url(db_url)
except Exception:
raise exc.ArPrDataBadDBURLFormatException(
"Incorrect DB URL Format. Check http://docs.sqlalchemy.org"
"/en/latest/core/engines.html?highlight=url#database-urls")
def _url_from_target(self, target):
if isinstance(target, Connectable):
return target.engine.url
elif isinstance(target, six.string_types):
if "://" not in target:
target_url = sa_url.make_url("%s://" % target)
else:
target_url = sa_url.make_url(target)
return target_url
elif isinstance(target, sa_url.URL):
return target
else:
raise ValueError("Invalid target type: %r" % target)
def get_sqla_engine(self, schema=None, nullpool=True, user_name=None):
extra = self.get_extra()
url = make_url(self.sqlalchemy_uri_decrypted)
url = self.db_engine_spec.adjust_database_uri(url, schema)
effective_username = self.get_effective_user(url, user_name)
# If using MySQL or Presto for example, will set url.username
# If using Hive, will not do anything yet since that relies on a
# configuration parameter instead.
self.db_engine_spec.modify_url_for_impersonation(
url,
self.impersonate_user,
effective_username)
masked_url = self.get_password_masked_url(url)
logging.info('Database.get_sqla_engine(). Masked URL: {0}'.format(masked_url))
params = extra.get('engine_params', {})
if nullpool:
params['poolclass'] = NullPool
# If using Hive, this will set hive.server2.proxy.user=$effective_username
configuration = {}
configuration.update(
self.db_engine_spec.get_configuration_for_impersonation(
str(url),
self.impersonate_user,
effective_username))
if configuration:
params['connect_args'] = {'configuration': configuration}
DB_CONNECTION_MUTATOR = config.get('DB_CONNECTION_MUTATOR')
if DB_CONNECTION_MUTATOR:
url, params = DB_CONNECTION_MUTATOR(
url, params, effective_username, security_manager)
return create_engine(url, **params)
def test_engine_from_config(self):
dbapi = mock_dbapi
config = \
{'sqlalchemy.url': 'postgresql://*****:*****@somehost/test'\
'?fooz=somevalue', 'sqlalchemy.pool_recycle': '50',
'sqlalchemy.echo': 'true'}
e = engine_from_config(config, module=dbapi, _initialize=False)
assert e.pool._recycle == 50
assert e.url \
== url.make_url('postgresql://*****:*****@somehost/test?foo'
'z=somevalue')
assert e.echo is True
for param, values in [
('convert_unicode', ('true', 'false', 'force')),
('echo', ('true', 'false', 'debug')),
('echo_pool', ('true', 'false', 'debug')),
('use_native_unicode', ('true', 'false')),
]:
for value in values:
config = {
'sqlalchemy.url': 'postgresql://*****:*****@somehost/test',
'sqlalchemy.%s' % param : value
}
cfg = _coerce_config(config, 'sqlalchemy.')
assert cfg[param] == {'true':True, 'false':False}.get(value, value)
def __init__(self, url):
self.table = Table(self.__tablename__, MetaData(),
Column('name', String(64)),
Column('group', String(64)),
Column('status', String(16)),
Column('script', Text),
Column('comments', String(1024)),
Column('rate', Float(11)),
Column('burst', Float(11)),
Column('updatetime', Float(32)),
mysql_engine='InnoDB',
mysql_charset='utf8'
)
self.url = make_url(url)
if self.url.database:
database = self.url.database
self.url.database = None
try:
engine = create_engine(self.url)
conn = engine.connect()
conn.execute("commit")
conn.execute("CREATE DATABASE %s" % database)
except sqlalchemy.exc.SQLAlchemyError:
pass
self.url.database = database
self.engine = create_engine(url)
self.table.create(self.engine, checkfirst=True)
def test_sqlite_abspath(self):
u = url.make_url("sqlite:////x/state.sqlite")
kwargs = dict(basedir='/my-base-dir')
u, kwargs, max_conns = self.strat.special_case_sqlite(u, kwargs)
self.assertEqual([ str(u), max_conns, self.filter_kwargs(kwargs) ],
[ "sqlite:////x/state.sqlite", None,
dict(basedir='/my-base-dir') ])
def _read_connection_has_correct_privileges(self):
''' Returns True if the right permissions are set for the read
only user. A table is created by the write user to test the
read only user.
'''
write_connection = db._get_engine(
{'connection_url': self.write_url}).connect()
read_connection_user = sa_url.make_url(self.read_url).username
drop_foo_sql = u'DROP TABLE IF EXISTS _foo'
write_connection.execute(drop_foo_sql)
try:
write_connection.execute(u'CREATE TEMP TABLE _foo ()')
for privilege in ['INSERT', 'UPDATE', 'DELETE']:
test_privilege_sql = u"SELECT has_table_privilege(%s, '_foo', %s)"
have_privilege = write_connection.execute(
test_privilege_sql, (read_connection_user, privilege)).first()[0]
if have_privilege:
return False
finally:
write_connection.execute(drop_foo_sql)
write_connection.close()
return True
def test_rfc1738(self):
for text in (
'dbtype://*****:*****@hostspec:110//usr/db_file.db',
'dbtype://*****:*****@hostspec/database',
'dbtype://*****:*****@hostspec',
'dbtype://*****:*****@/database',
'dbtype://username@hostspec',
'dbtype://*****:*****@127.0.0.1:1521',
'dbtype://hostspec/database',
'dbtype://hostspec',
'dbtype://hostspec/?arg1=val1&arg2=val2',
'dbtype:///database',
'dbtype:///:memory:',
'dbtype:///foo/bar/im/a/file',
'dbtype:///E:/work/src/LEM/db/hello.db',
'dbtype:///E:/work/src/LEM/db/hello.db?foo=bar&hoho=lala',
'dbtype://',
'dbtype://*****:*****@/db',
'dbtype:////usr/local/mailman/lists/[email protected]/memb'
'ers.db',
'dbtype://*****:*****@hostspec/mydatabase',
):
u = url.make_url(text)
assert u.drivername == 'dbtype'
assert u.username == 'username' or u.username is None
assert u.password == 'password' or u.password \
== 'apples/oranges' or u.password is None
assert u.host == 'hostspec' or u.host == '127.0.0.1' \
or not u.host
assert str(u) == text
def dress_url(url):
# If no explicit driver has been set, we default to pymysql
if url.startswith("mysql://"):
url = sqlalchemy_url.make_url(url)
url.drivername = "mysql+pymysql"
return str(url)
return url
def setup_parser(self, parser):
parser.formatter_class = RawDescriptionHelpFormatter
if parser.description is None:
parser.description = ''
if self.default_url:
parser.description += (
'\nThe database to be acted on is at:\n%r' % (
make_url(self.default_url)
))
parser.description += (
'\n\nThe following tables are in the current configuration:\n' +
', '.join(self.config.tables)
)
parser.add_argument(
'--url',
default = self.default_url,
help='Override the database url used.'
)
commands = parser.add_subparsers()
for name in dir(self.__class__):
if name[0] == '_':
continue
doc = getattr(self, name).__doc__
if doc is None:
continue
doc = doc.strip()
command = commands.add_parser(
name,
help=doc
)
command.set_defaults(method=getattr(self,name))
def create(self, name_or_url, **kwargs):
if 'basedir' not in kwargs:
raise TypeError('no basedir supplied to create_engine')
max_conns = None
# apply special cases
u = url.make_url(name_or_url)
if u.drivername.startswith('sqlite'):
u, kwargs, max_conns = self.special_case_sqlite(u, kwargs)
elif u.drivername.startswith('mysql'):
u, kwargs, max_conns = self.special_case_mysql(u, kwargs)
# remove the basedir as it may confuse sqlalchemy
basedir = kwargs.pop('basedir')
# calculate the maximum number of connections from the pool parameters,
# if it hasn't already been specified
if max_conns is None:
max_conns = kwargs.get('pool_size', 5) + kwargs.get('max_overflow', 10)
engine = strategies.ThreadLocalEngineStrategy.create(self,
u, **kwargs)
# annotate the engine with the optimal thread pool size; this is used
# by DBConnector to configure the surrounding thread pool
engine.optimal_thread_pool_size = max_conns
# and keep the basedir
engine.buildbot_basedir = basedir
return engine
def __init__(self, url):
self.table = Table('__tablename__', MetaData(),
Column('taskid', String(64), primary_key=True, nullable=False),
Column('project', String(64)),
Column('url', String(1024)),
Column('status', Integer),
Column('schedule', LargeBinary),
Column('fetch', LargeBinary),
Column('process', LargeBinary),
Column('track', LargeBinary),
Column('lastcrawltime', Float(32)),
Column('updatetime', Float(32)),
mysql_engine='InnoDB',
mysql_charset='utf8'
)
self.url = make_url(url)
if self.url.database:
database = self.url.database
self.url.database = None
try:
engine = create_engine(self.url, convert_unicode=True)
engine.execute("CREATE DATABASE IF NOT EXISTS %s" % database)
except sqlalchemy.exc.SQLAlchemyError:
pass
self.url.database = database
self.engine = create_engine(self.url, convert_unicode=True)
self._list_project()
def test_mysql_simple(self):
u = url.make_url("mysql://host/dbname")
kwargs = dict(basedir='my-base-dir')
u, kwargs, max_conns = self.strat.special_case_mysql(u, kwargs)
self.assertEqual([str(u), max_conns, self.filter_kwargs(kwargs)],
["mysql://host/dbname?charset=utf8&use_unicode=True", None,
self.mysql_kwargs])
def create_database(url, encoding='utf8', template=None):
"""Issue the appropriate CREATE DATABASE statement.
:param url: A SQLAlchemy engine URL.
:param encoding: The encoding to create the database as.
:param template:
The name of the template from which to create the new database. At the
moment only supported by PostgreSQL driver.
To create a database, you can pass a simple URL that would have
been passed to ``create_engine``. ::
create_database('postgres://postgres@localhost/name')
You may also pass the url from an existing engine. ::
create_database(engine.url)
Has full support for mysql, postgres, and sqlite. In theory,
other database engines should be supported.
"""
url = copy(make_url(url))
database = url.database
if url.drivername.startswith('postgresql'):
url.database = 'template1'
elif not url.drivername.startswith('sqlite'):
url.database = None
engine = sa.create_engine(url)
if engine.dialect.name == 'postgresql':
if engine.driver == 'psycopg2':
from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
engine.raw_connection().set_isolation_level(
ISOLATION_LEVEL_AUTOCOMMIT
)
if not template:
template = 'template0'
text = "CREATE DATABASE {0} ENCODING '{1}' TEMPLATE {2}".format(
quote(engine, database),
encoding,
quote(engine, template)
)
engine.execute(text)
elif engine.dialect.name == 'mysql':
text = "CREATE DATABASE {0} CHARACTER SET = '{1}'".format(
quote(engine, database),
encoding
)
engine.execute(text)
elif engine.dialect.name == 'sqlite' and database != ':memory:':
if database:
open(database, 'w').close()
else:
text = 'CREATE DATABASE {0}'.format(quote(engine, database))
engine.execute(text)
def _get_db_from_url(self, url):
db_url = sa_url.make_url(url)
return db_url.host, db_url.port, db_url.database
def backend(self) -> str:
sqlalchemy_url = make_url(self.sqlalchemy_uri_decrypted)
return sqlalchemy_url.get_backend_name() # pylint: disable=no-member
def backend(self):
url = make_url(self.sqlalchemy_uri_decrypted)
return url.get_backend_name()
def test_mysql_mysqldb_connect_args_default(self):
session._init_connection_args(
url.make_url("mysql+mysqldb://u:p@host/test"), self.args)
self.assertEqual(self.args['connect_args'],
{'charset': 'utf8', 'use_unicode': 0})
def _sqlite_follower_url_from_main(url, ident):
url = sa_url.make_url(url)
if not url.database or url.database == ':memory:':
return url
else:
return sa_url.make_url("sqlite:///%s.db" % ident)
def validate_password(self, data: Dict[str, Any], **kwargs: Any) -> None:
"""If sqlalchemy_uri has a masked password, password is required"""
uri = data["sqlalchemy_uri"]
password = make_url(uri).password
if password == PASSWORD_MASK and data.get("password") is None:
raise ValidationError("Must provide a password for the database")
def __init__(self, db_url, database):
self.url = make_url(db_url)
self.database = database
if not database:
self._handle_no_database()
def create_app(config='CTFd.config.Config'):
app = Flask(__name__)
with app.app_context():
app.config.from_object(config)
app.jinja_loader = ThemeLoader(os.path.join(app.root_path, app.template_folder), followlinks=True)
from CTFd.models import db, Teams, Solves, Challenges, WrongKeys, Keys, Tags, Files, Tracking
url = make_url(app.config['SQLALCHEMY_DATABASE_URI'])
if url.drivername == 'postgres':
url.drivername = 'postgresql'
# Creates database if the database database does not exist
if not database_exists(url):
create_database(url)
# Register database
db.init_app(app)
# Register Flask-Migrate
migrate.init_app(app, db)
# This creates tables instead of db.create_all()
# Allows migrations to happen properly
migrate_upgrade()
# Alembic sqlite support is lacking so we should just create_all anyway
if url.drivername.startswith('sqlite'):
db.create_all()
app.db = db
cache.init_app(app)
app.cache = cache
version = utils.get_config('ctf_version')
if not version: # Upgrading from an unversioned CTFd
utils.set_config('ctf_version', __version__)
if version and (StrictVersion(version) < StrictVersion(__version__)): # Upgrading from an older version of CTFd
print("/*\\ CTFd has updated and must update the database! /*\\")
print("/*\\ Please backup your database before proceeding! /*\\")
print("/*\\ CTFd maintainers are not responsible for any data loss! /*\\")
if input('Run database migrations (Y/N)').lower().strip() == 'y':
migrate_stamp()
migrate_upgrade()
utils.set_config('ctf_version', __version__)
else:
print('/*\\ Ignored database migrations... /*\\')
exit()
if not utils.get_config('ctf_theme'):
utils.set_config('ctf_theme', 'original')
from CTFd.views import views
from CTFd.challenges import challenges
from CTFd.scoreboard import scoreboard
from CTFd.auth import auth
from CTFd.admin import admin, admin_statistics, admin_challenges, admin_pages, admin_scoreboard, admin_containers, admin_keys, admin_teams
from CTFd.utils import init_utils, init_errors, init_logs
init_utils(app)
init_errors(app)
init_logs(app)
app.register_blueprint(views)
app.register_blueprint(challenges)
app.register_blueprint(scoreboard)
app.register_blueprint(auth)
app.register_blueprint(admin)
app.register_blueprint(admin_statistics)
app.register_blueprint(admin_challenges)
app.register_blueprint(admin_teams)
app.register_blueprint(admin_scoreboard)
app.register_blueprint(admin_keys)
app.register_blueprint(admin_containers)
app.register_blueprint(admin_pages)
from CTFd.plugins import init_plugins
init_plugins(app)
return app
def create(self, name_or_url, **kwargs):
# create url.URL object
u = url.make_url(name_or_url)
dialect_cls = u.get_dialect()
if kwargs.pop('_coerce_config', False):
def pop_kwarg(key, default=None):
value = kwargs.pop(key, default)
if key in dialect_cls.engine_config_types:
value = dialect_cls.engine_config_types[key](value)
return value
else:
pop_kwarg = kwargs.pop
dialect_args = {}
# consume dialect arguments from kwargs
for k in util.get_cls_kwargs(dialect_cls):
if k in kwargs:
dialect_args[k] = pop_kwarg(k)
dbapi = kwargs.pop('module', None)
if dbapi is None:
dbapi_args = {}
for k in util.get_func_kwargs(dialect_cls.dbapi):
if k in kwargs:
dbapi_args[k] = pop_kwarg(k)
dbapi = dialect_cls.dbapi(**dbapi_args)
dialect_args['dbapi'] = dbapi
# create dialect
dialect = dialect_cls(**dialect_args)
# assemble connection arguments
(cargs, cparams) = dialect.create_connect_args(u)
cparams.update(pop_kwarg('connect_args', {}))
# look for existing pool or create
pool = pop_kwarg('pool', None)
if pool is None:
def connect():
return dialect.connect(*cargs, **cparams)
creator = pop_kwarg('creator', connect)
poolclass = pop_kwarg('poolclass', None)
if poolclass is None:
poolclass = dialect_cls.get_pool_class(u)
pool_args = {}
# consume pool arguments from kwargs, translating a few of
# the arguments
translate = {'logging_name': 'pool_logging_name',
'echo': 'echo_pool',
'timeout': 'pool_timeout',
'recycle': 'pool_recycle',
'events': 'pool_events',
'use_threadlocal': 'pool_threadlocal',
'reset_on_return': 'pool_reset_on_return'}
for k in util.get_cls_kwargs(poolclass):
tk = translate.get(k, k)
if tk in kwargs:
pool_args[k] = pop_kwarg(tk)
pool = poolclass(creator, **pool_args)
else:
if isinstance(pool, poollib._DBProxy):
pool = pool.get_pool(*cargs, **cparams)
else:
pool = pool
# create engine.
engineclass = self.engine_cls
engine_args = {}
for k in util.get_cls_kwargs(engineclass):
if k in kwargs:
engine_args[k] = pop_kwarg(k)
_initialize = kwargs.pop('_initialize', True)
# all kwargs should be consumed
if kwargs:
raise TypeError(
"Invalid argument(s) %s sent to create_engine(), "
"using configuration %s/%s/%s. Please check that the "
"keyword arguments are appropriate for this combination "
"of components." % (','.join("'%s'" % k for k in kwargs),
dialect.__class__.__name__,
pool.__class__.__name__,
engineclass.__name__))
engine = engineclass(pool, dialect, u, **engine_args)
if _initialize:
do_on_connect = dialect.on_connect()
if do_on_connect:
def on_connect(dbapi_connection, connection_record):
conn = getattr(
dbapi_connection, '_sqla_unwrap', dbapi_connection)
if conn is None:
return
do_on_connect(conn)
event.listen(pool, 'first_connect', on_connect)
event.listen(pool, 'connect', on_connect)
def first_connect(dbapi_connection, connection_record):
c = base.Connection(engine, connection=dbapi_connection,
_has_events=False)
c._execution_options = util.immutabledict()
dialect.initialize(c)
event.listen(pool, 'first_connect', first_connect, once=True)
return engine
def obfuscate_url_pw(u):
u = url.make_url(u)
if u.password:
u.password = "******"
return str(u)
def get_dialect(self):
sqla_url = url.make_url(self.sqlalchemy_uri_decrypted)
return sqla_url.get_dialect()()
def get_sqla_engine(self, schema=None):
extra = self.get_extra()
uri = make_url(self.sqlalchemy_uri_decrypted)
params = extra.get('engine_params', {})
uri = self.db_engine_spec.adjust_database_uri(uri, schema)
return create_engine(uri, **params)
def provisioned_database_url(self, base_url, ident):
if base_url.database:
return sa_url.make_url("sqlite:////tmp/%s.db" % ident)
else:
return base_url
def _oracle_follower_url_from_main(url, ident):
url = sa_url.make_url(url)
url.username = ident
url.password = '******'
return url
def main():
parser = argparse.ArgumentParser(description="Qserv database schema migration.")
parser.add_argument("-v", "--verbose", default=0, action="count",
help="Use one -v for INFO logging, two for DEBUG.")
group = parser.add_mutually_exclusive_group()
group.add_argument("-m", "--do-migrate", default=False, action="store_true",
help="Do migration, without this option script prints various info "
"and exits.")
group.add_argument("--check", default=False, action="store_true",
help="Check that migration is needed, script returns 0 if schema is "
"up-to-date, 1 otherwise.")
parser.add_argument("-n", "--final", default=None, action="store", type=int,
metavar="VERSION",
help="Stop migration at given version, by default update to "
"latest version.")
parser.add_argument("--scripts", default=_def_scripts, action="store",
metavar="PATH",
help="Location for migration scripts, def: %(default)s.")
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument("-c", "--connection", metavar="CONNECTION",
help="Connection string in format mysql://user:pass@host:port/database.")
group.add_argument("-f", "--config-file", metavar="PATH",
help="Name of configuration file in INI format with connection parameters.")
parser.add_argument("-s", "--config-section", metavar="NAME",
help="Name of configuration section in configuration file.")
parser.add_argument("module",
help="Name of Qserv module for which to update schema, e.g. qmeta.")
args = parser.parse_args()
# configure logging
levels = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG}
level = levels.get(args.verbose, logging.DEBUG)
fmt = "%(asctime)s [%(levelname)s] %(name)s: %(message)s"
logging.basicConfig(level=level, format=fmt)
if args.connection:
url = make_url(args.connection)
engine = sqlalchemy.create_engine(url)
elif args.config_file:
if not args.config_section:
parser.error("-s options required with -f")
cfg = configparser.SafeConfigParser()
if not cfg.read([args.config_file]):
# file was not found, generate exception which should happen
# if we tried to open that file
raise IOError(2, "No such file or directory: '{}'".format(args.config_file))
# will throw is section is missing
config = dict(cfg.items(args.config_section))
# instantiate database engine
config = _normalizeConfig(config)
engine = engineFactory.getEngineFromArgs(**config)
# make an object which will manage migration process
mgr = _load_migration_mgr(args.module, engine=engine, scripts_dir=args.scripts)
current = mgr.current_version()
print("Current schema version: {}".format(current))
latest = mgr.latest_version()
print("Latest schema version: {}".format(latest))
migrations = mgr.migrations()
print("Known migrations:")
for v0, v1, script in migrations:
tag = " (X)" if v0 >= current else ""
print(" {} -> {} : {}{}".format(v0, v1, script, tag))
if args.check:
return 0 if mgr.current_version() == mgr.latest_version() else 1
# do the migrations
final = mgr.migrate(args.final, args.do_migrate)
if final is None:
print("No migration was needed")
else:
if args.do_migrate:
print("Database was migrated to version {}".format(final))
else:
print("Database would be migrated to version {}".format(final))
def _follower_url_from_main(url, ident):
url = sa_url.make_url(url)
url.database = ident
return url
os.environ.get('GEOSERVER_ENDPOINT',
'http://*****:*****@172.17.0.1:5432/tds_tests')
_url_object = make_url(_URL)
_PUBLIC_URL = os.environ.get(
'POSTGIS_PUBLIC_URL',
'postgresql://*****:*****@localhost:5432/tds_tests')
_public_url_object = make_url(_PUBLIC_URL)
TEST_POSTGIS_SERVICE = {
'URL': _URL,
'DRIVER': _url_object.drivername,
'USERNAME': _url_object.username,
'PASSWORD': _url_object.password,
'HOST': _url_object.host,
'PORT': _url_object.port,
'DATABASE': _url_object.database,
'PUBLIC_URL': _PUBLIC_URL,
'PUBLIC_HOST': _public_url_object.host,
def test_psycopg2_nonempty_connection_string_w_query(self):
dialect = psycopg2_dialect.dialect()
u = url.make_url("postgresql://somehost/?any_random_thing=yes")
cargs, cparams = dialect.create_connect_args(u)
eq_(cargs, [])
eq_(cparams, {"host": "somehost", "any_random_thing": "yes"})
def test_queuepool_args(self):
session._init_connection_args(
url.make_url("mysql://*****:*****@host/test"), self.args,
max_pool_size=10, max_overflow=10)
self.assertEqual(self.args['pool_size'], 10)
self.assertEqual(self.args['max_overflow'], 10)
def test_psycopg2_nonempty_connection_string(self):
dialect = psycopg2_dialect.dialect()
u = url.make_url("postgresql://host")
cargs, cparams = dialect.create_connect_args(u)
eq_(cargs, [])
eq_(cparams, {"host": "host"})
def test_psycopg2_empty_connection_string_w_query_one(self):
dialect = psycopg2_dialect.dialect()
u = url.make_url("postgresql:///?service=swh-log")
cargs, cparams = dialect.create_connect_args(u)
eq_(cargs, [])
eq_(cparams, {"service": "swh-log"})
def extract_active_data(trans, url, dump_location="dump.sql"):
"""
Stores sqldump data in the specified location. If not specified, stores it in current directory in file dump.sql
If file already exists it will override that file and not append it.
Function added in to enhance testing in Balrog's stage environment.
:param trans: Transaction Object for an SQL connection
:param url: Database. eg : mysql://balrogadmin:balrogadmin@balrogdb/balrog
:param dump_location: location where sqldump file must be created
"""
url = make_url(url)
user = url.username
password = url.password
host = url.host
db = url.database
# Extract the entire database schema, without any rows.
# This is done to ensure that any database dump generated can be
# imported to an empty database without issue. From there, a Balrog
# installation can be upgraded or downgraded to a different database
# schema version if desired.
# See https://bugzilla.mozilla.org/show_bug.cgi?id=1376331 for additional
# background on this.
with open(dump_location, "w+") as dump_file:
run(mysql_command(host, user, password, db, "--no-data").split(), stdout=dump_file, check=True)
# Now extract the data we actually want....
# We always want all the data from a few tables...
run(mysql_data_only_command(host, user, password, db, "dockerflow rules rules_history migrate_version").split(), stdout=dump_file, check=True)
# Because Releases are so massive, we only want the actively used ones,
# and very little Release history. Specifically:
# - All releases referenced by a Rule or a Active Scheduled Rule Change
# - All releases referenced by a Release from the above query
# - 50 rows of history for the "Firefox-mozilla-central-nightly-latest" Release
# - Full history for the Release currently referenced by the "firefox-release" Rule.
query_release_mapping = """SELECT DISTINCT releases.* \
FROM releases, rules, rules_scheduled_changes \
WHERE (releases.name IN (rules.mapping, rules.fallbackMapping))
OR (rules_scheduled_changes.complete = 0 AND
releases.name IN (rules_scheduled_changes.base_mapping, rules_scheduled_changes.base_fallbackMapping))
"""
result = trans.execute(query_release_mapping).fetchall()
release_names = set()
for row in result:
try:
release_names.add(str(row["name"]))
release_blob = createBlob(row["data"])
release_names.update(release_blob.getReferencedReleases())
except ValueError:
continue
if release_names:
batch_generator = chunk_list(list(release_names), 30)
for batched_release_list in batch_generator:
query = ", ".join("'" + names + "'" for names in batched_release_list)
cmd = mysql_data_only_command(host, user, password, db, "releases").split()
cmd.append('--where="releases.name IN ({})"'.format(query))
run(cmd, stdout=dump_file, check=True)
cmd = mysql_data_only_command(host, user, password, db, "releases_history").split()
cmd.append("--where=\"release_history.name='Firefox-mozilla-central-nightly-latest' ORDER BY timestamp DESC LIMIT 50\"")
run(cmd, stdout=dump_file, check=True)
query = "SELECT rules.mapping FROM rules WHERE rules.alias='firefox-release'"
cmd = mysql_data_only_command(host, user, password, db, "releases_history").split()
cmd.append('--where="name = ({}) ORDER BY timestamp DESC LIMIT 50"'.format(query))
run(cmd, stdout=dump_file, check=True)
def test_random_arg(self):
dialect = testing.db.dialect
kw = dialect.create_connect_args(
make_url("mysql://*****:*****@host/db?foo=true"))[1]
eq_(kw["foo"], "true")
def create(self, name_or_url, **kwargs):
# create url.URL object
u = url.make_url(name_or_url)
dialect_cls = u.get_dialect()
dialect_args = {}
# consume dialect arguments from kwargs
for k in util.get_cls_kwargs(dialect_cls):
if k in kwargs:
dialect_args[k] = kwargs.pop(k)
dbapi = kwargs.pop('module', None)
if dbapi is None:
dbapi_args = {}
for k in util.get_func_kwargs(dialect_cls.dbapi):
if k in kwargs:
dbapi_args[k] = kwargs.pop(k)
dbapi = dialect_cls.dbapi(**dbapi_args)
dialect_args['dbapi'] = dbapi
# create dialect
dialect = dialect_cls(**dialect_args)
# assemble connection arguments
(cargs, cparams) = dialect.create_connect_args(u)
cparams.update(kwargs.pop('connect_args', {}))
# look for existing pool or create
pool = kwargs.pop('pool', None)
if pool is None:
def connect():
try:
return dialect.connect(*cargs, **cparams)
except dialect.dbapi.Error, e:
invalidated = dialect.is_disconnect(e, None, None)
# Py3K
#raise exc.DBAPIError.instance(None, None,
# e, dialect.dbapi.Error,
# connection_invalidated=invalidated
#) from e
# Py2K
import sys
raise exc.DBAPIError.instance(
None,
None,
e,
dialect.dbapi.Error,
connection_invalidated=invalidated
), None, sys.exc_info()[2]
# end Py2K
creator = kwargs.pop('creator', connect)
poolclass = kwargs.pop('poolclass', None)
if poolclass is None:
poolclass = dialect_cls.get_pool_class(u)
pool_args = {}
# consume pool arguments from kwargs, translating a few of
# the arguments
translate = {
'logging_name': 'pool_logging_name',
'echo': 'echo_pool',
'timeout': 'pool_timeout',
'recycle': 'pool_recycle',
'events': 'pool_events',
'use_threadlocal': 'pool_threadlocal',
'reset_on_return': 'pool_reset_on_return'
}
for k in util.get_cls_kwargs(poolclass):
tk = translate.get(k, k)
if tk in kwargs:
pool_args[k] = kwargs.pop(tk)
pool = poolclass(creator, **pool_args)
def get_password_masked_url_from_uri(cls, uri):
url = make_url(uri)
return cls.get_password_masked_url(url)
def get_dialect(self) -> Dialect:
sqla_url = url.make_url(self.sqlalchemy_uri_decrypted)
return sqla_url.get_dialect()() # pylint: disable=no-member
def get_password_masked_url_from_uri( # pylint: disable=invalid-name
cls, uri: str) -> URL:
sqlalchemy_url = make_url(uri)
return cls.get_password_masked_url(sqlalchemy_url)
def test_postgresql_connect_args_default(self):
session._init_connection_args(
url.make_url("postgresql://*****:*****@host/test"), self.args)
self.assertEqual(self.args['client_encoding'], 'utf8')
self.assertFalse(self.args['connect_args'])
def url_object(self):
return make_url(self.sqlalchemy_uri_decrypted)
