Exemple #1
0
def all_to_intermediary(filename_or_input, schema=None):
    """ Dispatch the filename_or_input to the different function to produce the intermediary syntax.
    All the supported classes names are in `swich_input_class_to_method`.
    The input can also be a list of strings in markdown format or a filename finishing by '.er' containing markdown
    format.
    """
    # Try to convert from the name of the class
    input_class_name = filename_or_input.__class__.__name__
    try:
        this_to_intermediary = switch_input_class_to_method[input_class_name]
        tables, relationships = this_to_intermediary(filename_or_input)
        return tables, relationships
    except KeyError:
        pass

    # try to read markdown file.
    if isinstance(filename_or_input, basestring):
        if filename_or_input.split('.')[-1] == 'er':
            return markdown_file_to_intermediary(filename_or_input)

    # try to read a markdown in a string
    if not isinstance(filename_or_input, basestring):
        if all(isinstance(e, basestring) for e in filename_or_input):
            return line_iterator_to_intermediary(filename_or_input)

    # try to read DB URI.
    try:
        make_url(filename_or_input)
        return database_to_intermediary(filename_or_input, schema=schema)
    except ArgumentError:
        pass

    msg = 'Cannot process filename_or_input {}'.format(input_class_name)
    raise ValueError(msg)
    def test_comparison(self):
        components = (
            "drivername",
            "username",
            "password",
            "host",
            "database",
            "query",
            "port",
        )

        common_url = (
            "dbtype://*****:*****@[2001:da8:2004:1000:202:116:160:90]:80/database?foo=bar"
        )
        other_url = "dbtype://*****:*****@host/"

        url1 = url.make_url(common_url)
        url2 = url.make_url(common_url)
        url3 = url.make_url(other_url)

        is_true(url1 == url2)
        is_false(url1 != url2)
        is_true(url1 != url3)
        is_false(url1 == url3)

        for curr_component in components:
            setattr(url2, curr_component, "new_changed_value")
            is_true(url1 != url2)
            is_false(url1 == url2)
            setattr(url2, curr_component, getattr(url1, curr_component))
Exemple #3
0
def normalize_url(raw):
    """
    Build a URL from a string according to obvious rules.

    Converts *.sqlite paths to sqlite:/// URLs, etc.

    >>> str(normalize_url("/tmp/foo.sqlite"))
    'sqlite:////tmp/foo.sqlite'
    >>> str(normalize_url("postgresql://user@host/database"))
    'postgresql://user@host/database'
    """

    from sqlalchemy.engine.url import make_url
    from sqlalchemy.exceptions import ArgumentError

    try:
        return make_url(raw)
    except ArgumentError:
        from os.path import (
            abspath,
            splitext,
            )

        (root, extension) = splitext(raw)

        if extension == ".sqlite":
            return make_url("sqlite:///%s" % abspath(raw))
        else:
            raise ArgumentError("could not parse supposed URL \"%s\"" % raw)
Exemple #4
0
def _create_connection(db):
    if isinstance(db, str):
        url = make_url(connections.databases[db]['ENGINE'])
    elif isinstance(db, dict):
        url = make_url(db['ENGINE'])
    else:
        url = db

    db_engine = url.drivername.split('+')[0]
    database = url.database
    kwargs = {'autocommit': True}

    if db_engine == 'sqlite':
        return
    elif db_engine == 'postgresql':
        database = 'postgres'
    elif db_engine == 'mssql':
        kwargs['isolation_level'] = 'AUTOCOMMIT'
        database = 'master'
    elif db_engine == 'mysql':
        database = None
    elif db_engine == 'oracle':
        database = connections.databases[DEFAULT_DB_ALIAS].get('SYSTEM_DB', 'SYSTEM')

    url = URL(url.drivername, url.username, url.password, url.host, url.port, database, url.query)
    return create_engine(url).connect().execution_options(**kwargs)
    def test_rfc1738_password(self):
        u = url.make_url("dbtype://*****:*****@host/dbname")
        eq_(u.password, "pass word + other:words")
        eq_(str(u), "dbtype://*****:*****@host/dbname")

        u = url.make_url(
            "dbtype://*****:*****@hostspec/database"
        )
        eq_(u.password, "apples/oranges")
        eq_(str(u), "dbtype://*****:*****@hostspec/database")

        u = url.make_url(
            "dbtype://*****:*****@hostspec/database"
        )
        eq_(u.password, "apples@oranges@@")
        eq_(
            str(u),
            "dbtype://*****:*****@hostspec/database",
        )

        u = url.make_url("dbtype://username%40:@hostspec/database")
        eq_(u.password, "")
        eq_(u.username, "username@")
        eq_(str(u), "dbtype://username%40:@hostspec/database")

        u = url.make_url("dbtype://*****:*****@hostspec/database")
        eq_(u.password, "pass/word")
        eq_(str(u), "dbtype://*****:*****@hostspec/database")
    def __init__(self):

        try:

            super(SQLRepository, self).__init__()

            self.conf = cfg.CONF
            url = None
            if self.conf.mysql.database_name is not None:
                settings_db = (self.conf.mysql.username,
                               self.conf.mysql.password,
                               self.conf.mysql.hostname,
                               self.conf.mysql.database_name)
                url = make_url("mysql+pymysql://%s:%s@%s/%s" % settings_db)
            else:
                if self.conf.database.url is not None:
                    url = make_url(self.conf.database.url)
                else:
                    database_conf = dict(self.conf.database)
                    if 'url' in database_conf:
                        del database_conf['url']
                    url = URL(**database_conf)

            from sqlalchemy import create_engine
            self._db_engine = create_engine(url, pool_recycle=3600)

            self.metadata = MetaData()

        except Exception as ex:
            LOG.exception(ex)
            raise exceptions.RepositoryException(ex)
Exemple #7
0
    def configure(cls, connection=None, url=None, dialect_name=None, opts={}):
        """Create a new :class:`.MigrationContext`.

        This is a factory method usually called
        by :meth:`.EnvironmentContext.configure`.

        :param connection: a :class:`~sqlalchemy.engine.Connection`
         to use for SQL execution in "online" mode.  When present,
         is also used to determine the type of dialect in use.
        :param url: a string database url, or a
         :class:`sqlalchemy.engine.url.URL` object.
         The type of dialect to be used will be derived from this if
         ``connection`` is not passed.
        :param dialect_name: string name of a dialect, such as
         "postgresql", "mssql", etc.  The type of dialect to be used will be
         derived from this if ``connection`` and ``url`` are not passed.
        :param opts: dictionary of options.  Most other options
         accepted by :meth:`.EnvironmentContext.configure` are passed via
         this dictionary.

        """
        if connection:
            dialect = connection.dialect
        elif url:
            url = sqla_url.make_url(url)
            dialect = url.get_dialect()()
        elif dialect_name:
            url = sqla_url.make_url("%s://" % dialect_name)
            dialect = url.get_dialect()()
        else:
            raise Exception("Connection, url, or dialect_name is required.")

        return MigrationContext(dialect, connection, opts)
Exemple #8
0
    def test_pymssql_port_setting(self):
        dialect = pymssql.dialect()

        u = url.make_url("mssql+pymssql://scott:tiger@somehost/test")
        connection = dialect.create_connect_args(u)
        eq_(
            [
                [],
                {
                    "host": "somehost",
                    "password": "******",
                    "user": "******",
                    "database": "test",
                },
            ],
            connection,
        )

        u = url.make_url("mssql+pymssql://scott:tiger@somehost:5000/test")
        connection = dialect.create_connect_args(u)
        eq_(
            [
                [],
                {
                    "host": "somehost:5000",
                    "password": "******",
                    "user": "******",
                    "database": "test",
                },
            ],
            connection,
        )
    def test_database_schema_hive(self):
        sqlalchemy_uri = 'hive://[email protected]:10000/default?auth=NOSASL'
        model = Database(sqlalchemy_uri=sqlalchemy_uri)
        db = make_url(model.get_sqla_engine().url).database
        self.assertEquals('default', db)

        db = make_url(model.get_sqla_engine(schema='core_db').url).database
        self.assertEquals('core_db', db)
Exemple #10
0
    def test_database_schema_postgres(self):
        sqlalchemy_uri = 'postgresql+psycopg2://postgres.airbnb.io:5439/prod'
        model = Database(sqlalchemy_uri=sqlalchemy_uri)

        db = make_url(model.get_sqla_engine().url).database
        self.assertEquals('prod', db)

        db = make_url(model.get_sqla_engine(schema='foo').url).database
        self.assertEquals('prod', db)
Exemple #11
0
    def test_database_schema_mysql(self):
        sqlalchemy_uri = 'mysql://root@localhost/superset'
        model = Database(sqlalchemy_uri=sqlalchemy_uri)

        db = make_url(model.get_sqla_engine().url).database
        self.assertEquals('superset', db)

        db = make_url(model.get_sqla_engine(schema='staging').url).database
        self.assertEquals('staging', db)
def get_absolute_uri_for_session(session):
    engine = session.get_bind()
    url_ = engine.url
    if url_.drivername.startswith('sqlite'):
        database_path = url_.database
        abs_database_path = os.path.abspath(database_path)
        abs_database_path = '/'.join(abs_database_path.split(os.sep))
        return make_url("sqlite:///%s" % abs_database_path)
    else:
        return make_url(url_)
Exemple #13
0
    def test_database_impersonate_user(self):
        uri = 'mysql://root@localhost'
        example_user = '******'
        model = Database(sqlalchemy_uri=uri)

        model.impersonate_user = True
        user_name = make_url(model.get_sqla_engine(user_name=example_user).url).username
        self.assertEquals(example_user, user_name)

        model.impersonate_user = False
        user_name = make_url(model.get_sqla_engine(user_name=example_user).url).username
        self.assertNotEquals(example_user, user_name)
 def make_connection_manager(self, path):
     if isinstance(path, ConnectionManager):
         return path
     try:
         path = str(path)
         if path == "":
             return SQLiteConnectionManager("sqlite://")
         # Ensure that the path is in fact a URI
         make_url(path)
         return ConnectionManager(path)
     except:
         return self.connection_manager_type(path)
    def test_plugin_multiple_url_registration(self):
        from sqlalchemy.dialects import sqlite

        global MyEnginePlugin1
        global MyEnginePlugin2

        def side_effect_1(url, kw):
            eq_(kw, {"logging_name": "foob"})
            kw["logging_name"] = "bar"
            url.query.pop("myplugin1_arg", None)
            return MyEnginePlugin1

        def side_effect_2(url, kw):
            url.query.pop("myplugin2_arg", None)
            return MyEnginePlugin2

        MyEnginePlugin1 = Mock(side_effect=side_effect_1)
        MyEnginePlugin2 = Mock(side_effect=side_effect_2)

        plugins.register("engineplugin1", __name__, "MyEnginePlugin1")
        plugins.register("engineplugin2", __name__, "MyEnginePlugin2")

        e = create_engine(
            "sqlite:///?plugin=engineplugin1&foo=bar&myplugin1_arg=bat"
            "&plugin=engineplugin2&myplugin2_arg=hoho",
            logging_name="foob",
        )
        eq_(e.dialect.name, "sqlite")
        eq_(e.logging_name, "bar")

        # plugin args are removed from URL.
        eq_(e.url.query, {"foo": "bar"})
        assert isinstance(e.dialect, sqlite.dialect)

        eq_(
            MyEnginePlugin1.mock_calls,
            [
                call(url.make_url("sqlite:///?foo=bar"), {}),
                call.handle_dialect_kwargs(sqlite.dialect, mock.ANY),
                call.handle_pool_kwargs(mock.ANY, {"dialect": e.dialect}),
                call.engine_created(e),
            ],
        )

        eq_(
            MyEnginePlugin2.mock_calls,
            [
                call(url.make_url("sqlite:///?foo=bar"), {}),
                call.handle_dialect_kwargs(sqlite.dialect, mock.ANY),
                call.handle_pool_kwargs(mock.ANY, {"dialect": e.dialect}),
                call.engine_created(e),
            ],
        )
Exemple #16
0
def _assert_db_exists(engine):
    db_url = make_url(engine.url)
    db_name = db_url.database
    try:
        logger.debug('try to connect to database "%s": %s', db_name, db_url)
        connection = engine.connect()
        connection.close()
    except sqlalchemy.exc.OperationalError as err:
        db_url = make_url(engine.url)
        db_name = db_url.database
        logger.error('could not connect to database "%s": %s', db_name, str(err))
        raise ValueError('Cannot connect to database "%s".' % db_name)
    def test_query_string(self):
        u = url.make_url("dialect://*****:*****@host/db?arg1=param1&arg2=param2")
        eq_(u.query, {"arg1": "param1", "arg2": "param2"})
        eq_(str(u), "dialect://*****:*****@host/db?arg1=param1&arg2=param2")

        u = url.make_url(
            "dialect://*****:*****@host/db?arg1=param1&arg2=param2&arg2=param3"
        )
        eq_(u.query, {"arg1": "param1", "arg2": ["param2", "param3"]})
        eq_(
            str(u),
            "dialect://*****:*****@host/db?arg1=param1&arg2=param2&arg2=param3",
        )
Exemple #18
0
def check_db_url(db_url):
    """Check for the correct format of the DB URL.  Return none if check
    passed.  Raise exception otherwise.
    :param db_url: The DB URL to be checked
    :raise exc.ArPrDataBadDBURLFormatException:
    :rtype : None
    """
    try:
        make_url(db_url)
    except Exception:
        raise exc.ArPrDataBadDBURLFormatException(
            "Incorrect DB URL Format.  Check http://docs.sqlalchemy.org"
            "/en/latest/core/engines.html?highlight=url#database-urls")
Exemple #19
0
 def _url_from_target(self, target):
     if isinstance(target, Connectable):
         return target.engine.url
     elif isinstance(target, six.string_types):
         if "://" not in target:
             target_url = sa_url.make_url("%s://" % target)
         else:
             target_url = sa_url.make_url(target)
         return target_url
     elif isinstance(target, sa_url.URL):
         return target
     else:
         raise ValueError("Invalid target type: %r" % target)
    def get_sqla_engine(self, schema=None, nullpool=True, user_name=None):
        extra = self.get_extra()
        url = make_url(self.sqlalchemy_uri_decrypted)
        url = self.db_engine_spec.adjust_database_uri(url, schema)
        effective_username = self.get_effective_user(url, user_name)
        # If using MySQL or Presto for example, will set url.username
        # If using Hive, will not do anything yet since that relies on a
        # configuration parameter instead.
        self.db_engine_spec.modify_url_for_impersonation(
            url,
            self.impersonate_user,
            effective_username)

        masked_url = self.get_password_masked_url(url)
        logging.info('Database.get_sqla_engine(). Masked URL: {0}'.format(masked_url))

        params = extra.get('engine_params', {})
        if nullpool:
            params['poolclass'] = NullPool

        # If using Hive, this will set hive.server2.proxy.user=$effective_username
        configuration = {}
        configuration.update(
            self.db_engine_spec.get_configuration_for_impersonation(
                str(url),
                self.impersonate_user,
                effective_username))
        if configuration:
            params['connect_args'] = {'configuration': configuration}

        DB_CONNECTION_MUTATOR = config.get('DB_CONNECTION_MUTATOR')
        if DB_CONNECTION_MUTATOR:
            url, params = DB_CONNECTION_MUTATOR(
                url, params, effective_username, security_manager)
        return create_engine(url, **params)
    def test_engine_from_config(self):
        dbapi = mock_dbapi

        config = \
            {'sqlalchemy.url': 'postgresql://*****:*****@somehost/test'\
             '?fooz=somevalue', 'sqlalchemy.pool_recycle': '50',
             'sqlalchemy.echo': 'true'}

        e = engine_from_config(config, module=dbapi, _initialize=False)
        assert e.pool._recycle == 50
        assert e.url \
            == url.make_url('postgresql://*****:*****@somehost/test?foo'
                            'z=somevalue')
        assert e.echo is True

        for param, values in [
            ('convert_unicode', ('true', 'false', 'force')), 
            ('echo', ('true', 'false', 'debug')),
            ('echo_pool', ('true', 'false', 'debug')),
            ('use_native_unicode', ('true', 'false')),
        ]:
            for value in values:
                config = {
                        'sqlalchemy.url': 'postgresql://*****:*****@somehost/test',
                        'sqlalchemy.%s' % param : value
                }
                cfg = _coerce_config(config, 'sqlalchemy.')
                assert cfg[param] == {'true':True, 'false':False}.get(value, value)
Exemple #22
0
    def __init__(self, url):
        self.table = Table(self.__tablename__, MetaData(),
                           Column('name', String(64)),
                           Column('group', String(64)),
                           Column('status', String(16)),
                           Column('script', Text),
                           Column('comments', String(1024)),
                           Column('rate', Float(11)),
                           Column('burst', Float(11)),
                           Column('updatetime', Float(32)),
                           mysql_engine='InnoDB',
                           mysql_charset='utf8'
                           )

        self.url = make_url(url)
        if self.url.database:
            database = self.url.database
            self.url.database = None
            try:
                engine = create_engine(self.url)
                conn = engine.connect()
                conn.execute("commit")
                conn.execute("CREATE DATABASE %s" % database)
            except sqlalchemy.exc.SQLAlchemyError:
                pass
            self.url.database = database
        self.engine = create_engine(url)
        self.table.create(self.engine, checkfirst=True)
 def test_sqlite_abspath(self):
     u = url.make_url("sqlite:////x/state.sqlite")
     kwargs = dict(basedir='/my-base-dir')
     u, kwargs, max_conns = self.strat.special_case_sqlite(u, kwargs)
     self.assertEqual([ str(u), max_conns, self.filter_kwargs(kwargs) ],
         [ "sqlite:////x/state.sqlite", None,
           dict(basedir='/my-base-dir') ])
Exemple #24
0
    def _read_connection_has_correct_privileges(self):
        ''' Returns True if the right permissions are set for the read
        only user. A table is created by the write user to test the
        read only user.
        '''
        write_connection = db._get_engine(
            {'connection_url': self.write_url}).connect()
        read_connection_user = sa_url.make_url(self.read_url).username

        drop_foo_sql = u'DROP TABLE IF EXISTS _foo'

        write_connection.execute(drop_foo_sql)

        try:
            write_connection.execute(u'CREATE TEMP TABLE _foo ()')
            for privilege in ['INSERT', 'UPDATE', 'DELETE']:
                test_privilege_sql = u"SELECT has_table_privilege(%s, '_foo', %s)"
                have_privilege = write_connection.execute(
                    test_privilege_sql, (read_connection_user, privilege)).first()[0]
                if have_privilege:
                    return False
        finally:
            write_connection.execute(drop_foo_sql)
            write_connection.close()
        return True
 def test_rfc1738(self):
     for text in (
         'dbtype://*****:*****@hostspec:110//usr/db_file.db',
         'dbtype://*****:*****@hostspec/database',
         'dbtype://*****:*****@hostspec',
         'dbtype://*****:*****@/database',
         'dbtype://username@hostspec',
         'dbtype://*****:*****@127.0.0.1:1521',
         'dbtype://hostspec/database',
         'dbtype://hostspec',
         'dbtype://hostspec/?arg1=val1&arg2=val2',
         'dbtype:///database',
         'dbtype:///:memory:',
         'dbtype:///foo/bar/im/a/file',
         'dbtype:///E:/work/src/LEM/db/hello.db',
         'dbtype:///E:/work/src/LEM/db/hello.db?foo=bar&hoho=lala',
         'dbtype://',
         'dbtype://*****:*****@/db',
         'dbtype:////usr/local/mailman/lists/[email protected]/memb'
             'ers.db',
         'dbtype://*****:*****@hostspec/mydatabase',
         ):
         u = url.make_url(text)
         assert u.drivername == 'dbtype'
         assert u.username == 'username' or u.username is None
         assert u.password == 'password' or u.password \
             == 'apples/oranges' or u.password is None
         assert u.host == 'hostspec' or u.host == '127.0.0.1' \
             or not u.host
         assert str(u) == text
Exemple #26
0
 def dress_url(url):
     # If no explicit driver has been set, we default to pymysql
     if url.startswith("mysql://"):
         url = sqlalchemy_url.make_url(url)
         url.drivername = "mysql+pymysql"
         return str(url)
     return url
Exemple #27
0
    def setup_parser(self, parser):
        parser.formatter_class = RawDescriptionHelpFormatter
        if parser.description is None:
            parser.description = ''
        if self.default_url:
            parser.description += (
                '\nThe database to be acted on is at:\n%r' % (
                make_url(self.default_url)
            ))
        parser.description += (
            '\n\nThe following tables are in the current configuration:\n' +
            ', '.join(self.config.tables)
        )
        parser.add_argument(
            '--url',
            default = self.default_url,
            help='Override the database url used.'
            )

        commands = parser.add_subparsers()

        for name in dir(self.__class__):
            if name[0] == '_':
                continue
            doc = getattr(self, name).__doc__
            if doc is None:
                continue
            doc = doc.strip()
            command = commands.add_parser(
                name,
                help=doc
                )
            command.set_defaults(method=getattr(self,name))
    def create(self, name_or_url, **kwargs):
        if 'basedir' not in kwargs:
            raise TypeError('no basedir supplied to create_engine')

        max_conns = None

        # apply special cases
        u = url.make_url(name_or_url)
        if u.drivername.startswith('sqlite'):
            u, kwargs, max_conns = self.special_case_sqlite(u, kwargs)
        elif u.drivername.startswith('mysql'):
            u, kwargs, max_conns = self.special_case_mysql(u, kwargs)

        # remove the basedir as it may confuse sqlalchemy
        basedir = kwargs.pop('basedir')

        # calculate the maximum number of connections from the pool parameters,
        # if it hasn't already been specified
        if max_conns is None:
            max_conns = kwargs.get('pool_size', 5) + kwargs.get('max_overflow', 10)

        engine = strategies.ThreadLocalEngineStrategy.create(self,
                                            u, **kwargs)

        # annotate the engine with the optimal thread pool size; this is used
        # by DBConnector to configure the surrounding thread pool
        engine.optimal_thread_pool_size = max_conns

        # and keep the basedir
        engine.buildbot_basedir = basedir

        return engine
Exemple #29
0
    def __init__(self, url):
        self.table = Table('__tablename__', MetaData(),
                           Column('taskid', String(64), primary_key=True, nullable=False),
                           Column('project', String(64)),
                           Column('url', String(1024)),
                           Column('status', Integer),
                           Column('schedule', LargeBinary),
                           Column('fetch', LargeBinary),
                           Column('process', LargeBinary),
                           Column('track', LargeBinary),
                           Column('lastcrawltime', Float(32)),
                           Column('updatetime', Float(32)),
                           mysql_engine='InnoDB',
                           mysql_charset='utf8'
                           )

        self.url = make_url(url)
        if self.url.database:
            database = self.url.database
            self.url.database = None
            try:
                engine = create_engine(self.url, convert_unicode=True)
                engine.execute("CREATE DATABASE IF NOT EXISTS %s" % database)
            except sqlalchemy.exc.SQLAlchemyError:
                pass
            self.url.database = database
        self.engine = create_engine(self.url, convert_unicode=True)

        self._list_project()
 def test_mysql_simple(self):
     u = url.make_url("mysql://host/dbname")
     kwargs = dict(basedir='my-base-dir')
     u, kwargs, max_conns = self.strat.special_case_mysql(u, kwargs)
     self.assertEqual([str(u), max_conns, self.filter_kwargs(kwargs)],
                      ["mysql://host/dbname?charset=utf8&use_unicode=True", None,
                       self.mysql_kwargs])
Exemple #31
0
def create_database(url, encoding='utf8', template=None):
    """Issue the appropriate CREATE DATABASE statement.

    :param url: A SQLAlchemy engine URL.
    :param encoding: The encoding to create the database as.
    :param template:
        The name of the template from which to create the new database. At the
        moment only supported by PostgreSQL driver.

    To create a database, you can pass a simple URL that would have
    been passed to ``create_engine``. ::

        create_database('postgres://postgres@localhost/name')

    You may also pass the url from an existing engine. ::

        create_database(engine.url)

    Has full support for mysql, postgres, and sqlite. In theory,
    other database engines should be supported.
    """

    url = copy(make_url(url))

    database = url.database

    if url.drivername.startswith('postgresql'):
        url.database = 'template1'
    elif not url.drivername.startswith('sqlite'):
        url.database = None

    engine = sa.create_engine(url)

    if engine.dialect.name == 'postgresql':
        if engine.driver == 'psycopg2':
            from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
            engine.raw_connection().set_isolation_level(
                ISOLATION_LEVEL_AUTOCOMMIT
            )

        if not template:
            template = 'template0'

        text = "CREATE DATABASE {0} ENCODING '{1}' TEMPLATE {2}".format(
            quote(engine, database),
            encoding,
            quote(engine, template)
        )
        engine.execute(text)

    elif engine.dialect.name == 'mysql':
        text = "CREATE DATABASE {0} CHARACTER SET = '{1}'".format(
            quote(engine, database),
            encoding
        )
        engine.execute(text)

    elif engine.dialect.name == 'sqlite' and database != ':memory:':
        if database:
            open(database, 'w').close()

    else:
        text = 'CREATE DATABASE {0}'.format(quote(engine, database))
        engine.execute(text)
Exemple #32
0
 def _get_db_from_url(self, url):
     db_url = sa_url.make_url(url)
     return db_url.host, db_url.port, db_url.database
Exemple #33
0
 def backend(self) -> str:
     sqlalchemy_url = make_url(self.sqlalchemy_uri_decrypted)
     return sqlalchemy_url.get_backend_name()  # pylint: disable=no-member
Exemple #34
0
 def backend(self):
     url = make_url(self.sqlalchemy_uri_decrypted)
     return url.get_backend_name()
 def test_mysql_mysqldb_connect_args_default(self):
     session._init_connection_args(
         url.make_url("mysql+mysqldb://u:p@host/test"), self.args)
     self.assertEqual(self.args['connect_args'],
                      {'charset': 'utf8', 'use_unicode': 0})
Exemple #36
0
def _sqlite_follower_url_from_main(url, ident):
    url = sa_url.make_url(url)
    if not url.database or url.database == ':memory:':
        return url
    else:
        return sa_url.make_url("sqlite:///%s.db" % ident)
Exemple #37
0
 def validate_password(self, data: Dict[str, Any], **kwargs: Any) -> None:
     """If sqlalchemy_uri has a masked password, password is required"""
     uri = data["sqlalchemy_uri"]
     password = make_url(uri).password
     if password == PASSWORD_MASK and data.get("password") is None:
         raise ValidationError("Must provide a password for the database")
Exemple #38
0
 def __init__(self, db_url, database):
     self.url = make_url(db_url)
     self.database = database
     if not database:
         self._handle_no_database()
Exemple #39
0
def create_app(config='CTFd.config.Config'):
    app = Flask(__name__)
    with app.app_context():
        app.config.from_object(config)
        app.jinja_loader = ThemeLoader(os.path.join(app.root_path, app.template_folder), followlinks=True)

        from CTFd.models import db, Teams, Solves, Challenges, WrongKeys, Keys, Tags, Files, Tracking

        url = make_url(app.config['SQLALCHEMY_DATABASE_URI'])
        if url.drivername == 'postgres':
            url.drivername = 'postgresql'

        # Creates database if the database database does not exist
        if not database_exists(url):
            create_database(url)

        # Register database
        db.init_app(app)

        # Register Flask-Migrate
        migrate.init_app(app, db)

        # This creates tables instead of db.create_all()
        # Allows migrations to happen properly
        migrate_upgrade()

        # Alembic sqlite support is lacking so we should just create_all anyway
        if url.drivername.startswith('sqlite'):
            db.create_all()

        app.db = db

        cache.init_app(app)
        app.cache = cache

        version = utils.get_config('ctf_version')

        if not version:  # Upgrading from an unversioned CTFd
            utils.set_config('ctf_version', __version__)

        if version and (StrictVersion(version) < StrictVersion(__version__)):  # Upgrading from an older version of CTFd
            print("/*\\ CTFd has updated and must update the database! /*\\")
            print("/*\\ Please backup your database before proceeding! /*\\")
            print("/*\\ CTFd maintainers are not responsible for any data loss! /*\\")
            if input('Run database migrations (Y/N)').lower().strip() == 'y':
                migrate_stamp()
                migrate_upgrade()
                utils.set_config('ctf_version', __version__)
            else:
                print('/*\\ Ignored database migrations... /*\\')
                exit()

        if not utils.get_config('ctf_theme'):
            utils.set_config('ctf_theme', 'original')

        from CTFd.views import views
        from CTFd.challenges import challenges
        from CTFd.scoreboard import scoreboard
        from CTFd.auth import auth
        from CTFd.admin import admin, admin_statistics, admin_challenges, admin_pages, admin_scoreboard, admin_containers, admin_keys, admin_teams
        from CTFd.utils import init_utils, init_errors, init_logs

        init_utils(app)
        init_errors(app)
        init_logs(app)

        app.register_blueprint(views)
        app.register_blueprint(challenges)
        app.register_blueprint(scoreboard)
        app.register_blueprint(auth)

        app.register_blueprint(admin)
        app.register_blueprint(admin_statistics)
        app.register_blueprint(admin_challenges)
        app.register_blueprint(admin_teams)
        app.register_blueprint(admin_scoreboard)
        app.register_blueprint(admin_keys)
        app.register_blueprint(admin_containers)
        app.register_blueprint(admin_pages)

        from CTFd.plugins import init_plugins

        init_plugins(app)

        return app
Exemple #40
0
    def create(self, name_or_url, **kwargs):
        # create url.URL object
        u = url.make_url(name_or_url)

        dialect_cls = u.get_dialect()

        if kwargs.pop('_coerce_config', False):
            def pop_kwarg(key, default=None):
                value = kwargs.pop(key, default)
                if key in dialect_cls.engine_config_types:
                    value = dialect_cls.engine_config_types[key](value)
                return value
        else:
            pop_kwarg = kwargs.pop

        dialect_args = {}
        # consume dialect arguments from kwargs
        for k in util.get_cls_kwargs(dialect_cls):
            if k in kwargs:
                dialect_args[k] = pop_kwarg(k)

        dbapi = kwargs.pop('module', None)
        if dbapi is None:
            dbapi_args = {}
            for k in util.get_func_kwargs(dialect_cls.dbapi):
                if k in kwargs:
                    dbapi_args[k] = pop_kwarg(k)
            dbapi = dialect_cls.dbapi(**dbapi_args)

        dialect_args['dbapi'] = dbapi

        # create dialect
        dialect = dialect_cls(**dialect_args)

        # assemble connection arguments
        (cargs, cparams) = dialect.create_connect_args(u)
        cparams.update(pop_kwarg('connect_args', {}))

        # look for existing pool or create
        pool = pop_kwarg('pool', None)
        if pool is None:
            def connect():
                return dialect.connect(*cargs, **cparams)

            creator = pop_kwarg('creator', connect)

            poolclass = pop_kwarg('poolclass', None)
            if poolclass is None:
                poolclass = dialect_cls.get_pool_class(u)
            pool_args = {}

            # consume pool arguments from kwargs, translating a few of
            # the arguments
            translate = {'logging_name': 'pool_logging_name',
                         'echo': 'echo_pool',
                         'timeout': 'pool_timeout',
                         'recycle': 'pool_recycle',
                         'events': 'pool_events',
                         'use_threadlocal': 'pool_threadlocal',
                         'reset_on_return': 'pool_reset_on_return'}
            for k in util.get_cls_kwargs(poolclass):
                tk = translate.get(k, k)
                if tk in kwargs:
                    pool_args[k] = pop_kwarg(tk)
            pool = poolclass(creator, **pool_args)
        else:
            if isinstance(pool, poollib._DBProxy):
                pool = pool.get_pool(*cargs, **cparams)
            else:
                pool = pool

        # create engine.
        engineclass = self.engine_cls
        engine_args = {}
        for k in util.get_cls_kwargs(engineclass):
            if k in kwargs:
                engine_args[k] = pop_kwarg(k)

        _initialize = kwargs.pop('_initialize', True)

        # all kwargs should be consumed
        if kwargs:
            raise TypeError(
                "Invalid argument(s) %s sent to create_engine(), "
                "using configuration %s/%s/%s.  Please check that the "
                "keyword arguments are appropriate for this combination "
                "of components." % (','.join("'%s'" % k for k in kwargs),
                                    dialect.__class__.__name__,
                                    pool.__class__.__name__,
                                    engineclass.__name__))

        engine = engineclass(pool, dialect, u, **engine_args)

        if _initialize:
            do_on_connect = dialect.on_connect()
            if do_on_connect:
                def on_connect(dbapi_connection, connection_record):
                    conn = getattr(
                        dbapi_connection, '_sqla_unwrap', dbapi_connection)
                    if conn is None:
                        return
                    do_on_connect(conn)

                event.listen(pool, 'first_connect', on_connect)
                event.listen(pool, 'connect', on_connect)

            def first_connect(dbapi_connection, connection_record):
                c = base.Connection(engine, connection=dbapi_connection,
                                    _has_events=False)
                c._execution_options = util.immutabledict()
                dialect.initialize(c)
            event.listen(pool, 'first_connect', first_connect, once=True)

        return engine
Exemple #41
0
def obfuscate_url_pw(u):
    u = url.make_url(u)
    if u.password:
        u.password = "******"
    return str(u)
Exemple #42
0
 def get_dialect(self):
     sqla_url = url.make_url(self.sqlalchemy_uri_decrypted)
     return sqla_url.get_dialect()()
Exemple #43
0
 def get_sqla_engine(self, schema=None):
     extra = self.get_extra()
     uri = make_url(self.sqlalchemy_uri_decrypted)
     params = extra.get('engine_params', {})
     uri = self.db_engine_spec.adjust_database_uri(uri, schema)
     return create_engine(uri, **params)
Exemple #44
0
 def provisioned_database_url(self, base_url, ident):
     if base_url.database:
         return sa_url.make_url("sqlite:////tmp/%s.db" % ident)
     else:
         return base_url
Exemple #45
0
def _oracle_follower_url_from_main(url, ident):
    url = sa_url.make_url(url)
    url.username = ident
    url.password = '******'
    return url
Exemple #46
0
def main():

    parser = argparse.ArgumentParser(description="Qserv database schema migration.")

    parser.add_argument("-v", "--verbose", default=0, action="count",
                        help="Use one -v for INFO logging, two for DEBUG.")
    group = parser.add_mutually_exclusive_group()
    group.add_argument("-m", "--do-migrate", default=False, action="store_true",
                       help="Do migration, without this option script prints various info "
                       "and exits.")
    group.add_argument("--check", default=False, action="store_true",
                       help="Check that migration is needed, script returns 0 if schema is "
                       "up-to-date, 1 otherwise.")
    parser.add_argument("-n", "--final", default=None, action="store", type=int,
                        metavar="VERSION",
                        help="Stop migration at given version, by default update to "
                        "latest version.")
    parser.add_argument("--scripts", default=_def_scripts, action="store",
                        metavar="PATH",
                        help="Location for migration scripts, def: %(default)s.")
    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument("-c", "--connection", metavar="CONNECTION",
                       help="Connection string in format mysql://user:pass@host:port/database.")
    group.add_argument("-f", "--config-file", metavar="PATH",
                       help="Name of configuration file in INI format with connection parameters.")
    parser.add_argument("-s", "--config-section", metavar="NAME",
                        help="Name of configuration section in configuration file.")

    parser.add_argument("module",
                        help="Name of Qserv module for which to update schema, e.g. qmeta.")

    args = parser.parse_args()

    # configure logging
    levels = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG}
    level = levels.get(args.verbose, logging.DEBUG)
    fmt = "%(asctime)s [%(levelname)s] %(name)s: %(message)s"
    logging.basicConfig(level=level, format=fmt)

    if args.connection:
        url = make_url(args.connection)
        engine = sqlalchemy.create_engine(url)
    elif args.config_file:
        if not args.config_section:
            parser.error("-s options required with -f")

        cfg = configparser.SafeConfigParser()
        if not cfg.read([args.config_file]):
            # file was not found, generate exception which should happen
            # if we tried to open that file
            raise IOError(2, "No such file or directory: '{}'".format(args.config_file))

        # will throw is section is missing
        config = dict(cfg.items(args.config_section))

        # instantiate database engine
        config = _normalizeConfig(config)
        engine = engineFactory.getEngineFromArgs(**config)

    # make an object which will manage migration process
    mgr = _load_migration_mgr(args.module, engine=engine, scripts_dir=args.scripts)

    current = mgr.current_version()
    print("Current schema version: {}".format(current))

    latest = mgr.latest_version()
    print("Latest schema version: {}".format(latest))

    migrations = mgr.migrations()
    print("Known migrations:")
    for v0, v1, script in migrations:
        tag = " (X)" if v0 >= current else ""
        print("  {} -> {} : {}{}".format(v0, v1, script, tag))

    if args.check:
        return 0 if mgr.current_version() == mgr.latest_version() else 1

    # do the migrations
    final = mgr.migrate(args.final, args.do_migrate)
    if final is None:
        print("No migration was needed")
    else:
        if args.do_migrate:
            print("Database was migrated to version {}".format(final))
        else:
            print("Database would be migrated to version {}".format(final))
Exemple #47
0
def _follower_url_from_main(url, ident):
    url = sa_url.make_url(url)
    url.database = ident
    return url
Exemple #48
0
    os.environ.get('GEOSERVER_ENDPOINT',
                   'http://*****:*****@172.17.0.1:5432/tds_tests')
_url_object = make_url(_URL)
_PUBLIC_URL = os.environ.get(
    'POSTGIS_PUBLIC_URL',
    'postgresql://*****:*****@localhost:5432/tds_tests')
_public_url_object = make_url(_PUBLIC_URL)

TEST_POSTGIS_SERVICE = {
    'URL': _URL,
    'DRIVER': _url_object.drivername,
    'USERNAME': _url_object.username,
    'PASSWORD': _url_object.password,
    'HOST': _url_object.host,
    'PORT': _url_object.port,
    'DATABASE': _url_object.database,
    'PUBLIC_URL': _PUBLIC_URL,
    'PUBLIC_HOST': _public_url_object.host,
Exemple #49
0
 def test_psycopg2_nonempty_connection_string_w_query(self):
     dialect = psycopg2_dialect.dialect()
     u = url.make_url("postgresql://somehost/?any_random_thing=yes")
     cargs, cparams = dialect.create_connect_args(u)
     eq_(cargs, [])
     eq_(cparams, {"host": "somehost", "any_random_thing": "yes"})
 def test_queuepool_args(self):
     session._init_connection_args(
         url.make_url("mysql://*****:*****@host/test"), self.args,
         max_pool_size=10, max_overflow=10)
     self.assertEqual(self.args['pool_size'], 10)
     self.assertEqual(self.args['max_overflow'], 10)
Exemple #51
0
 def test_psycopg2_nonempty_connection_string(self):
     dialect = psycopg2_dialect.dialect()
     u = url.make_url("postgresql://host")
     cargs, cparams = dialect.create_connect_args(u)
     eq_(cargs, [])
     eq_(cparams, {"host": "host"})
Exemple #52
0
 def test_psycopg2_empty_connection_string_w_query_one(self):
     dialect = psycopg2_dialect.dialect()
     u = url.make_url("postgresql:///?service=swh-log")
     cargs, cparams = dialect.create_connect_args(u)
     eq_(cargs, [])
     eq_(cparams, {"service": "swh-log"})
Exemple #53
0
def extract_active_data(trans, url, dump_location="dump.sql"):
    """
    Stores sqldump data in the specified location. If not specified, stores it in current directory in file dump.sql
    If file already exists it will override that file and not append it.

    Function added in to enhance testing in Balrog's stage environment.

    :param trans: Transaction Object for an SQL connection
    :param url: Database. eg : mysql://balrogadmin:balrogadmin@balrogdb/balrog
    :param dump_location: location where sqldump file must be created
    """
    url = make_url(url)
    user = url.username
    password = url.password
    host = url.host
    db = url.database

    # Extract the entire database schema, without any rows.
    # This is done to ensure that any database dump generated can be
    # imported to an empty database without issue. From there, a Balrog
    # installation can be upgraded or downgraded to a different database
    # schema version if desired.
    # See https://bugzilla.mozilla.org/show_bug.cgi?id=1376331 for additional
    # background on this.
    with open(dump_location, "w+") as dump_file:
        run(mysql_command(host, user, password, db, "--no-data").split(), stdout=dump_file, check=True)

        # Now extract the data we actually want....
        # We always want all the data from a few tables...
        run(mysql_data_only_command(host, user, password, db, "dockerflow rules rules_history migrate_version").split(), stdout=dump_file, check=True)

        # Because Releases are so massive, we only want the actively used ones,
        # and very little Release history. Specifically:
        #   - All releases referenced by a Rule or a Active Scheduled Rule Change
        #   - All releases referenced by a Release from the above query
        #   - 50 rows of history for the "Firefox-mozilla-central-nightly-latest" Release
        #   - Full history for the Release currently referenced by the "firefox-release" Rule.
        query_release_mapping = """SELECT DISTINCT releases.* \
            FROM releases, rules, rules_scheduled_changes \
            WHERE (releases.name IN (rules.mapping, rules.fallbackMapping))
            OR (rules_scheduled_changes.complete = 0 AND
                releases.name IN (rules_scheduled_changes.base_mapping, rules_scheduled_changes.base_fallbackMapping))
            """

        result = trans.execute(query_release_mapping).fetchall()
        release_names = set()
        for row in result:
            try:
                release_names.add(str(row["name"]))
                release_blob = createBlob(row["data"])
                release_names.update(release_blob.getReferencedReleases())
            except ValueError:
                continue
        if release_names:
            batch_generator = chunk_list(list(release_names), 30)
            for batched_release_list in batch_generator:
                query = ", ".join("'" + names + "'" for names in batched_release_list)
                cmd = mysql_data_only_command(host, user, password, db, "releases").split()
                cmd.append('--where="releases.name IN ({})"'.format(query))
                run(cmd, stdout=dump_file, check=True)

        cmd = mysql_data_only_command(host, user, password, db, "releases_history").split()
        cmd.append("--where=\"release_history.name='Firefox-mozilla-central-nightly-latest' ORDER BY timestamp DESC LIMIT 50\"")
        run(cmd, stdout=dump_file, check=True)

        query = "SELECT rules.mapping FROM rules WHERE rules.alias='firefox-release'"
        cmd = mysql_data_only_command(host, user, password, db, "releases_history").split()
        cmd.append('--where="name = ({}) ORDER BY timestamp DESC LIMIT 50"'.format(query))
        run(cmd, stdout=dump_file, check=True)
Exemple #54
0
 def test_random_arg(self):
     dialect = testing.db.dialect
     kw = dialect.create_connect_args(
         make_url("mysql://*****:*****@host/db?foo=true"))[1]
     eq_(kw["foo"], "true")
Exemple #55
0
    def create(self, name_or_url, **kwargs):
        # create url.URL object
        u = url.make_url(name_or_url)

        dialect_cls = u.get_dialect()

        dialect_args = {}
        # consume dialect arguments from kwargs
        for k in util.get_cls_kwargs(dialect_cls):
            if k in kwargs:
                dialect_args[k] = kwargs.pop(k)

        dbapi = kwargs.pop('module', None)
        if dbapi is None:
            dbapi_args = {}
            for k in util.get_func_kwargs(dialect_cls.dbapi):
                if k in kwargs:
                    dbapi_args[k] = kwargs.pop(k)
            dbapi = dialect_cls.dbapi(**dbapi_args)

        dialect_args['dbapi'] = dbapi

        # create dialect
        dialect = dialect_cls(**dialect_args)

        # assemble connection arguments
        (cargs, cparams) = dialect.create_connect_args(u)
        cparams.update(kwargs.pop('connect_args', {}))

        # look for existing pool or create
        pool = kwargs.pop('pool', None)
        if pool is None:

            def connect():
                try:
                    return dialect.connect(*cargs, **cparams)
                except dialect.dbapi.Error, e:
                    invalidated = dialect.is_disconnect(e, None, None)
                    # Py3K
                    #raise exc.DBAPIError.instance(None, None,
                    #    e, dialect.dbapi.Error,
                    #    connection_invalidated=invalidated
                    #) from e
                    # Py2K
                    import sys
                    raise exc.DBAPIError.instance(
                        None,
                        None,
                        e,
                        dialect.dbapi.Error,
                        connection_invalidated=invalidated
                    ), None, sys.exc_info()[2]
                    # end Py2K

            creator = kwargs.pop('creator', connect)

            poolclass = kwargs.pop('poolclass', None)
            if poolclass is None:
                poolclass = dialect_cls.get_pool_class(u)
            pool_args = {}

            # consume pool arguments from kwargs, translating a few of
            # the arguments
            translate = {
                'logging_name': 'pool_logging_name',
                'echo': 'echo_pool',
                'timeout': 'pool_timeout',
                'recycle': 'pool_recycle',
                'events': 'pool_events',
                'use_threadlocal': 'pool_threadlocal',
                'reset_on_return': 'pool_reset_on_return'
            }
            for k in util.get_cls_kwargs(poolclass):
                tk = translate.get(k, k)
                if tk in kwargs:
                    pool_args[k] = kwargs.pop(tk)
            pool = poolclass(creator, **pool_args)
Exemple #56
0
 def get_password_masked_url_from_uri(cls, uri):
     url = make_url(uri)
     return cls.get_password_masked_url(url)
Exemple #57
0
 def get_dialect(self) -> Dialect:
     sqla_url = url.make_url(self.sqlalchemy_uri_decrypted)
     return sqla_url.get_dialect()()  # pylint: disable=no-member
Exemple #58
0
 def get_password_masked_url_from_uri(  # pylint: disable=invalid-name
         cls, uri: str) -> URL:
     sqlalchemy_url = make_url(uri)
     return cls.get_password_masked_url(sqlalchemy_url)
 def test_postgresql_connect_args_default(self):
     session._init_connection_args(
         url.make_url("postgresql://*****:*****@host/test"), self.args)
     self.assertEqual(self.args['client_encoding'], 'utf8')
     self.assertFalse(self.args['connect_args'])
Exemple #60
0
 def url_object(self):
     return make_url(self.sqlalchemy_uri_decrypted)