Example #1
0
def main(argv=sys.argv):
    import logging

    logging.basicConfig(level=logging.DEBUG)
    # if len(argv) != 2:
    #     usage(argv)
    # config_uri = argv[1]
    # setup_logging(config_uri)
    # settings = get_appsettings(config_uri)
    # engine = engine_from_config(settings, 'sqlalchemy.')
    logging.debug("Initialise Engine")
    engine = sqlalchemy.create_engine(DBFILE, echo=False)
    
    meta.Session.configure(bind=engine)
    Base.metadata.bind = engine
    Base.metadata.create_all(engine)
    


    DBSession = meta.Session()
    #DBSession.configure(bind=engine)
    logging.debug("Populating Data")
    populateData.init_data(DBSession)
    #populateUser()
    logging.debug("Populated")
    DBSession.flush()
    #DBSession.commit()


    #We also want any alembic scripts to be executed (or not if we build the DB
    #properly)
    alembic_cfg = Config("cogent/alembic.ini") #TODO: WARNING RELATIVE PATH
    command.stamp(alembic_cfg,"head")

    DBSession.close()
Example #2
0
 def create_schema(self, set_alembic_version=True):
     """Creates the 'mediatum' schema.
     :param set_alembic_version: Stamp database with current alembic revision information. Defaults to True.
     Can be disabled if a schema for testing is going to be created.
     """
     s = self.session
     logg.info("creating DB schema...")
     s.execute("CREATE SCHEMA mediatum")
     s.commit()
     try:
         self.create_all()
         if set_alembic_version:
             # create alembic version table and set current alembic version to head
             from alembic.config import Config
             from alembic import command
             alembic_cfg = Config(os.path.join(config.basedir, "alembic.ini"))
             alembic_cfg.attributes["running_in_mediatum"] = True
             command.stamp(alembic_cfg, "head")
         s.commit()
         logg.info("commited database structure")
     except:
         # I tried to use a transaction to enclose everything, but sqlalchemy (?) fails when the schema is created within the transaction
         # solution: just drop the schema it if something fails after schema creation
         s.execute("DROP SCHEMA mediatum CASCADE")
         raise
Example #3
0
def place_database_under_alembic_control():
    a_config = get_alembic_config()

    if not is_database_under_migrate_control():
        return

    if not is_database_under_alembic_control():
        print(_("Database is currently not under Alembic's migration "
                "control."))
        head = get_current_legacy_head()
        if head == 42:
            alembic_version = 'liberty'
        elif head == 43:
            alembic_version = 'mitaka01'
        elif head == 44:
            alembic_version = 'mitaka02'
        elif head == 45:
            alembic_version = 'ocata01'
        elif head in range(1, 42):
            print("Legacy head: ", head)
            sys.exit(_("The current database version is not supported any "
                       "more. Please upgrade to Liberty release first."))
        else:
            sys.exit(_("Unable to place database under Alembic's migration "
                       "control. Unknown database state, can't proceed "
                       "further."))

        print(_("Placing database under Alembic's migration control at "
                "revision:"), alembic_version)
        alembic_command.stamp(a_config, alembic_version)
Example #4
0
File: cli.py Project: DO101/pybossa
def db_create():
    '''Create the db'''
    db.create_all()
    # then, load the Alembic configuration and generate the
    # version table, "stamping" it with the most recent rev:
    alembic_cfg = Config("alembic.ini")
    command.stamp(alembic_cfg,"head")
Example #5
0
def bootstrap_db(config_uri=None, engine=None, with_migration=True):
    """Bring a blank database to a functional state."""
    if engine is None:
        engine = create_engine(config_uri)
    db.configure(bind=engine)

    if with_migration:
        context = MigrationContext.configure(engine.connect())
        db_version = context.get_current_revision()

        if db_version:
            sys.stderr.write('Database already initialized. Bailing out.\n')
            sys.exit(2)

        config = Config(config_uri)
        script_dir = ScriptDirectory.from_config(config)
        heads = script_dir.get_heads()

        if len(heads) > 1:
            sys.stderr.write('Error: migration scripts have more than one '
                             'head.\nPlease resolve the situation before '
                             'attempting to bootstrap the database.\n')
            sys.exit(2)

    metadata.create_all(engine)

    with transaction.manager:
        model = MyModel(name='one', value=1)
        db.add(model)

    # Clean up the sccoped session to allow a later app instantiation.
    db.remove()

    if with_migration and heads:
        command.stamp(config, 'head')
Example #6
0
def create_tables(db_url, alembic_ini=None, debug=False):
    """ Create the tables in the database using the information from the
    url obtained.

    :arg db_url, URL used to connect to the database. The URL contains
        information with regards to the database engine, the host to
        connect to, the user and password and the database name.
          ie: <engine>://<user>:<password>@<host>/<dbname>
    :kwarg alembic_ini, path to the alembic ini file. This is necessary to
        be able to use alembic correctly, but not for the unit-tests.
    :kwarg debug, a boolean specifying wether we should have the verbose
        output of sqlalchemy or not.
    :return a session that can be used to query the database.
    """
    engine = sa.create_engine(db_url, echo=debug)
    BASE.metadata.create_all(engine)

    if alembic_ini is not None:  # pragma: no cover
        # then, load the Alembic configuration and generate the
        # version table, "stamping" it with the most recent rev:
        from alembic.config import Config
        from alembic import command
        alembic_cfg = Config(alembic_ini)
        command.stamp(alembic_cfg, "head")

    scopedsession = scoped_session(sessionmaker(bind=engine))
    return scopedsession
Example #7
0
 def test_history_current_to_head_as_b(self):
     command.stamp(self.cfg, self.b)
     self.cfg.stdout = buf = StringIO()
     command.history(self.cfg, "current:")
     self._eq_cmd_output(buf, [
             '%(revb)s -> %(revc)s (head), Rev C',
         ])
Example #8
0
def setup_db():
    """
    Either initialize the database if none yet exists, or migrate as needed
    """

    from alembic.config import Config
    from alembic import command

    with current_app.app_context():
        # Alembic config used by migration or stamping
        alembic_cfg = Config(
            os.path.join(current_app.config["PROJECT_PATH"], "alembic.ini")
        )

        # Database connections
        g.db_session = create_db_session()
        con = g.db_session.connection()

        # Query list of existing tables
        tables = con.execute("show tables").fetchall()
        alembic_table = ('alembic_version',)
        if alembic_table in tables:
            # Latest version has been stamped or we have been upgrading
            logging.info("Database: Migrating")
            command.upgrade(alembic_cfg, "head")
        else:
            # The database needs to be initialized
            logging.info("Database: Initializing")
            init_db()
            command.stamp(alembic_cfg, "head")
Example #9
0
    def setup_tables(cls, engine):
        with engine.connect() as conn:
            trans = conn.begin()
            _Model.metadata.create_all(engine)
            # Now stamp the latest alembic version
            alembic_cfg = Config()
            alembic_cfg.set_section_option(
                'alembic', 'script_location', 'alembic')
            alembic_cfg.set_section_option(
                'alembic', 'sqlalchemy.url', str(engine.url))
            alembic_cfg.set_section_option(
                'alembic', 'sourceless', 'true')

            command.stamp(alembic_cfg, 'head')

            # always add a test API key
            conn.execute(ApiKey.__table__.delete())

            key1 = ApiKey.__table__.insert().values(
                valid_key='test', allow_fallback=False, allow_locate=True)
            conn.execute(key1)
            key2 = ApiKey.__table__.insert().values(
                valid_key='export', allow_fallback=False, allow_locate=False)
            conn.execute(key2)

            trans.commit()
Example #10
0
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--recreate',
                        help='Drop and re-create all tables.',
                        action='store_true')
    parser.add_argument('--dog-csv',
                        help='The csv file containing the Dog_info table.')
    parser.add_argument('--user-csv',
                        help='The csv file containing the auth table.')
    parser.add_argument('--app-csv',
                        help='The csv file containing the Apps table.')
    args = parser.parse_args()

    if args.recreate:
        print "Dropping all tables."
        db.drop_all()
        print "Creating all tables."
        db.create_all()
        print "Stamping alembic @ HEAD."
        with app.app_context():
            Migrate(app, db, directory=pkg_resources.resource_filename('crrr', 'migrations'))
            command.stamp(_get_config(None), 'head')
    if args.dog_csv:
        import_dogs(args.dog_csv)
    if args.user_csv:
        import_users(args.user_csv)
    if args.app_csv:
        import_apps(args.app_csv)
Example #11
0
def db_create():
    """Create the tables and do alembic stuff"""
    print 'db at: {0}'.format(
        app.config['SQLALCHEMY_DATABASE_URI'])
    try:
        db.engine.execute('select * from project')
        print 'Database already exists with tables.'
        return

    except (OperationalError, ProgrammingError):
        # An operational error here means that the "project" table
        # doesn't exist so we should create things!
        pass

    print 'Creating {0}....'.format(
        app.config['SQLALCHEMY_DATABASE_URI'])

    db.create_all()

    from alembic.config import Config
    from alembic import command
    alembic_cfg = Config('alembic.ini')
    alembic_cfg.set_main_option("sqlalchemy.url",
        app.config["SQLALCHEMY_DATABASE_URI"])

    command.stamp(alembic_cfg, 'head')
    print 'Done.'
Example #12
0
def reset_db():
    """Reset database"""

    print "WARNING: This will reset the database and may cause data loss."
    response = raw_input("Are you sure you want to continue? (Yes/No) ")
    if not response == "Yes":
        print "Aborted."
        sys.exit()

    db.drop_all()
    db.create_all()

    admin = User(
            name=u'admin',
            email=u'*****@*****.**',
            password=u'123456',
            role_code=ADMIN,
            status_code=ACTIVE,
            user_detail=UserDetail(
                gender_code=MALE,
                age=25,
                url=u'http://example.com',
                location=u'Kampala',
                bio=u''))
    db.session.add(admin)
    
    english = Language(name="English",iso639_1="en",iso639_2="eng",locale_code="en_UG")
    db.session.add(english)
    luganda = Language(name="Luganda",iso639_1="lg",iso639_2="lug",locale_code="lg_UG")
    db.session.add(luganda)

    db.session.commit()
    alembic_cfg = Config("alembic.ini")
    command.stamp(alembic_cfg, "head")
Example #13
0
    def test_noerr_transaction_opened_externally(self):
        a, b, c = self._opened_transaction_fixture()

        env_file_fixture("""
from sqlalchemy import engine_from_config, pool

def run_migrations_online():
    connectable = engine_from_config(
        config.get_section(config.config_ini_section),
        prefix='sqlalchemy.',
        poolclass=pool.NullPool)

    with connectable.connect() as connection:
        with connection.begin() as real_trans:
            context.configure(
                connection=connection,
                transactional_ddl=False,
                transaction_per_migration=False
            )

            with context.begin_transaction():
                context.run_migrations()

run_migrations_online()

""")

        command.stamp(self.cfg, c)
    def test_destination_rev_pre_context(self):
        env_file_fixture("""
assert context.get_revision_argument() == '%s'
""" % b)
        command.upgrade(self.cfg, b, sql=True)
        command.stamp(self.cfg, b, sql=True)
        command.downgrade(self.cfg, "%s:%s" % (c, b), sql=True)
Example #15
0
 def test_stamp_existing_downgrade(self):
     command.stamp(self.cfg, self.b)
     command.stamp(self.cfg, self.a)
     eq_(
         self.bind.scalar("select version_num from alembic_version"),
         self.a
     )
Example #16
0
def main(argv, _db_master=None):
    parser = argparse.ArgumentParser(
        prog=argv[0], description='Initialize Ichnaea database')

    parser.add_argument('--initdb', action='store_true',
                        help='Initialize database')

    args = parser.parse_args(argv[1:])

    if args.initdb:
        from ichnaea import config
        # make sure content models are imported
        from ichnaea.content import models  # NOQA

        conf = config()
        db_master = Database(conf.get('ichnaea', 'db_master'))
        engine = db_master.engine
        with engine.connect() as conn:
            trans = conn.begin()
            _Model.metadata.create_all(engine)
            trans.commit()

            # Now stamp the latest alembic version
            from alembic.config import Config
            from alembic import command
            import os
            ini = os.environ.get('ICHNAEA_CFG', 'ichnaea.ini')
            alembic_ini = os.path.join(os.path.split(ini)[0], 'alembic.ini')
            alembic_cfg = Config(alembic_ini)
            command.stamp(alembic_cfg, "head")
            command.current(alembic_cfg)
    else:
        parser.print_help()
Example #17
0
def init():
    parser = argparse.ArgumentParser(description='Initialise a database')
    parser.add_argument('dburl', help='Database URL for SQLAlchemy')
    args = parser.parse_args()
    repositories.sa_pixort_data(url=args.dburl, create_schema=True)

    command.stamp(_get_config(args.dburl), "head")
Example #18
0
def main(argv=sys.argv):
    if len(argv) < 2:
        usage(argv)
    config_uri = argv[1]
    options = parse_vars(argv[2:])
    setup_logging(config_uri)
    settings = get_appsettings(config_uri, options=options)
    engine = engine_from_config(settings, 'sqlalchemy.')
    DBSession.configure(bind=engine)
    Base.metadata.create_all(engine)

    with transaction.manager:
        user = User(email='*****@*****.**', password='******', admin=True)
        DBSession.add(user)
        site = Site(name='asd', key='80d621df066348e5938a469730ae0cab')
        DBSession.add(site)
        site.api_keys.append(SiteAPIKey(key='GIKfxIcIHPbM0uX9PrQ1To29Pb2on0pa'))
        site.users.append(user)

        aspect_ratio_1_1 = SiteAspectRatio(width=1, height=1)
        aspect_ratio_3_1 = SiteAspectRatio(width=3, height=1)
        site.aspect_ratios.append(aspect_ratio_1_1)
        site.aspect_ratios.append(aspect_ratio_3_1)

    from alembic.config import Config
    from alembic import command
    alembic_cfg = Config("alembic.ini")
    command.stamp(alembic_cfg, "head")
Example #19
0
 def test_history_indicate_current(self):
     command.stamp(self.cfg, (self.b,))
     self.cfg.stdout = buf = self._buf_fixture()
     command.history(self.cfg, indicate_current=True, verbose=True)
     self._eq_cmd_output(
         buf, [self.c, self.b, self.a], currents=(self.b,), env_token=True
     )
Example #20
0
def main(argv=sys.argv):
    if len(argv) != 2:
        usage(argv)
    config_uri = argv[1]
    if config_uri not in ["jenkins.ini"]:
        print "You are not initialising using a testing URI"
        print "Edit init_testingdb.py if you really want to do this"
        sys.exit(0)
    setup_logging(config_uri)
    settings = get_appsettings(config_uri)
    engine = engine_from_config(settings, "sqlalchemy.")
    meta.Session.configure(bind=engine)
    Base.metadata.bind = engine

    # Create Everythin
    print "Dropping Tables"
    Base.metadata.drop_all(engine)
    print "Creating Tables"
    Base.metadata.create_all(engine)

    # We also want any alembic scripts to be executed
    alembic_cfg = Config(config_uri)  # TODO: WARNING RELATIVE PATH
    command.stamp(alembic_cfg, "head")

    DBSession = meta.Session()
    # DBSession.configure(bind=engine)

    populateData.init_data(DBSession)
    populateUser()
    populatedata()
Example #21
0
    def run(self, args):
        pysite.models.create_all()
        alembic_cfg = Config(args.alembic_config)
        command.stamp(alembic_cfg, "head")

        with transaction.manager:
            self._setup_users()
Example #22
0
def initdb(args, settings):
    print('Creating DB structure...')
    models.Base.metadata.create_all(DBSession.bind)

    from alembic.config import Config
    from alembic import command
    alembic_cfg = Config("./alembic.ini")
    command.stamp(alembic_cfg, "head")
    
    print('Add default data...')
    def try_add(obj):
        try:
            DBSession.add(obj)
            DBSession.commit()
        except sqlalchemy.exc.IntegrityError:
            DBSession.rollback()
            pass

    hosted_group = models.Group(name='hosted', description='Hosted users')
    support_group = models.Group(name='support', description='Support')
    admin_group = models.Group(name='admin', description='Administrator')
    try_add(hosted_group)
    try_add(support_group)
    try_add(admin_group)
        
    admin_user = models.User(username='******', groups=[admin_group])
    admin_user.set_password('admin')
    try_add(admin_user)
Example #23
0
def init(uri=None, alembic_ini=None, engine=None, create=False):
    """ Initialize a connection.  Create tables if requested."""

    if uri and engine:
        raise ValueError("uri and engine cannot both be specified")

    if uri is None and not engine:
        uri = 'sqlite:////tmp/datanommer.db'
        log.warning("No db uri given.  Using %r" % uri)

    if uri and not engine:
        engine = create_engine(uri)

    # We need to hang our own attribute on the sqlalchemy session to stop
    # ourselves from initializing twice.  That is only a problem is the code
    # calling us isn't consistent.
    if getattr(session, '_datanommer_initialized', None):
        log.warning("Session already initialized.  Bailing")
        return
    session._datanommer_initialized = True

    session.configure(bind=engine)
    DeclarativeBase.query = session.query_property()

    # Loads the alembic configuration and generates the version table, with
    # the most recent revision stamped as head
    if alembic_ini is not None:
        from alembic.config import Config
        from alembic import command
        alembic_cfg = Config(alembic_ini)
        command.stamp(alembic_cfg, "head")

    if create:
        DeclarativeBase.metadata.create_all(engine)
Example #24
0
def main(argv=sys.argv):
    args = parser.parse_args(argv[1:])

    setup_logging(args.config)
    app_settings = get_appsettings(args.config)
    alembic_cfg = Config(args.config)

    blame = alembic_cfg.get_main_option('blame')
    engine = create_engine(alembic_cfg.get_main_option('sqlalchemy.url'))
    apps = aslist(app_settings['occams.apps'])
    errors = []

    assert blame, 'Need to blame someone!'

    with engine.begin() as connection:
        connection.info['blame'] = blame

        for app in apps:
            try:
                module = import_module(app)
            except ImportError:
                errors.append('{}: Unable to import'.format(app))
                continue
            else:
                if hasattr(module, 'initdb'):
                    module.initdb(connection)
                else:
                    errors.append('{}: Does not have "initdb"'.format(app))

        for error in errors:
            print(error)

        alembic_cfg.attributes['connection'] = connection
        command.stamp(alembic_cfg, 'heads')
 def stamp(context, directory='migrations', revision='head', sql=False, tag=None):
     """'stamp' the revision table with the given revision; don't run any
     migrations"""
     from app import create_app
     with create_app().app_context():
         config = _get_config(directory)
         command.stamp(config, revision, sql=sql, tag=tag)
Example #26
0
def reset_all_dbs(config_hint):
    """
    모든 데이터 베이스를 리셋합니다. 만약에 대비해 패스워드를 확인합니다.
    전체 리셋 패스워드를 지정하지 않았다면 사용할 수 없습니다.
    """

    config_file_path = pm.smart_find_file_path(
        config_hint, base_dir_path=CONFIG_DIR_PATH)

    app = create_application(config_file_path)

    print "#### reset all databases"
    print "* database uri: %s" % app.config['SQLALCHEMY_DATABASE_URI']
    for key, value in sorted(app.config['SQLALCHEMY_BINDS'].iteritems()):
        print " * bind_key: %s uri:%s" % (key, value)

    print "* reset_all_password:"******"head")
Example #27
0
    def install_or_upgrade_db(self, skip_backup=False):
        from domogik.common import sql_schema
        from domogik.common import database
        from sqlalchemy import create_engine, MetaData, Table
        from alembic.config import Config
        from alembic import command

        info("Installing or upgrading the db")
        if not sql_schema.Device.__table__.exists(bind=self._engine):
            sql_schema.metadata.drop_all(self._engine)
            ok("Droping all existing tables...")
            sql_schema.metadata.create_all(self._engine)
            ok("Creating all tables...")
            with self._db.session_scope():
                self._db.add_default_user_account()
            ok("Creating admin user...")
            command.stamp(self.alembic_cfg, "head")
            ok("Setting db version to head")
        else:
            if not skip_backup:
                ok("Creating backup")
                self.backup_existing_database()
            ok("Upgrading")
            command.upgrade(self.alembic_cfg, "head")
        return 
Example #28
0
    def upgrade(self, nocreate=False, create_legacy_resource_types=False):
        from alembic import command
        from alembic import migration

        cfg = self._get_alembic_config()
        cfg.conf = self.conf
        if nocreate:
            command.upgrade(cfg, "head")
        else:
            with self.facade.writer_connection() as connection:
                ctxt = migration.MigrationContext.configure(connection)
                current_version = ctxt.get_current_revision()
                if current_version is None:
                    Base.metadata.create_all(connection)
                    command.stamp(cfg, "head")
                else:
                    command.upgrade(cfg, "head")

        # TODO(sileht): generic shouldn't be a particular case
        # we must create a rt_generic and rt_generic_history table
        # like other type
        for rt in base.get_legacy_resource_types():
            if not (rt.name == "generic" or create_legacy_resource_types):
                continue
            try:
                with self.facade.writer() as session:
                    session.add(rt)
            except exception.DBDuplicateEntry:
                pass
            with self.facade.writer_connection() as connection:
                self._RESOURCE_TYPE_MANAGER.map_and_create_tables(
                    rt, connection)
Example #29
0
 def _create_tables(self, conn_string):
     print('Creating tables')
     from server.models import db
     current_app.config['SQLALCHEMY_DATABASE_URI'] = conn_string
     try:
         db.create_all()
     except OperationalError as ex:
         if 'could not connect to server' in ex.message:
             print('ERROR: {red}PostgreSQL service{white} is not running. Please verify that it is running in port 5432 before executing setup script.'.format(red=Fore.RED, white=Fore.WHITE))
             sys.exit(1)
         elif 'password authentication failed' in ex.message:
             print('ERROR: ')
             sys.exit(1)
         else:
             raise
     except ProgrammingError as ex:
         print(ex)
         print('Please check postgres user permissions.')
         sys.exit(1)
     except ImportError as ex:
         if 'psycopg2' in ex:
             print(
                 'ERROR: Missing python depency {red}psycopg2{white}. Please install it with {blue}pip install psycopg2'.format(red=Fore.RED, white=Fore.WHITE, blue=Fore.BLUE))
             sys.exit(1)
         else:
             raise
     else:
         from alembic.config import Config
         from alembic import command
         alembic_cfg = Config(os.path.join(os.getcwd(), 'alembic.ini'))
         command.stamp(alembic_cfg, "head")
Example #30
0
def create_schema(engine, alembic_cfg, location_cfg):  # pragma: no cover
    old_version = False
    with engine.connect() as conn:
        trans = conn.begin()
        stmt = text('select version_num from alembic_version')
        try:
            result = conn.execute(stmt).fetchall()
            if len(result):
                old_version = True
        except ProgrammingError:
            pass

        if not old_version:
            _Model.metadata.create_all(engine)

        add_api_key(conn)
        add_export_config(conn)
        add_users(conn, location_cfg)

        trans.commit()

    # Now stamp the latest alembic version
    if not old_version:
        command.stamp(alembic_cfg, 'head')
    command.current(alembic_cfg)
Example #31
0
def stamp(config, revision='head', sql=False, tag=None):
    """'stamp' the revision table with the given revision; don't run any
    migrations"""
    command.stamp(config, revision, sql=sql, tag=tag)
def stamp(directory=None, revision='head', sql=False, tag=None):
    """'stamp' the revision table with the given revision; don't run any
    migrations"""
    config = current_app.extensions['migrate'].migrate.get_config(directory)
    command.stamp(config, revision, sql=sql, tag=tag)
Example #33
0
def init():
    engine = create_engine(SQLALCHEMY_DATABASE_URL)
    Base.metadata.create_all(engine)

    alembic_cfg = Config("/app/alembic.ini")
    command.stamp(alembic_cfg, "head")
Example #34
0
 print()
 sys.path.append(
     os.path.join(os.path.dirname(os.path.realpath(__file__)), '..'))
 import config
 from pynab.db import Base, engine, Session, User, Group, Category, TvShow, Movie
 import pynab.util
 from scripts import nzedb_pre_import
 db = Session()
 start = time.time()
 print('Building tables...')
 Base.metadata.drop_all(engine)
 Base.metadata.create_all(engine)
 from alembic.config import Config
 from alembic import command
 alembic_cfg = Config("alembic.ini")
 command.stamp(alembic_cfg, "head")
 print('Installing admin user...')
 with open('db/initial/users.json', encoding='utf-8', errors='ignore') as f:
     data = json.load(f)
     try:
         engine.execute(User.__table__.insert(), data)
     except Exception as e:
         print('Problem inserting data into database: {}'.format(e))
         sys.exit(0)
 print('Copying groups into db...')
 with open('db/initial/groups.json', encoding='utf-8',
           errors='ignore') as f:
     data = json.load(f)
     try:
         engine.execute(Group.__table__.insert(), data)
     except Exception as e:
Example #35
0
def stamp_alembic_rev(alembic_config, conn, rev='head', quiet=True):
    with quieten(quiet):
        alembic_config.attributes['connection'] = conn
        stamp(alembic_config, rev)
Example #36
0
def main():
    sys.stderr.write("create_tables.main: starting\n")
    start_time = time.time()
    from docassemble.webapp.database import dbprefix
    if dbprefix.startswith('postgresql') and not daconfig.get(
            'force text to varchar upgrade', False):
        do_varchar_upgrade = False
    else:
        do_varchar_upgrade = True
    with app.app_context():
        sys.stderr.write("create_tables.main: inside app context after " +
                         str(time.time() - start_time) + "\n")
        if daconfig.get('use alembic', True):
            sys.stderr.write("create_tables.main: running alembic after " +
                             str(time.time() - start_time) + "\n")
            insp = inspect(db.engine)
            if do_varchar_upgrade:
                changed = False
                if insp.has_table(dbtableprefix + 'userdict'):
                    db.session.execute(
                        delete(UserDict).where(
                            db.func.length(UserDict.filename) > 255).
                        execution_options(synchronize_session=False))
                    changed = True
                if insp.has_table(dbtableprefix + 'userdictkeys'):
                    db.session.execute(
                        delete(UserDictKeys).where(
                            db.func.length(UserDictKeys.filename) > 255).
                        execution_options(synchronize_session=False))
                    changed = True
                if insp.has_table(dbtableprefix + 'chatlog'):
                    db.session.execute(
                        delete(ChatLog).where(
                            db.func.length(ChatLog.filename) > 255).
                        execution_options(synchronize_session=False))
                    changed = True
                if insp.has_table(dbtableprefix + 'uploads'):
                    db.session.execute(
                        delete(Uploads).where(
                            db.func.length(Uploads.filename) > 255).
                        execution_options(synchronize_session=False))
                    db.session.execute(
                        delete(Uploads).where(
                            db.func.length(Uploads.yamlfile) > 255).
                        execution_options(synchronize_session=False))
                    changed = True
                if insp.has_table(dbtableprefix + 'objectstorage'):
                    db.session.execute(
                        delete(ObjectStorage).where(
                            db.func.length(ObjectStorage.key) > 1024).
                        execution_options(synchronize_session=False))
                    changed = True
                if insp.has_table(dbtableprefix + 'speaklist'):
                    db.session.execute(
                        delete(SpeakList).where(
                            db.func.length(SpeakList.filename) > 255).
                        execution_options(synchronize_session=False))
                    changed = True
                if insp.has_table(dbtableprefix + 'shortener'):
                    db.session.execute(
                        delete(Shortener).where(
                            db.func.length(Shortener.filename) > 255).
                        execution_options(synchronize_session=False))
                    db.session.execute(
                        delete(Shortener).where(
                            db.func.length(Shortener.key) > 255).
                        execution_options(synchronize_session=False))
                    changed = True
                if insp.has_table(dbtableprefix + 'machinelearning'):
                    db.session.execute(
                        delete(MachineLearning).where(
                            db.func.length(MachineLearning.key) > 1024).
                        execution_options(synchronize_session=False))
                    db.session.execute(
                        delete(MachineLearning).where(
                            db.func.length(MachineLearning.group_id) > 1024).
                        execution_options(synchronize_session=False))
                    changed = True
                if insp.has_table(dbtableprefix + 'globalobjectstorage'):
                    db.session.execute(
                        delete(GlobalObjectStorage).where(
                            db.func.length(GlobalObjectStorage.key) > 1024).
                        execution_options(synchronize_session=False))
                    changed = True
                if changed:
                    db.session.commit()
            packagedir = pkg_resources.resource_filename(
                pkg_resources.Requirement.parse('docassemble.webapp'),
                'docassemble/webapp')
            if not os.path.isdir(packagedir):
                sys.exit("path for running alembic could not be found")
            from alembic.config import Config
            from alembic import command
            alembic_cfg = Config(os.path.join(packagedir, 'alembic.ini'))
            alembic_cfg.set_main_option("sqlalchemy.url",
                                        alchemy_connection_string())
            alembic_cfg.set_main_option("script_location",
                                        os.path.join(packagedir, 'alembic'))
            if not insp.has_table(dbtableprefix + 'alembic_version'):
                sys.stderr.write(
                    "create_tables.main: creating alembic stamp\n")
                command.stamp(alembic_cfg, "head")
                sys.stderr.write(
                    "create_tables.main: done creating alembic stamp after " +
                    str(time.time() - start_time) + " seconds\n")
            if insp.has_table(dbtableprefix + 'user'):
                sys.stderr.write(
                    "create_tables.main: creating alembic stamp\n")
                sys.stderr.write(
                    "create_tables.main: running alembic upgrade\n")
                command.upgrade(alembic_cfg, "head")
                sys.stderr.write(
                    "create_tables.main: done running alembic upgrade after " +
                    str(time.time() - start_time) + " seconds\n")
        #db.drop_all()
        try:
            sys.stderr.write("create_tables.main: trying to create tables\n")
            db.create_all()
        except:
            sys.stderr.write(
                "create_tables.main: error trying to create tables; trying a second time.\n"
            )
            try:
                db.create_all()
            except:
                sys.stderr.write(
                    "create_tables.main: error trying to create tables; trying a third time.\n"
                )
                db.create_all()
        sys.stderr.write(
            "create_tables.main: finished creating tables after " +
            str(time.time() - start_time) + " seconds.\n")
        if dbprefix.startswith('postgresql'):
            try:
                test_for_errors(start_time=start_time)
            except:
                sys.stderr.write(
                    "create_tables.main: unable to test for errors after " +
                    str(time.time() - start_time) + " seconds.\n")
        populate_tables(start_time=start_time)
        db.engine.dispose()
Example #37
0
def stamp_alembic_rev(alembic_config, conn, rev="head", quiet=True):
    with _alembic_lock, quieten(quiet):
        alembic_config.attributes["connection"] = conn
        stamp(alembic_config, rev)
Example #38
0
 def stamp_db(self):
     if self.rev:
         with self.db.engine.connect() as conn:
             trans = conn.begin()
             alembic_command.stamp(self.alembic_config(), self.rev)
             trans.commit()
Example #39
0
def stamp(revision):
    """Fake a migration to a particular revision"""
    alembic_cfg = Config("alembic.ini")
    command.stamp(alembic_cfg, revision)
Example #40
0
def initialize(settings,options):
    engine = engine_from_config(settings, 'sqlalchemy.')
    
    config = Configurator(settings=settings)
    pyramid_dogpile_cache.includeme(config)
    
    from gengine.metadata import (
        init_session,
        init_declarative_base,
        init_db
    )
    init_caches()
    init_session()
    init_declarative_base()
    init_db(engine)
    
    from gengine.metadata import (
        Base,
        DBSession
    )

    if options.get("reset_db",False):
        Base.metadata.drop_all(engine)
        engine.execute("DROP SCHEMA IF EXISTS public CASCADE")

    engine.execute("CREATE SCHEMA IF NOT EXISTS public")

    from alembic.config import Config
    from alembic import command

    alembic_cfg = Config(attributes={
        'engine' : engine,
        'schema' : 'public'
    })
    script_location = os.path.join(
        os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
        'app/alembic'
    )
    alembic_cfg.set_main_option("script_location", script_location)

    do_upgrade = options.get("upgrade",False)
    if not do_upgrade:
        #init
        from gengine.app import model

        tables = [t for name, t in model.__dict__.items() if isinstance(t, Table)]
        Base.metadata.create_all(engine, tables=tables)

        command.stamp(alembic_cfg, "head")

        if options.get("populate_demo", False):
            populate_demo(DBSession)
    else:
        #upgrade
        command.upgrade(alembic_cfg,'head')

    admin_user = options.get("admin_user", False)
    admin_password = options.get("admin_password", False)

    if admin_user and admin_password:
        create_user(DBSession = DBSession, user=admin_user,password=admin_password)

    engine.dispose()
Example #41
0
 def tearDown(self):
     app = self.create_app()
     with app.app_context():
         command.stamp(self.alembic_configuration, "base")
         db_session.remove()
         Base.metadata.drop_all(bind=engine)
Example #42
0
    def main(self):

        database_name = self.config.database_name
        if not database_name:
            self.config.logger.error(
                '"database_name" cannot be an empty string')
            return 1

        # superuser credentials for overall database
        superuser_pg_url = self.create_connection_url(
            'postgres', self.config.database_superusername,
            self.config.database_superuserpassword)

        # superuser credentials for working database
        superuser_normaldb_pg_url = self.create_connection_url(
            database_name, self.config.database_superusername,
            self.config.database_superuserpassword)

        # normal user credentials
        normal_user_pg_url = self.create_connection_url(
            database_name, self.config.database_username,
            self.config.database_password)

        # ensure that if on Heroku the the normal_user_pg_url and the
        # superuser_pg_url are the same
        if self.config.on_heroku and (normal_user_pg_url != superuser_pg_url):
            self.config.logger.error(
                'there is no superuser (%s) when using Heroku',
                self.config.database_superusername)
            return 1

        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        # table logging section
        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        if self.config.unlogged:

            @compiles(CreateTable)
            def create_table(element, compiler, **kw):
                text = compiler.visit_create_table(element, **kw)
                text = re.sub("^\sCREATE(.*TABLE)",
                              lambda m: "CREATE UNLOGGED %s" % m.group(1),
                              text)
                return text

        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        # Postgres version check section
        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        self.config.logger.info('Postgres version check section with %s',
                                superuser_pg_url)
        with PostgreSQLAlchemyManager(superuser_pg_url,
                                      self.config.logger,
                                      autocommit=False) as db:
            if not db.min_ver_check(90200):
                self.config.logger.error('unrecognized PostgreSQL version: %s',
                                         db.version_string())
                self.config.logger.error('Only 9.2+ is supported at this time')
                return 1

        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        # drop database section
        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        # We can only do the following if the DB is not Heroku
        # XXX Might add the heroku commands for resetting a DB here
        if self.config.dropdb and not self.config.on_heroku:
            self.config.logger.info('drop database section with %s',
                                    superuser_pg_url)
            with PostgreSQLAlchemyManager(superuser_pg_url,
                                          self.config.logger,
                                          autocommit=False) as db:
                if 'test' not in database_name and not self.config.force:
                    confirm = raw_input('drop database %s [y/N]: ' %
                                        database_name)
                    if not confirm == "y":
                        self.config.logger.warn('NOT dropping table')
                        return 2
                db.drop_database(database_name)
                db.commit()

        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        # create database section
        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        if self.config.createdb:
            self.config.logger.info('create database section with %s',
                                    superuser_pg_url)
            with PostgreSQLAlchemyManager(superuser_pg_url,
                                          self.config.logger,
                                          autocommit=False) as db:
                db.create_database(database_name)
                if self.config.no_roles:
                    self.config.logger.info("Skipping role creation")
                else:
                    db.create_roles(self.config)
                db.commit()

        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        # database extensions section
        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        self.config.logger.info('database extensions section with %s',
                                superuser_normaldb_pg_url)
        with PostgreSQLAlchemyManager(superuser_normaldb_pg_url,
                                      self.config.logger,
                                      autocommit=False,
                                      on_heroku=self.config.on_heroku) as db:
            db.setup_extensions()
            db.grant_public_schema_ownership(self.config.database_username)
            db.commit()

        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        # database schema section
        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        if self.config.no_schema:
            self.config.logger.info("not adding a schema")
            return 0

        alembic_cfg = Config(self.config.alembic_config)
        alembic_cfg.set_main_option('sqlalchemy.url', normal_user_pg_url)

        self.config.logger.info('database schema section with %s',
                                normal_user_pg_url)
        with PostgreSQLAlchemyManager(normal_user_pg_url,
                                      self.config.logger,
                                      autocommit=False,
                                      on_heroku=self.config.on_heroku) as db:
            # Order matters below
            db.turn_function_body_checks_off()
            db.load_raw_sql('types')
            db.load_raw_sql('procs')
            # We need to commit to make a type visible for table creation
            db.commit()

            db.create_tables()
            db.load_raw_sql('views')
            db.commit()

            if not self.config.get('no_staticdata'):
                self.import_staticdata(db)
            if self.config['fakedata']:
                self.generate_fakedata(db, self.config['fakedata_days'])
            db.commit()
            command.stamp(alembic_cfg, "heads")
            db.session.close()

        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        # database owner section
        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        self.config.logger.info('database extensions section with %s',
                                superuser_normaldb_pg_url)
        with PostgreSQLAlchemyManager(superuser_normaldb_pg_url,
                                      self.config.logger,
                                      autocommit=False,
                                      on_heroku=self.config.on_heroku) as db:
            db.set_table_owner(self.config.database_username)
            db.set_default_owner(database_name, self.config.database_username)
            db.set_grants(self.config)  # config has user lists

        return 0
Example #43
0
 def setUp(self):
     command.stamp(self.cfg, "base")
Example #44
0
def test_stamp():
    with capture_context_buffer() as buf:
        command.stamp(cfg, "head", sql=True)
    assert "UPDATE alembic_version SET version_num='%s';" % c in buf.getvalue()
Example #45
0
 def test_stamp_creates_table(self):
     command.stamp(self.cfg, "head")
     eq_(
         self.bind.scalar("select version_num from alembic_version"), self.b
     )
Example #46
0
 def test_stamp_existing_downgrade(self):
     command.stamp(self.cfg, self.b)
     command.stamp(self.cfg, self.a)
     eq_(
         self.bind.scalar("select version_num from alembic_version"), self.a
     )
Example #47
0
 def test_heads_one_is_dependent(self):
     command.stamp(self.cfg, ())
     command.stamp(self.cfg, (self.b2.revision,))
     with self._assert_lines(["a2", "b2"]):
         command.current(self.cfg)
Example #48
0
 def test_heads_upg(self):
     command.stamp(self.cfg, (self.b2.revision,))
     command.upgrade(self.cfg, (self.b3.revision))
     with self._assert_lines(["a2", "b3"]):
         command.current(self.cfg)
Example #49
0
 def test_plain_current(self):
     command.stamp(self.cfg, ())
     command.stamp(self.cfg, self.a3.revision)
     with self._assert_lines(["a3"]):
         command.current(self.cfg)
Example #50
0
 def test_two_heads(self):
     command.stamp(self.cfg, ())
     command.stamp(self.cfg, (self.a1.revision, self.b1.revision))
     with self._assert_lines(["a1", "b1"]):
         command.current(self.cfg)
Example #51
0
 def test_history_current_to_head_as_base(self):
     command.stamp(self.cfg, "base")
     self.cfg.stdout = buf = self._buf_fixture()
     command.history(self.cfg, "current:", verbose=True)
     self._eq_cmd_output(buf, [self.c, self.b, self.a], env_token=True)
Example #52
0
 def test_no_current(self):
     command.stamp(self.cfg, ())
     with self._assert_lines([]):
         command.current(self.cfg)
Example #53
0
def stamp_db(revision, dburl):
    command.stamp(alembic_config(dburl=dburl), revision)
def initialize_database(engine: Engine):
    import_all_modules()
    Base.metadata.create_all(engine, checkfirst=False)
    alembic_cfg = get_alembic_config(engine)
    stamp(alembic_cfg, 'head')
Example #55
0
 def stamp_metadata_database(self):
     alembic_cfg = self.get_alembic_config()
     if self.engine is not None:
         alembic_cfg.attributes["connection"] = self.engine
     command.stamp(alembic_cfg, "23dd1cc88eb2")
Example #56
0
def initialize_db(alembic_config) -> None:
    log.info('[•] Initializing DB...')
    Base.metadata.create_all(bind=engine, checkfirst=True)
    command.stamp(alembic_config, 'head')
    log.info('[✔] DB created ({path})'.format(path=DB.SQLALCHEMY_DATABASE_URI))
Example #57
0
def setup_alembic_config():
    alembic_cfg = Config("alembic.ini")
    command.stamp(alembic_cfg, "head")
Example #58
0
def initialize(args):
    """
    This command exists to:

    - Prevent the user having to type more than one thing
    - Prevent the user seeing internals like 'manage.py' which we would
      rather people were not messing with on production systems.
    """
    log.info("Loading configuration..")
    config = CalamariConfig()

    # Generate django's SECRET_KEY setting
    # Do this first, otherwise subsequent django ops will raise ImproperlyConfigured.
    # Write into a file instead of directly, so that package upgrades etc won't spuriously
    # prompt for modified config unless it really is modified.
    if not os.path.exists(config.get('calamari_web', 'secret_key_path')):
        chars = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)'
        open(config.get('calamari_web', 'secret_key_path'),
             'w').write(get_random_string(50, chars))

    run_local_salt(sls=RELAX_SALT_PERMS_SLS, message='salt')
    run_local_salt(sls=POSTGRES_SLS, message='postgres')

    # Cthulhu's database
    db_path = config.get('cthulhu', 'db_path')
    engine = create_engine(db_path)
    Base.metadata.reflect(engine)
    alembic_config = AlembicConfig()
    if ALEMBIC_TABLE in Base.metadata.tables:
        log.info("Updating database...")
        # Database already populated, migrate forward
        command.upgrade(alembic_config, "head")
    else:
        log.info("Initializing database...")
        # Blank database, do initial population
        Base.metadata.create_all(engine)
        command.stamp(alembic_config, "head")

    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "calamari_web.settings")

    # Django's database
    with quiet():
        execute_from_command_line(["", "syncdb", "--noinput"])

    create_default_roles()
    create_admin_users(args)
    log.info("Initializing web interface...")

    # Django's static files
    with quiet():
        execute_from_command_line(["", "collectstatic", "--noinput"])

    # Because we've loaded Django, it will have written log files as
    # this user (probably root).  Fix it so that apache can write them later.
    apache_user = pwd.getpwnam(config.get('calamari_web', 'username'))
    os.chown(config.get('calamari_web', 'log_path'), apache_user.pw_uid,
             apache_user.pw_gid)

    # Handle SQLite case, otherwise no chown is needed
    if config.get('calamari_web', 'db_engine').endswith("sqlite3"):
        os.chown(config.get('calamari_web', 'db_name'), apache_user.pw_uid,
                 apache_user.pw_gid)

    # Start services, configure to run on boot
    run_local_salt(sls=SERVICES_SLS, message='services')

    # During an upgrade: update minions that were connected previously
    update_connected_minions()

    # Signal supervisor to restart cthulhu as we have created its database
    log.info("Restarting services...")
    subprocess.call(['supervisorctl', 'restart', 'cthulhu'],
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE)
    subprocess.call(['supervisorctl', 'restart', 'carbon-cache'],
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE)

    # TODO: optionally generate or install HTTPS certs + hand to apache
    log.info("Complete.")
Example #59
0
def stamp(to_version='head', sql_url=None):
    """Stamp the specified version, with no migration performed."""
    alembic_cfg = init_config(sql_url)
    if alembic_cfg:
        alembic_command.stamp(alembic_cfg, to_version)
Example #60
0
    def install(self, req, sess, allow_upgrades=False):
        """
		Run module's installation hooks and register the module in DB.
		"""
        from alembic import command
        from netprofile_core.models import NPModule

        if not isinstance(req, Requirement):
            req = Requirement(req)
        moddef = req.name
        modspec = req.specifier
        if ('core' not in self.loaded) and (moddef != 'core'):
            raise ModuleError(
                'Unable to install anything prior to loading core module.')
        if self.is_installed(req, sess):
            if allow_upgrades and self.is_installed(Requirement(moddef), sess):
                return self.upgrade(req, sess)
            return False

        ep = self._find_ep(moddef)
        try:
            modcls = ep.load()
        except ImportError as e:
            raise ModuleError(
                'Can\'t locate ModuleBase class for module \'%s\'.' %
                (moddef, )) from e

        modversion = self._cls_version(modcls)
        if modversion not in modspec:
            raise ModuleError(
                'Module \'%s\' version %s can\'t satisfy requirements: %s.' %
                (moddef, str(modversion), str(modspec)))
        vpair = VersionPair(None, modversion)

        get_deps = getattr(modcls, 'get_deps', None)
        if callable(get_deps):
            for dep in get_deps():
                depreq = Requirement(dep)
                if not self.is_installed(depreq, sess):
                    self.install(depreq, sess, allow_upgrades=allow_upgrades)

        modprep = getattr(modcls, 'prepare', None)
        if callable(modprep):
            modprep()

        if moddef != 'core':
            self.preload(moddef)

        get_models = getattr(modcls, 'get_models', None)
        if callable(get_models):
            tables = [model.__table__ for model in get_models()]
            Base.metadata.create_all(sess.bind, tables)

        get_sql_functions = getattr(modcls, 'get_sql_functions', None)
        if callable(get_sql_functions):
            for func in get_sql_functions():
                sess.execute(func.create(moddef))

        get_sql_views = getattr(modcls, 'get_sql_views', None)
        if callable(get_sql_views):
            for view in get_sql_views():
                sess.execute(view.create())

        modobj = NPModule(id=None)
        modobj.name = moddef
        modobj.current_version = str(modversion)
        if moddef == 'core':
            modobj.enabled = True
        sess.add(modobj)
        sess.flush()

        get_sql_data = getattr(modcls, 'get_sql_data', None)
        if callable(get_sql_data):
            get_sql_data(modobj, vpair, sess)

        mod_install = getattr(modcls, 'install', None)
        if callable(mod_install):
            mod_install(sess)

        if self.installed is None:
            self.installed = dict()
        self.installed[moddef] = modversion
        transaction.commit()

        get_sql_events = getattr(modcls, 'get_sql_events', None)
        if callable(get_sql_events):
            for evt in get_sql_events():
                sess.execute(evt.create(moddef))
            transaction.commit()

        alembic_cfg = get_alembic_config(self, stdout=self.stdout)
        command.stamp(alembic_cfg, moddef + '@head')
        transaction.commit()

        if moddef == 'core':
            self.load(moddef)
        return True