Exemple #1
0
    def __init__(self, cfg_manager):
        engine   = cfg_manager.get_value(COORDINATOR_DB_ENGINE,  DEFAULT_COORDINATOR_DB_ENGINE)
        username = CoordinationDatabaseManager.username = cfg_manager.get_value(COORDINATOR_DB_USERNAME) # REQUIRED!
        password = CoordinationDatabaseManager.password = cfg_manager.get_value(COORDINATOR_DB_PASSWORD) # REQUIRED!
        host     = CoordinationDatabaseManager.host     = cfg_manager.get_value(COORDINATOR_DB_HOST,    DEFAULT_COORDINATOR_DB_HOST)
        dbname   = CoordinationDatabaseManager.dbname   = cfg_manager.get_value(COORDINATOR_DB_NAME,    DEFAULT_COORDINATOR_DB_NAME)

        if CoordinationDatabaseManager.engine is None or cfg_manager.get_doc_value(WEBLAB_DB_FORCE_ENGINE_CREATION):
            getconn = generate_getconn(engine, username, password, host, dbname)

            connect_args = {}

            if engine == 'sqlite':
                sqlalchemy_engine_str = 'sqlite:///%s' % get_sqlite_dbname(dbname)
                if dbname == ':memory:':
                    connect_args['check_same_thread'] = False
                    pool = sqlalchemy.pool.StaticPool(getconn)
                else:
                    pool = sqlalchemy.pool.NullPool(getconn)
            else:
                sqlalchemy_engine_str = "%s://%s:%s@%s/%s" % (engine, username, password, host, dbname)

                pool = sqlalchemy.pool.QueuePool(getconn, pool_size=15, max_overflow=20, recycle=3600)

            CoordinationDatabaseManager.engine = sqlalchemy.create_engine(sqlalchemy_engine_str, convert_unicode=True, echo=False, connect_args = connect_args, pool = pool)

            if engine == 'sqlite' and dbname == ':memory:':
                coord_model.load()
                metadata = coord_model.Base.metadata
                metadata.drop_all(self.engine)
                metadata.create_all(self.engine)


        self.session_maker = sessionmaker(bind=self.engine, autoflush = True, autocommit = False)
Exemple #2
0
    def __init__(self, cfg_manager):
        engine = cfg_manager.get_doc_value(
            configuration_doc.COORDINATOR_DB_ENGINE)
        username = CoordinationDatabaseManager.username = cfg_manager.get_doc_value(
            configuration_doc.COORDINATOR_DB_USERNAME)
        password = CoordinationDatabaseManager.password = cfg_manager.get_doc_value(
            configuration_doc.COORDINATOR_DB_PASSWORD)
        host = CoordinationDatabaseManager.host = cfg_manager.get_doc_value(
            configuration_doc.COORDINATOR_DB_HOST)
        port = CoordinationDatabaseManager.port = cfg_manager.get_doc_value(
            configuration_doc.COORDINATOR_DB_PORT)
        dbname = CoordinationDatabaseManager.dbname = cfg_manager.get_doc_value(
            configuration_doc.COORDINATOR_DB_NAME)

        if CoordinationDatabaseManager.engine is None or cfg_manager.get_doc_value(
                configuration_doc.DB_FORCE_ENGINE_CREATION):
            getconn = generate_getconn(engine, username, password, host, port,
                                       dbname)

            connect_args = {}

            if engine == 'sqlite':
                sqlalchemy_engine_str = 'sqlite:///%s' % get_sqlite_dbname(
                    dbname)
                if dbname == ':memory:':
                    connect_args['check_same_thread'] = False
                    pool = sqlalchemy.pool.StaticPool(getconn)
                else:
                    pool = sqlalchemy.pool.NullPool(getconn)
            else:
                if port is None:
                    port_str = ''
                else:
                    port_str = ':%s' % port
                sqlalchemy_engine_str = "%s://%s:%s@%s%s/%s" % (
                    engine, username, password, host, port_str, dbname)

                pool = sqlalchemy.pool.QueuePool(getconn,
                                                 pool_size=15,
                                                 max_overflow=20,
                                                 recycle=3600)

            CoordinationDatabaseManager.engine = sqlalchemy.create_engine(
                sqlalchemy_engine_str,
                convert_unicode=True,
                echo=False,
                connect_args=connect_args,
                pool=pool)

            if engine == 'sqlite' and dbname == ':memory:':
                coord_model.load()
                metadata = coord_model.Base.metadata
                metadata.drop_all(self.engine)
                metadata.create_all(self.engine)

        self.session_maker = sessionmaker(bind=self.engine,
                                          autoflush=True,
                                          autocommit=False)
Exemple #3
0
def deploy_testdb(options):
    from weblab.admin.deploy import insert_required_initial_data, populate_weblab_tests, generate_create_database
    import weblab.db.model as Model
    import weblab.core.coordinator.sql.model as CoordinatorModel

    import voodoo.sessions.db_lock_data as DbLockData
    import voodoo.sessions.sqlalchemy_data as SessionSqlalchemyData

    from sqlalchemy import create_engine
    
    try:
        import MySQLdb
        dbi = MySQLdb
    except ImportError:
        try:
            import pymysql_sa
        except ImportError:
            raise Exception("Neither MySQLdb nor pymysql have been installed. First install them by running 'pip install pymysql' or 'pip install python-mysql'")
        pymysql_sa.make_default_mysql_dialect()

    t_initial = time.time()
    
    db_dir = 'db'
    if not os.path.exists(db_dir):
        os.mkdir(db_dir)

    db_engine                = options.testdb_engine
    weblab_db_username       = options.testdb_user
    weblab_db_password       = options.testdb_passwd
    weblab_admin_db_username = options.testdb_admin_user
    weblab_admin_db_password = options.testdb_admin_passwd

    if db_engine == 'mysql':
        weblab_test_db_str = 'mysql://%s:%s@localhost/WebLabTests%s'         % (weblab_db_username, weblab_db_password,'%s')
        weblab_coord_db_str = 'mysql://%s:%s@localhost/WebLabCoordination%s' % (weblab_db_username, weblab_db_password, '%s')
        weblab_sessions_db_str = 'mysql://%s:%s@localhost/WebLabSessions'    % (weblab_db_username, weblab_db_password)
    elif db_engine == 'sqlite':
        weblab_test_db_str = 'sqlite:///db/WebLabTests%s.db'
        weblab_coord_db_str = 'sqlite:///db/WebLabCoordination%s.db'
        weblab_sessions_db_str = 'sqlite:///db/WebLabSessions.db'
    else:
        raise Exception("db engine %s not supported" % db_engine)

    if options.testdb_create_db:
        create_database = generate_create_database(db_engine)
        if create_database is None:
            raise Exception("db engine %s not supported for creating database" % db_engine)

        t = time.time()

        error_message = 'Could not create database. This may happen if the admin db credentials are wrong.'

        create_database(error_message, weblab_admin_db_username, weblab_admin_db_password, "WebLab",              weblab_db_username, weblab_db_password, db_dir = db_dir)
        create_database(error_message, weblab_admin_db_username, weblab_admin_db_password, "WebLabTests",         weblab_db_username, weblab_db_password, db_dir = db_dir)
        create_database(error_message, weblab_admin_db_username, weblab_admin_db_password, "WebLabTests2",        weblab_db_username, weblab_db_password, db_dir = db_dir)
        create_database(error_message, weblab_admin_db_username, weblab_admin_db_password, "WebLabTests3",        weblab_db_username, weblab_db_password, db_dir = db_dir)

        create_database(error_message, weblab_admin_db_username, weblab_admin_db_password, "WebLabCoordination",  weblab_db_username, weblab_db_password, db_dir = db_dir)
        create_database(error_message, weblab_admin_db_username, weblab_admin_db_password, "WebLabCoordination2", weblab_db_username, weblab_db_password, db_dir = db_dir)
        create_database(error_message, weblab_admin_db_username, weblab_admin_db_password, "WebLabCoordination3", weblab_db_username, weblab_db_password, db_dir = db_dir)
        create_database(error_message, weblab_admin_db_username, weblab_admin_db_password, "WebLabSessions",      weblab_db_username, weblab_db_password, db_dir = db_dir)

        print "Databases created.\t\t\t\t[done] [%1.2fs]" % (time.time() - t)
    
    #####################################################################
    # 
    # Populating main database
    #
    for tests in ('','2','3'):
        print "Populating 'WebLabTests%s' database...  \t\t" % tests, 
        t = time.time()

        engine = create_engine(weblab_test_db_str % tests, echo = False)
        metadata = Model.Base.metadata
        metadata.drop_all(engine)
        metadata.create_all(engine)

        insert_required_initial_data(engine)
        populate_weblab_tests(engine, tests)

        print "[done] [%1.2fs]" % (time.time() - t)

    #####################################################################
    # 
    # Populating Coordination database
    # 

    for coord in ('','2','3'):
        print "Populating 'WebLabCoordination%s' database...\t" % coord,
        t = time.time()

        engine = create_engine(weblab_coord_db_str % coord, echo = False)

        CoordinatorModel.load()

        metadata = CoordinatorModel.Base.metadata
        metadata.drop_all(engine)
        metadata.create_all(engine)    

        print "[done] [%1.2fs]" % (time.time() - t)


    #####################################################################
    # 
    # Populating Sessions database
    # 


    print "Populating 'WebLabSessions' database...\t\t",
    t = time.time()

    engine = create_engine(weblab_sessions_db_str, echo = False)

    metadata = DbLockData.SessionLockBase.metadata
    metadata.drop_all(engine)
    metadata.create_all(engine)    

    metadata = SessionSqlalchemyData.SessionBase.metadata
    metadata.drop_all(engine)
    metadata.create_all(engine)   

    print "[done] [%1.2fs]" % (time.time() - t)

    print "Total database deployment: \t\t\t[done] [%1.2fs]" % (time.time() - t_initial)
Exemple #4
0
def deploy_testdb(options):
    from weblab.admin.deploy import insert_required_initial_data, populate_weblab_tests, generate_create_database
    import weblab.db.model as Model
    import weblab.core.coordinator.sql.model as CoordinatorModel

    import voodoo.sessions.db_lock_data as DbLockData
    import voodoo.sessions.sqlalchemy_data as SessionSqlalchemyData

    from sqlalchemy import create_engine

    try:
        import MySQLdb
        dbi = MySQLdb
    except ImportError:
        try:
            import pymysql_sa
        except ImportError:
            raise Exception(
                "Neither MySQLdb nor pymysql have been installed. First install them by running 'pip install pymysql' or 'pip install python-mysql'"
            )
        pymysql_sa.make_default_mysql_dialect()

    t_initial = time.time()

    db_dir = 'db'
    if not os.path.exists(db_dir):
        os.mkdir(db_dir)

    db_engine = options.testdb_engine
    weblab_db_username = options.testdb_user
    weblab_db_password = options.testdb_passwd
    weblab_admin_db_username = options.testdb_admin_user
    weblab_admin_db_password = options.testdb_admin_passwd

    if db_engine == 'mysql':
        weblab_test_db_str = 'mysql://%s:%s@localhost/WebLabTests%s' % (
            weblab_db_username, weblab_db_password, '%s')
        weblab_coord_db_str = 'mysql://%s:%s@localhost/WebLabCoordination%s' % (
            weblab_db_username, weblab_db_password, '%s')
        weblab_sessions_db_str = 'mysql://%s:%s@localhost/WebLabSessions' % (
            weblab_db_username, weblab_db_password)
    elif db_engine == 'sqlite':
        weblab_test_db_str = 'sqlite:///db/WebLabTests%s.db'
        weblab_coord_db_str = 'sqlite:///db/WebLabCoordination%s.db'
        weblab_sessions_db_str = 'sqlite:///db/WebLabSessions.db'
    else:
        raise Exception("db engine %s not supported" % db_engine)

    if options.testdb_create_db:
        create_database = generate_create_database(db_engine)
        if create_database is None:
            raise Exception(
                "db engine %s not supported for creating database" % db_engine)

        t = time.time()

        error_message = 'Could not create database. This may happen if the admin db credentials are wrong.'

        create_database(error_message,
                        weblab_admin_db_username,
                        weblab_admin_db_password,
                        "WebLab",
                        weblab_db_username,
                        weblab_db_password,
                        db_dir=db_dir)
        create_database(error_message,
                        weblab_admin_db_username,
                        weblab_admin_db_password,
                        "WebLabTests",
                        weblab_db_username,
                        weblab_db_password,
                        db_dir=db_dir)
        create_database(error_message,
                        weblab_admin_db_username,
                        weblab_admin_db_password,
                        "WebLabTests2",
                        weblab_db_username,
                        weblab_db_password,
                        db_dir=db_dir)
        create_database(error_message,
                        weblab_admin_db_username,
                        weblab_admin_db_password,
                        "WebLabTests3",
                        weblab_db_username,
                        weblab_db_password,
                        db_dir=db_dir)

        create_database(error_message,
                        weblab_admin_db_username,
                        weblab_admin_db_password,
                        "WebLabCoordination",
                        weblab_db_username,
                        weblab_db_password,
                        db_dir=db_dir)
        create_database(error_message,
                        weblab_admin_db_username,
                        weblab_admin_db_password,
                        "WebLabCoordination2",
                        weblab_db_username,
                        weblab_db_password,
                        db_dir=db_dir)
        create_database(error_message,
                        weblab_admin_db_username,
                        weblab_admin_db_password,
                        "WebLabCoordination3",
                        weblab_db_username,
                        weblab_db_password,
                        db_dir=db_dir)
        create_database(error_message,
                        weblab_admin_db_username,
                        weblab_admin_db_password,
                        "WebLabSessions",
                        weblab_db_username,
                        weblab_db_password,
                        db_dir=db_dir)

        print "Databases created.\t\t\t\t[done] [%1.2fs]" % (time.time() - t)

    #####################################################################
    #
    # Populating main database
    #
    for tests in ('', '2', '3'):
        print "Populating 'WebLabTests%s' database...  \t\t" % tests,
        t = time.time()

        engine = create_engine(weblab_test_db_str % tests, echo=False)
        metadata = Model.Base.metadata
        metadata.drop_all(engine)
        metadata.create_all(engine)

        insert_required_initial_data(engine)
        populate_weblab_tests(engine, tests)

        print "[done] [%1.2fs]" % (time.time() - t)

    #####################################################################
    #
    # Populating Coordination database
    #

    for coord in ('', '2', '3'):
        print "Populating 'WebLabCoordination%s' database...\t" % coord,
        t = time.time()

        engine = create_engine(weblab_coord_db_str % coord, echo=False)

        CoordinatorModel.load()

        metadata = CoordinatorModel.Base.metadata
        metadata.drop_all(engine)
        metadata.create_all(engine)

        print "[done] [%1.2fs]" % (time.time() - t)

    #####################################################################
    #
    # Populating Sessions database
    #

    print "Populating 'WebLabSessions' database...\t\t",
    t = time.time()

    engine = create_engine(weblab_sessions_db_str, echo=False)

    metadata = DbLockData.SessionLockBase.metadata
    metadata.drop_all(engine)
    metadata.create_all(engine)

    metadata = SessionSqlalchemyData.SessionBase.metadata
    metadata.drop_all(engine)
    metadata.create_all(engine)

    print "[done] [%1.2fs]" % (time.time() - t)

    print "Total database deployment: \t\t\t[done] [%1.2fs]" % (time.time() -
                                                                t_initial)