def _do_convert_db_in_place(self, dst_dburi): dbm = DatabaseManager(self.env) src_dburi = dbm.connection_uri if src_dburi == dst_dburi: printferr("Source database and destination database are same: %s", dst_dburi) return 1 env_path = mkdtemp(prefix='convert_db-', dir=os.path.dirname(self.env.path)) try: dst_env = self._create_env(env_path, dst_dburi) src_db = dbm.get_connection() dst_db = DatabaseManager(dst_env).get_connection() self._copy_tables(dst_env, src_db, dst_db, src_dburi, dst_dburi) del src_db del dst_db dst_env.shutdown() dst_env = None schema, params = parse_connection_uri(dst_dburi) if schema == 'sqlite': dbpath = os.path.join(self.env.path, params['path']) dbdir = os.path.dirname(dbpath) if not os.path.isdir(dbdir): os.makedirs(dbdir) shutil.copy(os.path.join(env_path, params['path']), dbpath) finally: shutil.rmtree(env_path) backup_config_file(self.env, '.convert_db-%d' % int(time.time())) self.config.set('trac', 'database', dst_dburi) self.config.save()
def engine(env): engine = _engines.get(env) if engine is None: schema = DatabaseManager(env).connection_uri.split(':')[0] echo = env.config.get('logging', 'log_level').lower() == 'debug' def connect(): cnx = env.get_db_cnx().cnx while isinstance(cnx, ConnectionWrapper): cnx = cnx.cnx return cnx engine = create_engine(URL(schema), poolclass=DummyPool, creator=connect, echo=echo) if echo: # make sqlalchemy log to trac's logger if hasattr(env, 'get_logger'): engine.logger = env.get_logger(__name__) else: engine.logger = env.log _engines[env] = engine try: return engine.begin() except AttributeError: # Older SQLAlchemy return engine
def do_upgrade(env, ver, cursor): """Add an auto-increment primary key to `node_change` table and indices (repos, rev, path) and (repos, path, rev) (#3676). """ db_connector, _ = DatabaseManager(env)._get_connector() table = Table('node_change', key='id')[Column('id', auto_increment=True), Column('repos', type='int'), Column('rev', key_size=40), Column('path', key_size=255), Column('node_type', size=1), Column('change_type', size=1), Column('base_path'), Column('base_rev'), Index(['repos', 'rev', 'path']), Index(['repos', 'path', 'rev'])] with env.db_transaction: cursor.execute("""CREATE TEMPORARY TABLE node_change_old AS SELECT * FROM node_change""") cursor.execute("DROP TABLE node_change") for stmt in db_connector.to_sql(table): cursor.execute(stmt) cursor.execute("""\ INSERT INTO node_change (repos,rev,path,node_type,change_type,base_path,base_rev) SELECT repos,rev,path,node_type,change_type,base_path,base_rev FROM node_change_old""") cursor.execute("DROP TABLE node_change_old")
def upgrade(self, backup=False, backup_dest=None): """Upgrade database. :param backup: whether or not to backup before upgrading :param backup_dest: name of the backup file :return: whether the upgrade was performed """ upgraders = [] for participant in self.setup_participants: if participant.environment_needs_upgrade(): upgraders.append(participant) if not upgraders: return if backup: try: self.backup(backup_dest) except Exception as e: raise BackupError(e) for participant in upgraders: self.log.info("%s.%s upgrading...", participant.__module__, participant.__class__.__name__) participant.upgrade_environment() # Database schema may have changed, so close all connections DatabaseManager(self).shutdown() del self.database_version return True
def do_upgrade(env, ver, cursor): """Changes to subscription db table: - 'subscriptions.destination', 'subscriptions.format' + 'subscriptions.authenticated', 'subscriptions.transport' 'subscriptions.managed' type='int' --> (default == char) """ cursor.execute(""" CREATE TEMPORARY TABLE subscriptions_old AS SELECT * FROM subscriptions """) cursor.execute("DROP TABLE subscriptions") connector = DatabaseManager(env).get_connector()[0] for table in schema: for stmt in connector.to_sql(table): cursor.execute(stmt) cursor.execute(""" INSERT INTO subscriptions (sid,authenticated,enabled,managed, realm,category,rule,transport) SELECT o.sid,s.authenticated,o.enabled,'watcher', o.realm,o.category,rule,'email' FROM subscriptions_old AS o LEFT JOIN session AS s ON o.sid=s.sid """) cursor.execute("DROP TABLE subscriptions_old")
def wiki_setup(tc): tc.env = EnvironmentStub(default_data=True, enable=['trac.*', 'tractags.*']) tc.env.path = tempfile.mkdtemp() tc.db_mgr = DatabaseManager(tc.env) tc.db = tc.env.get_db_cnx() TagSetup(tc.env).upgrade_environment(tc.db) now = datetime.now(utc) wiki = WikiPage(tc.env) wiki.name = 'TestPage' wiki.text = '--' wiki.save('joe', 'TagsPluginTestPage', '::1', now) req = Mock(href=Href('/'), abs_href=Href('http://www.example.com/'), authname='anonymous', perm=MockPerm(), tz=utc, args={}, locale=Locale.parse('en_US') if Locale else None) tc.env.href = req.href tc.env.abs_href = req.abs_href tc.context = Context.from_request(req) # Enable big diff output. tc.maxDiff = None
def reset_mysql_db(env, db_prop): """Deletes all data from the tables and resets autoincrement indexes. :since 1.1.3: deprecated and will be removed in 1.3.1. Use `reset_tables` from the database connection class instead. """ return DatabaseManager(env).reset_tables()
def do_upgrade(env, version, cursor): """Add `description` column to `enum` table.""" new_schema = [ Table('enum', key=('type', 'name'))[Column('type'), Column('name'), Column('value'), Column('description'), ] ] with env.db_transaction: DatabaseManager(env).upgrade_tables(new_schema) failures = [] for id_ in [1, 2, 3, 4, 5, 7, 8]: try: r = Report(env, id_) except ResourceNotFound: pass else: query = replace_sql_fragment(r.query) if query: r.query = query r.update() else: failures.append(unicode(id_)) if failures: printout( _("""\ Report(s) %(ids)s could not be upgraded and may need to be manually edited to avoid an "ambiguous column name" error. See %(url)s for more information. """, ids=', '.join(failures), url=url))
def upgrade_environment(self, db): db_connector, _ = DatabaseManager(self.env).get_connector() cursor = db.cursor() dbver = self.get_db_version() if dbver == 0: self.env.log.info("Initialize %s database schema to version %s", PLUGIN_NAME, PLUGIN_VERSION) for table in SCHEMA: for stmt in db_connector.to_sql(table): cursor.execute(stmt) cursor.execute( """ INSERT INTO system (name, value) VALUES (%s, %s) """, (PLUGIN_NAME, PLUGIN_VERSION)) else: while dbver != PLUGIN_VERSION: dbver = dbver + 1 self.env.log.info("Upgrade %s database schema to version %s", PLUGIN_NAME, dbver) modulename = 'db%i' % dbver upgrades = __import__('cards.upgrades', globals(), locals(), [modulename]) script = getattr(upgrades, modulename) script.do_upgrade(self.env, dbver, cursor) cursor.execute( """ UPDATE system SET value=%s WHERE name=%s """, (PLUGIN_VERSION, PLUGIN_NAME))
def upgrade_environment(self, db=None): dbm = DatabaseManager(self.env) if dbm.get_database_version(DB_NAME) is False: dbm.create_tables(schema) dbm.set_database_version(DB_VERSION, DB_NAME) else: dbm.upgrade(DB_VERSION, DB_NAME, 'coderev.upgrades')
def trac_db_query(env): """ Context manager to handle database connection and cursor from trac environment's read only connection. This does not attempt to roll back or commit, the connection is meant only for accessing the data. For examples on use, see db_query(). Internally, this uses Trac's connection pool via the Trac DatabaseManager class :param Environment env: The Trac environment """ conn = None cursor = None try: dm = DatabaseManager(env) conn = dm.get_connection() except Exception: env.log.exception( "Failed to get database connection from trac database manager") raise try: # NOTE: Trac's connection does not support alternative cursor types! cursor = conn.cursor() yield cursor except Exception: env.log.error("Failed to query database") raise finally: cursor.close() conn.close()
def setUp(self): self.env = EnvironmentStub() self.dbm = DatabaseManager(self.env) self.schema = [ Table('table1', key='col1')[ Column('col1', auto_increment=True), Column('col2'), Column('col3'), ], Table('table2', key='col1')[ Column('col1'), Column('col2'), ], Table('table3', key='col2')[ Column('col1'), Column('col2', type='int'), Column('col3') ] ] self.dbm.create_tables(self.schema) self.new_schema = copy.deepcopy([self.schema[0], self.schema[2]]) self.new_schema[0].remove_columns(('col2',)) self.new_schema[1].columns.append(Column('col4')) self.new_schema.append( Table('table4')[ Column('col1'), ] )
def do_upgrade(env, ver, cursor): """Change `subscription_attribute` db table: + 'subscription_attribute.authenticated' """ cursor.execute(""" CREATE TEMPORARY TABLE subscription_attribute_old AS SELECT * FROM subscription_attribute """) cursor.execute("DROP TABLE subscription_attribute") connector = DatabaseManager(env).get_connector()[0] for table in schema: for stmt in connector.to_sql(table): cursor.execute(stmt) cursor.execute(""" INSERT INTO subscription_attribute (sid,authenticated,class,realm,target) SELECT o.sid,s.authenticated,o.class,o.realm,o.target FROM subscription_attribute_old AS o LEFT JOIN session AS s ON o.sid=s.sid """) cursor.execute("DROP TABLE subscription_attribute_old") # DEVEL: Think that an old 'subscriptions' db table may still exist here. cursor.execute("DROP TABLE IF EXISTS subscriptions")
def do_upgrade(env, ver, cursor): """Add two more subscription db tables for a better normalized schema.""" connector = DatabaseManager(env).get_connector()[0] for table in schema: for stmt in connector.to_sql(table): cursor.execute(stmt)
def setUp(self): self.env = EnvironmentStub(enable=['trac.*', 'crashdump.*']) self.env.path = tempfile.mkdtemp() self.db_mgr = DatabaseManager(self.env) self.env.upgrade() #self.db = self.env.get_db_cnx() self.crashdump_module = CrashDumpModule(self.env)
def environment_needs_upgrade(self): """ Called when Trac checks whether the environment needs to be upgraded. Returns `True` if upgrade is needed, `False` otherwise. """ dbm = DatabaseManager(self.env) return dbm.get_database_version(db_version_key) != db_version
def __init__(self, path): self.env = Environment(path) self.loginNameCache = {} self.fieldNameCache = {} from trac.db.api import DatabaseManager self.using_postgres = \ DatabaseManager(self.env).connection_uri.startswith("postgres:")
def get_db_cnx(self): """Return a database connection from the connection pool :deprecated: Use :meth:`db_transaction` or :meth:`db_query` instead `db_transaction` for obtaining the `db` database connection which can be used for performing any query (SELECT/INSERT/UPDATE/DELETE):: with env.db_transaction as db: ... Note that within the block, you don't need to (and shouldn't) call ``commit()`` yourself, the context manager will take care of it (if it's the outermost such context manager on the stack). `db_query` for obtaining a `db` database connection which can be used for performing SELECT queries only:: with env.db_query as db: ... """ return DatabaseManager(self).get_connection()
def setUp(self): self.env = EnvironmentStub(path=mkdtemp()) self.dbm = DatabaseManager(self.env) with self.env.db_transaction: self.dbm.drop_tables(new_schema) self.dbm.create_tables(old_schema) self.dbm.set_database_version(VERSION - 1)
def upgrade(self, backup=False, backup_dest=None): """Upgrade database. :param backup: whether or not to backup before upgrading :param backup_dest: name of the backup file :return: whether the upgrade was performed """ upgraders = [] with self.db_query as db: for participant in self.setup_participants: if participant.environment_needs_upgrade(db): upgraders.append(participant) if not upgraders: return if backup: self.backup(backup_dest) for participant in upgraders: self.log.info("%s.%s upgrading...", participant.__module__, participant.__class__.__name__) with self.db_transaction as db: participant.upgrade_environment(db) # Database schema may have changed, so close all connections DatabaseManager(self).shutdown() return True
def setUp(self): self.env = EnvironmentStub(enable=['trac.*']) self.env.path = tempfile.mkdtemp() self.db_mgr = DatabaseManager(self.env) # Setup current announcer db schema tables. self.an_sys = AnnouncementSystem(self.env) self.an_sys.upgrade_environment()
def setUp(self): self.env = EnvironmentStub() self.schema = [ Table('test_simple', key='id')[Column('id', auto_increment=True), Column('username'), Column('email'), Column('enabled', type='int'), Column('extra'), Index(['username'], unique=True), Index(['email'], unique=False), ], Table('test_composite', key=['id', 'name'])[Column('id', type='int'), Column('name'), Column('value'), Column('enabled', type='int'), Index(['name', 'value'], unique=False), Index(['enabled', 'name'], unique=True), ], ] self.dbm = DatabaseManager(self.env) self.dbm.drop_tables(self.schema) self.dbm.create_tables(self.schema) self.dbm.insert_into_tables([ ('test_simple', ('username', 'email', 'enabled'), [('joe', '*****@*****.**', 1), (u'joé', '*****@*****.**', 0)]), ('test_composite', ('id', 'name', 'value', 'enabled'), [(1, 'foo', '42', 1), (1, 'bar', '42', 1), (2, 'foo', '43', 0), (2, 'bar', '43', 0)]), ])
def backup(self, dest=None): """Create a backup of the database. :param dest: Destination file; if not specified, the backup is stored in a file called db_name.trac_version.bak """ return DatabaseManager(self).backup(dest)
def database_version(self): """Returns the current version of the database. :since 1.0.2: """ return DatabaseManager(self) \ .get_database_version('database_version')
def upgrade(self, backup=False, backup_dest=None): """Upgrade database. :param backup: whether or not to backup before upgrading :param backup_dest: name of the backup file :return: whether the upgrade was performed """ upgraders = [] for participant in self.setup_participants: with self.component_guard(participant, reraise=True): if participant.environment_needs_upgrade(): upgraders.append(participant) if not upgraders: return if backup: try: self.backup(backup_dest) except Exception as e: raise BackupError(e) for participant in upgraders: self.log.info("upgrading %s...", participant) with self.component_guard(participant, reraise=True): participant.upgrade_environment() # Database schema may have changed, so close all connections dbm = DatabaseManager(self) if dbm.connection_uri != 'sqlite::memory:': dbm.shutdown() self._update_sample_config() del self.database_version return True
def shutdown(self, tid=None): """Close the environment.""" from trac.versioncontrol.api import RepositoryManager RepositoryManager(self).shutdown(tid) DatabaseManager(self).shutdown(tid) if tid is None: log.shutdown(self.log)
def reset_db(self, default_data=None): """Remove all data from Trac tables, keeping the tables themselves. :param default_data: after clean-up, initialize with default data :return: True upon success """ from trac import db_default tables = [] dbm = DatabaseManager(self) try: db_version = dbm.get_database_version() except (TracError, self.db_exc.DatabaseError): pass else: if db_version == db_default.db_version: # same version, simply clear the tables (faster) tables = dbm.reset_tables() else: # different version or version unknown, drop the tables self.destroy_db() if not tables: dbm.init_db() # Make sure the next db_query()/db_transaction() will create # a new connection aware of the new data model - see #8518. if self.dburi != 'sqlite::memory:': dbm.shutdown() if default_data: dbm.insert_into_tables(db_default.get_data) else: dbm.set_database_version(db_default.db_version)
def create(self, options=[], default_data=True): """Create the basic directory structure of the environment, initialize the database and populate the configuration file with default values. If options contains ('inherit', 'file'), default values will not be loaded; they are expected to be provided by that file or other options. :raises TracError: if the base directory of `path` does not exist. :raises TracError: if `path` exists and is not empty. """ base_dir = os.path.dirname(self.path) if not os.path.exists(base_dir): raise TracError( _( "Base directory '%(env)s' does not exist. Please create it " "and retry.", env=base_dir)) if os.path.exists(self.path) and os.listdir(self.path): raise TracError(_("Directory exists and is not empty.")) # Create the directory structure if not os.path.exists(self.path): os.mkdir(self.path) os.mkdir(self.htdocs_dir) os.mkdir(self.log_dir) os.mkdir(self.plugins_dir) os.mkdir(self.templates_dir) # Create a few files create_file(os.path.join(self.path, 'VERSION'), _VERSION + '\n') create_file( os.path.join(self.path, 'README'), 'This directory contains a Trac environment.\n' 'Visit https://trac.edgewall.org/ for more information.\n') # Setup the default configuration os.mkdir(self.conf_dir) config = Configuration(self.config_file_path) for section, name, value in options: config.set(section, name, value) config.save() self.setup_config() if not any((section, option) == ('inherit', 'file') for section, option, value in options): self.config.set_defaults(self) self.config.save() # Create the sample configuration create_file(self.config_file_path + '.sample') self._update_sample_config() # Create the database dbm = DatabaseManager(self) dbm.init_db() if default_data: dbm.insert_default_data()
def _do_convert_db_in_new_env(self, dst_dburi, env_path): try: os.rmdir(env_path) # remove directory if it's empty except OSError: pass if os.path.exists(env_path) or os.path.lexists(env_path): printferr("Cannot create Trac environment: %s: File exists", env_path) return 1 dst_env = self._create_env(env_path, dst_dburi) dbm = DatabaseManager(self.env) src_dburi = dbm.connection_uri src_db = dbm.get_connection() dst_db = DatabaseManager(dst_env).get_connection() self._copy_tables(dst_env, src_db, dst_db, src_dburi, dst_dburi) self._copy_directories(dst_env)
def _max_bytes(self, cnx): if cnx is None: connector, args = DatabaseManager(self.env).get_connector() with closing(connector.get_connection(**args)) as cnx: charset = cnx.charset else: charset = cnx.charset return 4 if charset == 'utf8mb4' else 3