def _do_convert_db_in_place(self, dst_dburi):
dbm = DatabaseManager(self.env)
src_dburi = dbm.connection_uri
if src_dburi == dst_dburi:
printferr("Source database and destination database are same: %s",
dst_dburi)
return 1
env_path = mkdtemp(prefix='convert_db-',
dir=os.path.dirname(self.env.path))
try:
dst_env = self._create_env(env_path, dst_dburi)
src_db = dbm.get_connection()
dst_db = DatabaseManager(dst_env).get_connection()
self._copy_tables(dst_env, src_db, dst_db, src_dburi, dst_dburi)
del src_db
del dst_db
dst_env.shutdown()
dst_env = None
schema, params = parse_connection_uri(dst_dburi)
if schema == 'sqlite':
dbpath = os.path.join(self.env.path, params['path'])
dbdir = os.path.dirname(dbpath)
if not os.path.isdir(dbdir):
os.makedirs(dbdir)
shutil.copy(os.path.join(env_path, params['path']), dbpath)
finally:
shutil.rmtree(env_path)
backup_config_file(self.env, '.convert_db-%d' % int(time.time()))
self.config.set('trac', 'database', dst_dburi)
self.config.save()
def engine(env):
engine = _engines.get(env)
if engine is None:
schema = DatabaseManager(env).connection_uri.split(':')[0]
echo = env.config.get('logging', 'log_level').lower() == 'debug'
def connect():
cnx = env.get_db_cnx().cnx
while isinstance(cnx, ConnectionWrapper):
cnx = cnx.cnx
return cnx
engine = create_engine(URL(schema), poolclass=DummyPool,
creator=connect, echo=echo)
if echo:
# make sqlalchemy log to trac's logger
if hasattr(env, 'get_logger'):
engine.logger = env.get_logger(__name__)
else:
engine.logger = env.log
_engines[env] = engine
try:
return engine.begin()
except AttributeError:
# Older SQLAlchemy
return engine
def do_upgrade(env, ver, cursor):
"""Add an auto-increment primary key to `node_change` table and indices
(repos, rev, path) and (repos, path, rev) (#3676).
"""
db_connector, _ = DatabaseManager(env)._get_connector()
table = Table('node_change', key='id')[Column('id', auto_increment=True),
Column('repos', type='int'),
Column('rev', key_size=40),
Column('path', key_size=255),
Column('node_type', size=1),
Column('change_type', size=1),
Column('base_path'),
Column('base_rev'),
Index(['repos', 'rev', 'path']),
Index(['repos', 'path', 'rev'])]
with env.db_transaction:
cursor.execute("""CREATE TEMPORARY TABLE node_change_old AS
SELECT * FROM node_change""")
cursor.execute("DROP TABLE node_change")
for stmt in db_connector.to_sql(table):
cursor.execute(stmt)
cursor.execute("""\
INSERT INTO node_change
(repos,rev,path,node_type,change_type,base_path,base_rev)
SELECT repos,rev,path,node_type,change_type,base_path,base_rev
FROM node_change_old""")
cursor.execute("DROP TABLE node_change_old")
def upgrade(self, backup=False, backup_dest=None):
"""Upgrade database.
:param backup: whether or not to backup before upgrading
:param backup_dest: name of the backup file
:return: whether the upgrade was performed
"""
upgraders = []
for participant in self.setup_participants:
if participant.environment_needs_upgrade():
upgraders.append(participant)
if not upgraders:
return
if backup:
try:
self.backup(backup_dest)
except Exception as e:
raise BackupError(e)
for participant in upgraders:
self.log.info("%s.%s upgrading...", participant.__module__,
participant.__class__.__name__)
participant.upgrade_environment()
# Database schema may have changed, so close all connections
DatabaseManager(self).shutdown()
del self.database_version
return True
def do_upgrade(env, ver, cursor):
"""Changes to subscription db table:
- 'subscriptions.destination', 'subscriptions.format'
+ 'subscriptions.authenticated', 'subscriptions.transport'
'subscriptions.managed' type='int' --> (default == char)
"""
cursor.execute("""
CREATE TEMPORARY TABLE subscriptions_old
AS SELECT * FROM subscriptions
""")
cursor.execute("DROP TABLE subscriptions")
connector = DatabaseManager(env).get_connector()[0]
for table in schema:
for stmt in connector.to_sql(table):
cursor.execute(stmt)
cursor.execute("""
INSERT INTO subscriptions
(sid,authenticated,enabled,managed,
realm,category,rule,transport)
SELECT o.sid,s.authenticated,o.enabled,'watcher',
o.realm,o.category,rule,'email'
FROM subscriptions_old AS o
LEFT JOIN session AS s
ON o.sid=s.sid
""")
cursor.execute("DROP TABLE subscriptions_old")
def wiki_setup(tc):
tc.env = EnvironmentStub(default_data=True,
enable=['trac.*', 'tractags.*'])
tc.env.path = tempfile.mkdtemp()
tc.db_mgr = DatabaseManager(tc.env)
tc.db = tc.env.get_db_cnx()
TagSetup(tc.env).upgrade_environment(tc.db)
now = datetime.now(utc)
wiki = WikiPage(tc.env)
wiki.name = 'TestPage'
wiki.text = '--'
wiki.save('joe', 'TagsPluginTestPage', '::1', now)
req = Mock(href=Href('/'),
abs_href=Href('http://www.example.com/'),
authname='anonymous',
perm=MockPerm(),
tz=utc,
args={},
locale=Locale.parse('en_US') if Locale else None)
tc.env.href = req.href
tc.env.abs_href = req.abs_href
tc.context = Context.from_request(req)
# Enable big diff output.
tc.maxDiff = None
def reset_mysql_db(env, db_prop):
"""Deletes all data from the tables and resets autoincrement indexes.
:since 1.1.3: deprecated and will be removed in 1.3.1. Use `reset_tables`
from the database connection class instead.
"""
return DatabaseManager(env).reset_tables()
def do_upgrade(env, version, cursor):
"""Add `description` column to `enum` table."""
new_schema = [
Table('enum', key=('type', 'name'))[Column('type'),
Column('name'),
Column('value'),
Column('description'), ]
]
with env.db_transaction:
DatabaseManager(env).upgrade_tables(new_schema)
failures = []
for id_ in [1, 2, 3, 4, 5, 7, 8]:
try:
r = Report(env, id_)
except ResourceNotFound:
pass
else:
query = replace_sql_fragment(r.query)
if query:
r.query = query
r.update()
else:
failures.append(unicode(id_))
if failures:
printout(
_("""\
Report(s) %(ids)s could not be upgraded and may need to be manually
edited to avoid an "ambiguous column name" error. See %(url)s for more
information.
""",
ids=', '.join(failures),
url=url))
def upgrade_environment(self, db):
db_connector, _ = DatabaseManager(self.env).get_connector()
cursor = db.cursor()
dbver = self.get_db_version()
if dbver == 0:
self.env.log.info("Initialize %s database schema to version %s",
PLUGIN_NAME, PLUGIN_VERSION)
for table in SCHEMA:
for stmt in db_connector.to_sql(table):
cursor.execute(stmt)
cursor.execute(
"""
INSERT INTO system (name, value)
VALUES (%s, %s)
""", (PLUGIN_NAME, PLUGIN_VERSION))
else:
while dbver != PLUGIN_VERSION:
dbver = dbver + 1
self.env.log.info("Upgrade %s database schema to version %s",
PLUGIN_NAME, dbver)
modulename = 'db%i' % dbver
upgrades = __import__('cards.upgrades', globals(), locals(),
[modulename])
script = getattr(upgrades, modulename)
script.do_upgrade(self.env, dbver, cursor)
cursor.execute(
"""
UPDATE system
SET value=%s
WHERE name=%s
""", (PLUGIN_VERSION, PLUGIN_NAME))
def upgrade_environment(self, db=None):
dbm = DatabaseManager(self.env)
if dbm.get_database_version(DB_NAME) is False:
dbm.create_tables(schema)
dbm.set_database_version(DB_VERSION, DB_NAME)
else:
dbm.upgrade(DB_VERSION, DB_NAME, 'coderev.upgrades')
def trac_db_query(env):
"""
Context manager to handle database connection and cursor from trac environment's
read only connection. This does not attempt to roll back or commit, the connection
is meant only for accessing the data. For examples on use, see db_query().
Internally, this uses Trac's connection pool via the Trac DatabaseManager class
:param Environment env: The Trac environment
"""
conn = None
cursor = None
try:
dm = DatabaseManager(env)
conn = dm.get_connection()
except Exception:
env.log.exception(
"Failed to get database connection from trac database manager")
raise
try:
# NOTE: Trac's connection does not support alternative cursor types!
cursor = conn.cursor()
yield cursor
except Exception:
env.log.error("Failed to query database")
raise
finally:
cursor.close()
conn.close()
def setUp(self):
self.env = EnvironmentStub()
self.dbm = DatabaseManager(self.env)
self.schema = [
Table('table1', key='col1')[
Column('col1', auto_increment=True),
Column('col2'),
Column('col3'),
],
Table('table2', key='col1')[
Column('col1'),
Column('col2'),
],
Table('table3', key='col2')[
Column('col1'),
Column('col2', type='int'),
Column('col3')
]
]
self.dbm.create_tables(self.schema)
self.new_schema = copy.deepcopy([self.schema[0], self.schema[2]])
self.new_schema[0].remove_columns(('col2',))
self.new_schema[1].columns.append(Column('col4'))
self.new_schema.append(
Table('table4')[
Column('col1'),
]
)
def do_upgrade(env, ver, cursor):
"""Change `subscription_attribute` db table:
+ 'subscription_attribute.authenticated'
"""
cursor.execute("""
CREATE TEMPORARY TABLE subscription_attribute_old
AS SELECT * FROM subscription_attribute
""")
cursor.execute("DROP TABLE subscription_attribute")
connector = DatabaseManager(env).get_connector()[0]
for table in schema:
for stmt in connector.to_sql(table):
cursor.execute(stmt)
cursor.execute("""
INSERT INTO subscription_attribute
(sid,authenticated,class,realm,target)
SELECT o.sid,s.authenticated,o.class,o.realm,o.target
FROM subscription_attribute_old AS o
LEFT JOIN session AS s
ON o.sid=s.sid
""")
cursor.execute("DROP TABLE subscription_attribute_old")
# DEVEL: Think that an old 'subscriptions' db table may still exist here.
cursor.execute("DROP TABLE IF EXISTS subscriptions")
def do_upgrade(env, ver, cursor):
"""Add two more subscription db tables for a better normalized schema."""
connector = DatabaseManager(env).get_connector()[0]
for table in schema:
for stmt in connector.to_sql(table):
cursor.execute(stmt)
def setUp(self):
self.env = EnvironmentStub(enable=['trac.*', 'crashdump.*'])
self.env.path = tempfile.mkdtemp()
self.db_mgr = DatabaseManager(self.env)
self.env.upgrade()
#self.db = self.env.get_db_cnx()
self.crashdump_module = CrashDumpModule(self.env)
def environment_needs_upgrade(self):
"""
Called when Trac checks whether the environment needs to be upgraded.
Returns `True` if upgrade is needed, `False` otherwise.
"""
dbm = DatabaseManager(self.env)
return dbm.get_database_version(db_version_key) != db_version
def __init__(self, path):
self.env = Environment(path)
self.loginNameCache = {}
self.fieldNameCache = {}
from trac.db.api import DatabaseManager
self.using_postgres = \
DatabaseManager(self.env).connection_uri.startswith("postgres:")
def get_db_cnx(self):
"""Return a database connection from the connection pool
:deprecated: Use :meth:`db_transaction` or :meth:`db_query` instead
`db_transaction` for obtaining the `db` database connection
which can be used for performing any query
(SELECT/INSERT/UPDATE/DELETE)::
with env.db_transaction as db:
...
Note that within the block, you don't need to (and shouldn't)
call ``commit()`` yourself, the context manager will take care
of it (if it's the outermost such context manager on the
stack).
`db_query` for obtaining a `db` database connection which can
be used for performing SELECT queries only::
with env.db_query as db:
...
"""
return DatabaseManager(self).get_connection()
def setUp(self):
self.env = EnvironmentStub(path=mkdtemp())
self.dbm = DatabaseManager(self.env)
with self.env.db_transaction:
self.dbm.drop_tables(new_schema)
self.dbm.create_tables(old_schema)
self.dbm.set_database_version(VERSION - 1)
def upgrade(self, backup=False, backup_dest=None):
"""Upgrade database.
:param backup: whether or not to backup before upgrading
:param backup_dest: name of the backup file
:return: whether the upgrade was performed
"""
upgraders = []
with self.db_query as db:
for participant in self.setup_participants:
if participant.environment_needs_upgrade(db):
upgraders.append(participant)
if not upgraders:
return
if backup:
self.backup(backup_dest)
for participant in upgraders:
self.log.info("%s.%s upgrading...", participant.__module__,
participant.__class__.__name__)
with self.db_transaction as db:
participant.upgrade_environment(db)
# Database schema may have changed, so close all connections
DatabaseManager(self).shutdown()
return True
def setUp(self):
self.env = EnvironmentStub(enable=['trac.*'])
self.env.path = tempfile.mkdtemp()
self.db_mgr = DatabaseManager(self.env)
# Setup current announcer db schema tables.
self.an_sys = AnnouncementSystem(self.env)
self.an_sys.upgrade_environment()
def setUp(self):
self.env = EnvironmentStub()
self.schema = [
Table('test_simple', key='id')[Column('id', auto_increment=True),
Column('username'),
Column('email'),
Column('enabled', type='int'),
Column('extra'),
Index(['username'], unique=True),
Index(['email'], unique=False), ],
Table('test_composite',
key=['id',
'name'])[Column('id', type='int'),
Column('name'),
Column('value'),
Column('enabled', type='int'),
Index(['name', 'value'], unique=False),
Index(['enabled', 'name'], unique=True), ],
]
self.dbm = DatabaseManager(self.env)
self.dbm.drop_tables(self.schema)
self.dbm.create_tables(self.schema)
self.dbm.insert_into_tables([
('test_simple', ('username', 'email', 'enabled'),
[('joe', '*****@*****.**', 1), (u'joé', '*****@*****.**', 0)]),
('test_composite', ('id', 'name', 'value', 'enabled'),
[(1, 'foo', '42', 1), (1, 'bar', '42', 1), (2, 'foo', '43', 0),
(2, 'bar', '43', 0)]),
])
def backup(self, dest=None):
"""Create a backup of the database.
:param dest: Destination file; if not specified, the backup is
stored in a file called db_name.trac_version.bak
"""
return DatabaseManager(self).backup(dest)
def database_version(self):
"""Returns the current version of the database.
:since 1.0.2:
"""
return DatabaseManager(self) \
.get_database_version('database_version')
def upgrade(self, backup=False, backup_dest=None):
"""Upgrade database.
:param backup: whether or not to backup before upgrading
:param backup_dest: name of the backup file
:return: whether the upgrade was performed
"""
upgraders = []
for participant in self.setup_participants:
with self.component_guard(participant, reraise=True):
if participant.environment_needs_upgrade():
upgraders.append(participant)
if not upgraders:
return
if backup:
try:
self.backup(backup_dest)
except Exception as e:
raise BackupError(e)
for participant in upgraders:
self.log.info("upgrading %s...", participant)
with self.component_guard(participant, reraise=True):
participant.upgrade_environment()
# Database schema may have changed, so close all connections
dbm = DatabaseManager(self)
if dbm.connection_uri != 'sqlite::memory:':
dbm.shutdown()
self._update_sample_config()
del self.database_version
return True
def shutdown(self, tid=None):
"""Close the environment."""
from trac.versioncontrol.api import RepositoryManager
RepositoryManager(self).shutdown(tid)
DatabaseManager(self).shutdown(tid)
if tid is None:
log.shutdown(self.log)
def reset_db(self, default_data=None):
"""Remove all data from Trac tables, keeping the tables themselves.
:param default_data: after clean-up, initialize with default data
:return: True upon success
"""
from trac import db_default
tables = []
dbm = DatabaseManager(self)
try:
db_version = dbm.get_database_version()
except (TracError, self.db_exc.DatabaseError):
pass
else:
if db_version == db_default.db_version:
# same version, simply clear the tables (faster)
tables = dbm.reset_tables()
else:
# different version or version unknown, drop the tables
self.destroy_db()
if not tables:
dbm.init_db()
# Make sure the next db_query()/db_transaction() will create
# a new connection aware of the new data model - see #8518.
if self.dburi != 'sqlite::memory:':
dbm.shutdown()
if default_data:
dbm.insert_into_tables(db_default.get_data)
else:
dbm.set_database_version(db_default.db_version)
def create(self, options=[], default_data=True):
"""Create the basic directory structure of the environment,
initialize the database and populate the configuration file
with default values.
If options contains ('inherit', 'file'), default values will
not be loaded; they are expected to be provided by that file
or other options.
:raises TracError: if the base directory of `path` does not exist.
:raises TracError: if `path` exists and is not empty.
"""
base_dir = os.path.dirname(self.path)
if not os.path.exists(base_dir):
raise TracError(
_(
"Base directory '%(env)s' does not exist. Please create it "
"and retry.",
env=base_dir))
if os.path.exists(self.path) and os.listdir(self.path):
raise TracError(_("Directory exists and is not empty."))
# Create the directory structure
if not os.path.exists(self.path):
os.mkdir(self.path)
os.mkdir(self.htdocs_dir)
os.mkdir(self.log_dir)
os.mkdir(self.plugins_dir)
os.mkdir(self.templates_dir)
# Create a few files
create_file(os.path.join(self.path, 'VERSION'), _VERSION + '\n')
create_file(
os.path.join(self.path, 'README'),
'This directory contains a Trac environment.\n'
'Visit https://trac.edgewall.org/ for more information.\n')
# Setup the default configuration
os.mkdir(self.conf_dir)
config = Configuration(self.config_file_path)
for section, name, value in options:
config.set(section, name, value)
config.save()
self.setup_config()
if not any((section, option) == ('inherit', 'file')
for section, option, value in options):
self.config.set_defaults(self)
self.config.save()
# Create the sample configuration
create_file(self.config_file_path + '.sample')
self._update_sample_config()
# Create the database
dbm = DatabaseManager(self)
dbm.init_db()
if default_data:
dbm.insert_default_data()
def _do_convert_db_in_new_env(self, dst_dburi, env_path):
try:
os.rmdir(env_path) # remove directory if it's empty
except OSError:
pass
if os.path.exists(env_path) or os.path.lexists(env_path):
printferr("Cannot create Trac environment: %s: File exists",
env_path)
return 1
dst_env = self._create_env(env_path, dst_dburi)
dbm = DatabaseManager(self.env)
src_dburi = dbm.connection_uri
src_db = dbm.get_connection()
dst_db = DatabaseManager(dst_env).get_connection()
self._copy_tables(dst_env, src_db, dst_db, src_dburi, dst_dburi)
self._copy_directories(dst_env)
def _max_bytes(self, cnx):
if cnx is None:
connector, args = DatabaseManager(self.env).get_connector()
with closing(connector.get_connection(**args)) as cnx:
charset = cnx.charset
else:
charset = cnx.charset
return 4 if charset == 'utf8mb4' else 3
