def setUp(self):
self.env = EnvironmentStub(enable=['trac.*', 'bitten.*'])
self.env.path = tempfile.mkdtemp()
# Create tables
db = self.env.get_db_cnx()
cursor = db.cursor()
connector, _ = DatabaseManager(self.env)._get_connector()
for table in schema:
for stmt in connector.to_sql(table):
cursor.execute(stmt)
# Set up permissions
self.env.config.set('trac', 'permission_store',
'DefaultPermissionStore')
PermissionSystem(self.env).grant_permission('joe', 'BUILD_ADMIN')
if DefaultPermissionPolicy is not None and hasattr(
DefaultPermissionPolicy, "CACHE_EXPIRY"):
self.old_perm_cache_expiry = DefaultPermissionPolicy.CACHE_EXPIRY
DefaultPermissionPolicy.CACHE_EXPIRY = -1
# Hook up a dummy repository
self.repos = Mock(get_node=lambda path, rev=None: Mock(
get_history=lambda: [], isdir=True),
normalize_path=lambda path: path,
sync=lambda: None)
self.env.get_repository = lambda authname=None: self.repos
def do_upgrade(env, ver, cursor):
"""Add new table for tag change records."""
connector = DatabaseManager(env)._get_connector()[0]
for table in schema:
for stmt in connector.to_sql(table):
cursor.execute(stmt)
def upgrade_to_0_1_9(env, db, installed_version):
if installed_version >= [0, 1, 9]:
return
cursor = db.cursor()
mil_table = None
for table in schema:
if table.name == 'milestone':
mil_table = table
break
if mil_table:
cols = ",".join([c.name for c in mil_table.columns])
cursor.execute(
"CREATE TEMPORARY TABLE milestone_old as SELECT %s FROM milestone;"
% cols)
cursor.execute("DROP TABLE milestone;")
new_mil_table = Table('milestone',
key='id')[mil_table.columns + mil_table.indices +
[Column('started', type='int')]]
db_backend, _ = DatabaseManager(env)._get_connector()
for stmt in db_backend.to_sql(new_mil_table):
cursor.execute(stmt)
cursor.execute(
"INSERT INTO milestone (%s) SELECT %s FROM milestone_old;" %
(cols, cols))
env.log.debug("Upgrading: upgrade_to_0_1_9")
def add_log_table(env, db):
"""Add a table for storing the builds logs."""
from bitten.model import BuildLog, BuildStep
cursor = db.cursor()
connector, _ = DatabaseManager(env)._get_connector()
for table in BuildLog._schema:
for stmt in connector.to_sql(table):
cursor.execute(stmt)
cursor.execute("SELECT build,name,log FROM bitten_step "
"WHERE log IS NOT NULL")
for build, step, log in cursor:
build_log = BuildLog(env, build, step)
build_log.messages = [(BuildLog.INFO, msg) for msg in log.splitlines()]
build_log.insert(db)
cursor.execute(
"CREATE TEMPORARY TABLE old_step AS SELECT * FROM bitten_step")
cursor.execute("DROP TABLE bitten_step")
for table in BuildStep._schema:
for stmt in connector.to_sql(table):
cursor.execute(stmt)
cursor.execute("INSERT INTO bitten_step (build,name,description,status,"
"started,stopped) SELECT build,name,description,status,"
"started,stopped FROM old_step")
def create_table(env, table, conn=None):
"""
Creates a the given table in the given environment. The Table
has to be of type trac.db.Table, and the Environment a
trac.env.Environment.
"""
assert isinstance(env, Environment), \
"[DB]: env should be an instance of trac.env.Environment, got %s" % type(env)
assert isinstance(table, Table), \
"[DB]: table should be an instance of trac.sb.Table, got %s" % type(table)
# Get The Databse Manager
dbm = DatabaseManager(env)
# Get the Connector Object for the current DB schema
connector, args = dbm._get_connector()
# Ask the connector to generate the proper DDL for the table
ddl_gen = connector.to_sql(table)
# Get a DB Connection from the pool, create a cursor and the table
conn, handle_ta = get_db_for_write(env, conn)
try:
cursor = conn.cursor()
for statement in ddl_gen:
debug(env, "[DB]: Table: %s\n%s" % (table.name, statement))
cursor.execute(statement)
if handle_ta:
conn.commit()
debug(env, "[DB]: Successfully Created Table %s" % table.name)
except Exception, e:
if handle_ta:
conn.rollback()
error(env, "[DB]: Unable to Create Table %s, an error occurred: %s" % \
(table.name, exception_to_unicode(e)))
raise
def do_upgrade(env, cursor):
db_connector, _ = DatabaseManager(env)._get_connector()
print "Upgrading forum database tables to version 2\n"
# Backup old screenshot table.
cursor.execute("CREATE TEMPORARY TABLE forum_old AS SELECT * FROM forum")
cursor.execute("DROP TABLE forum")
# Create tables
for table in tables:
for statement in db_connector.to_sql(table):
cursor.execute(statement)
sql = "INSERT INTO forum (id, name, time, moderators, subject, description) " \
"SELECT id, name, time, moderators, subject, description " \
" FROM forum_old"
print sql + "\n"
# Copy old forums
cursor.execute(sql)
# Set database schema version.
cursor.execute("UPDATE system SET value = '2' WHERE" \
" name = 'discussion_version'")
print "done.\n"
def setUp(self):
self.env = EnvironmentStub(enable=['trac.*', 'bitten.notify.*',
'bitten.tests.notify.*'])
db = self.env.get_db_cnx()
cursor = db.cursor()
connector, _ = DatabaseManager(self.env)._get_connector()
for table in schema:
for stmt in connector.to_sql(table):
cursor.execute(stmt)
db.commit()
# Hook up a dummy repository
self.repos = Mock(
get_changeset=lambda rev: Mock(author='author', rev=rev),
normalize_rev=lambda rev: rev)
self.env.get_repository = lambda authname=None: self.repos # 0.11
try: # 0.12+
from trac.core import Component, implements
from trac.versioncontrol.api import IRepositoryConnector, \
IRepositoryProvider
class DummyRepos(Component):
implements(IRepositoryConnector, IRepositoryProvider)
def get_supported_types(self):
yield ('dummy', 9)
def get_repository(this, repos_type, repos_dir, params):
return self.repos # Note: 'this' vs 'self' usage
def get_repositories(self):
yield ('', {'dir': 'dummy_dir', 'type': 'dummy'})
self.dummy = DummyRepos
except ImportError:
self.dummy = None # not supported, will use get_repository()
def upgrade_environment(self, db):
# 0.10 compatibility hack (thanks Alec)
try:
from trac.db import DatabaseManager
db_manager, _ = DatabaseManager(self.env)._get_connector()
except ImportError:
db_manager = db
# Insert the default table
cursor = db.cursor()
if not self.found_db_version:
cursor.execute(
"INSERT INTO system (name, value) VALUES ('codereview_version', %s)",
(db_default.version, ))
cursor.execute(
"INSERT INTO system VALUES ('CodeReviewVoteThreshold', '0')")
else:
cursor.execute(
"UPDATE system SET value = %s WHERE name = 'codereview_version'",
(db_default.version, ))
for tbl in db_default.tables:
try:
cursor.execute('DROP TABLE %s' % tbl.name, )
except:
pass
for tbl in db_default.tables:
for sql in db_manager.to_sql(tbl):
cursor.execute(sql)
def do_upgrade(env, ver, cursor):
"""Change `subscription_attribute` db table:
+ 'subscription_attribute.authenticated'
"""
cursor.execute("""
CREATE TEMPORARY TABLE subscription_attribute_old
AS SELECT * FROM subscription_attribute
""")
cursor.execute("DROP TABLE subscription_attribute")
connector = DatabaseManager(env)._get_connector()[0]
for table in schema:
for stmt in connector.to_sql(table):
cursor.execute(stmt)
cursor.execute("""
INSERT INTO subscription_attribute
(sid,authenticated,class,realm,target)
SELECT o.sid,s.authenticated,o.class,o.realm,o.target
FROM subscription_attribute_old AS o
LEFT JOIN session AS s
ON o.sid=s.sid
""")
cursor.execute("DROP TABLE subscription_attribute_old")
# DEVEL: Think that an old 'subscriptions' db table may still exist here.
cursor.execute("DROP TABLE IF EXISTS subscriptions")
def setUp(self):
self.env = EnvironmentStub(enable=['trac.*', 'bitten.*'])
self.env.path = tempfile.mkdtemp()
db = self.env.get_db_cnx()
cursor = db.cursor()
connector, _ = DatabaseManager(self.env)._get_connector()
for table in schema:
for stmt in connector.to_sql(table):
cursor.execute(stmt)
self.config = BuildConfig(self.env, name='test', path='somepath')
self.config.insert(db=db)
self.platform = TargetPlatform(self.env, config='test', name='Foo')
self.platform.insert(db=db)
db.commit()
# Hook up a dummy repository
self.repos = Mock()
self.env.get_repository = lambda authname=None: self.repos # 0.11
try: # 0.12+
from trac.core import Component, implements
from trac.versioncontrol.api import IRepositoryConnector, \
IRepositoryProvider
class DummyRepos(Component):
implements(IRepositoryConnector, IRepositoryProvider)
def get_supported_types(self):
yield ('dummy', 9)
def get_repository(this, repos_type, repos_dir, params):
return self.repos # Note: 'this' vs 'self' usage
def get_repositories(self):
yield ('', {'dir': 'dummy_dir', 'type': 'dummy'})
self.dummy = DummyRepos
except ImportError:
self.dummy = None # not supported, will use get_repository()
def do_upgrade(env, cursor):
db_connector, _ = DatabaseManager(env)._get_connector()
# Backup old topic table
cursor.execute("CREATE TEMPORARY TABLE topic_old AS "
"SELECT * "
"FROM topic")
cursor.execute("DROP TABLE topic")
# Create tables.
for table in tables:
for statement in db_connector.to_sql(table):
cursor.execute(statement)
# Add two columns that uses constraints.
# TODO: No other way how to do it.
cursor.execute("ALTER TABLE topic ADD COLUMN status INT DEFAULT 0 NOT NULL")
cursor.execute("ALTER TABLE topic ADD COLUMN priority INT DEFAULT 0 NOT NULL")
# Copy old topics.
cursor.execute("INSERT INTO topic "
"(id, forum, time, author, subscribers, subject, body, "
"status, priority) "
"SELECT id, forum, time, author, subscribers, subject, "
"body, 0, 0 "
"FROM topic_old")
cursor.execute("DROP TABLE topic_old")
# Set database schema version.
cursor.execute("UPDATE system "
"SET value = '5' "
"WHERE name = 'discussion_version'")
def do_upgrade(env, cursor):
db_connector, _ = DatabaseManager(env)._get_connector()
# Backup old forum table.
cursor.execute("CREATE TEMPORARY TABLE forum_old AS "
"SELECT * "
"FROM forum")
cursor.execute("DROP TABLE forum")
# Create tables.
for table in tables:
for statement in db_connector.to_sql(table):
cursor.execute(statement)
# Copy old forums.
cursor.execute("INSERT INTO forum "
"(id, name, time, moderators, subject, description) "
"SELECT id, name, time, moderators, subject, description "
"FROM forum_old")
cursor.execute("DROP TABLE forum_old")
# Set database schema version.
cursor.execute("UPDATE system "
"SET value = '2' "
"WHERE name = 'discussion_version'")
def do_upgrade(env):
""" Upgrade the database schema so that it is compatible with version 1
of the plugin.
:param env: the current Trac environment
"""
db_connector, _ = DatabaseManager(env)._get_connector()
db = env.get_db_cnx()
cursor = db.cursor()
# add 'ticket_split' table to the schema
split_ticket_table = Table('ticket_split', key=('ticket', 'split_to'))[
Column('ticket', type='int'),
Column('split_to', type='int'),
Column('split_at', type='int')
]
for statement in db_connector.to_sql(split_ticket_table):
cursor.execute(statement)
# update stored schema version for the plugin
sql = ("INSERT INTO system(name, value) "
"VALUES ('splitticket_plugin_version', '1')")
cursor.execute(sql)
def upgrade_environment(self, db):
self.log.debug("Upgrading schema for bi history plugin")
cursor = db.cursor()
db_connector, _ = DatabaseManager(self.env).get_connector()
found_version = self._check_schema_version(db)
if not found_version:
# Create tables
self.environment_created()
elif found_version == 2:
# We've not released anywhere yet, so this seems more practical
# than writing a database-agnostic way to convert the isclosed column
cursor.execute("DROP table ticket_bi_historical")
for table in self.schema:
for statement in db_connector.to_sql(table):
cursor.execute(statement)
cursor.execute("UPDATE system SET value = %s WHERE name = 'bi_history_schema'",
(str(self._schema_version),))
elif found_version == 3:
cursor.execute("CREATE INDEX ticket_bi_historical_isclosed "
"ON ticket_bi_historical (isclosed)")
cursor.execute("CREATE INDEX ticket_bi_historical_id "
"ON ticket_bi_historical (id)")
cursor.execute("CREATE INDEX ticket_bi_historical_milestone "
"ON ticket_bi_historical (milestone)")
cursor.execute("UPDATE system SET value = %s "
"WHERE name = 'bi_history_schema'",
(str(self._schema_version),))
elif found_version == 4:
if self.env.config.get('trac', 'database').startswith('postgres'):
cursor.execute("ALTER TABLE ticket_bi_historical ALTER COLUMN _snapshottime SET NOT NULL")
cursor.execute("UPDATE system SET value = %s "
"WHERE name = 'bi_history_schema'",
(str(self._schema_version),))
def _initialize_db(self, db):
# pylint: disable=protected-access
self.log.debug("creating initial db schema for %s.", PLUGIN_NAME)
db_connector, dummy = DatabaseManager(self.env)._get_connector()
for table in db_default.SCHEMA:
for statement in db_connector.to_sql(table):
db(statement)
def add_ticket_template_store(env, db):
"""Add table ticket_template_store."""
from tickettemplate.model import schema, schema_version, TT_Template
from trac.db import DatabaseManager
connector, _ = DatabaseManager(env)._get_connector()
cursor = db.cursor()
table = schema[0]
for stmt in connector.to_sql(table):
try:
cursor.execute(stmt)
except:
pass
from default_templates import DEFAULT_TEMPLATES
from ttadmin import SYSTEM_USER
now = int(time.time())
for tt_name, tt_value in DEFAULT_TEMPLATES:
record = (now, SYSTEM_USER, tt_name, "description", tt_value,)
TT_Template.insert(env, record)
for id, modi_time, tt_name, tt_text in cursor.fetchall():
record = (modi_time, SYSTEM_USER, tt_name, "description", tt_text,)
TT_Template.insert(env, record)
def test_db_table():
# Create a EnvironmentStub
env = EnvironmentStub()
# Create a test table
table = Table('test', key=['id'])[
Column('id', type='integer'),
Column(Key.NAME, type='text')
]
# Get The Databse Manager
dbm = DatabaseManager(env)
# Get the Connector Object for the current DB schema
connector, args = dbm._get_connector()
# Ask the connector to generate the proper DDL for the table
ddl_gen = connector.to_sql(table)
# Get a DB Connection from the pool, create a cursor and the table
conn = dbm.get_connection()
try:
cursor = conn.cursor()
for statement in ddl_gen:
print "Table: %s\n%s" % (table.name, statement)
cursor.execute(statement)
conn.commit()
print "Successfully Created Table %s" % table.name
except Exception, e:
conn.rollback()
print "[ERROR]: Unable to Create Table %s, an error occurred: %s" % \
(table.name, str(e))
def environment_created(self):
# Create the required tables
db = self.env.get_db_cnx()
connector, _ = DatabaseManager(self.env)._get_connector()
cursor = db.cursor()
for table in schema:
for stmt in connector.to_sql(table):
cursor.execute(stmt)
# Insert a global version flag
cursor.execute(
"INSERT INTO system (name,value) "
"VALUES ('tt_version',%s)", (schema_version, ))
# Create some default templates
now = int(time.time())
from default_templates import DEFAULT_TEMPLATES
for tt_name, tt_value in DEFAULT_TEMPLATES:
record = [
now,
SYSTEM_USER,
tt_name,
"description",
tt_value,
]
TT_Template.insert(self.env, record)
db.commit()
def upgrade_environment(self, db):
if not type(self).__dict__.get('applySchema', False):
self.log.debug("""Not updating schema for \"%s\",
since applySchema is not defined or is False.
""" % type(self).__name__)
return
installed = self.get_installed_version(db)
if installed is None:
self.log.info(
'Installing TracForm plugin schema %s' % db_version)
db_connector, _ = DatabaseManager(self.env)._get_connector()
db = self._get_db(db)
cursor = db.cursor()
for table in schema:
for stmt in db_connector.to_sql(table):
cursor.execute(stmt)
self.set_installed_version(db, db_version)
self.log.info('Installation of %s successful.' % db_version)
return
self.log.debug(
'Upgrading schema for "%s".' % type(self).__name__)
for version, fn in self.get_schema_functions():
if version > installed:
self.log.info(
'Upgrading TracForm plugin schema to %s' % version)
self.log.info('- %s: %s' % (fn.__name__, fn.__doc__))
db = self._get_db(db)
cursor = db.cursor()
fn(self.env, cursor)
self.set_installed_version(db, version)
installed = version
self.log.info('Upgrade to %s successful.' % version)
def _upgrade_db(self):
db_backend = DatabaseManager(self.env)._get_connector()[0]
with self.env.db_transaction as db:
cursor = db.cursor()
for table in self.SCHEMA:
for stmt in db_backend.to_sql(table):
cursor.execute(stmt)
def do_upgrade(env, ver, cursor):
"""Add new table for tag change records."""
connector = DatabaseManager(env).get_connector()[0]
for table in schema:
for stmt in connector.to_sql(table):
cursor.execute(stmt)
def do_upgrade(env, ver, cursor):
"""Changes to subscription db table:
- 'subscriptions.destination', 'subscriptions.format'
+ 'subscriptions.authenticated', 'subscriptions.transport'
'subscriptions.managed' type='int' --> (default == char)
"""
cursor.execute("""
CREATE TEMPORARY TABLE subscriptions_old
AS SELECT * FROM subscriptions
""")
cursor.execute("DROP TABLE subscriptions")
connector = DatabaseManager(env)._get_connector()[0]
for table in schema:
for stmt in connector.to_sql(table):
cursor.execute(stmt)
cursor.execute("""
INSERT INTO subscriptions
(sid,authenticated,enabled,managed,
realm,category,rule,transport)
SELECT o.sid,s.authenticated,o.enabled,'watcher',
o.realm,o.category,rule,'email'
FROM subscriptions_old AS o
LEFT JOIN session AS s
ON o.sid=s.sid
""")
cursor.execute("DROP TABLE subscriptions_old")
def setUp(self):
self.env = EnvironmentStub(enable=['trac.*', 'bitten.*'])
self.env.path = tempfile.mkdtemp()
# Create tables
db = self.env.get_db_cnx()
cursor = db.cursor()
connector, _ = DatabaseManager(self.env)._get_connector()
for table in schema:
for stmt in connector.to_sql(table):
cursor.execute(stmt)
# Set up permissions
self.env.config.set('trac', 'permission_store',
'DefaultPermissionStore')
PermissionSystem(self.env).grant_permission('joe', 'BUILD_ADMIN')
if DefaultPermissionPolicy is not None and hasattr(DefaultPermissionPolicy, "CACHE_EXPIRY"):
self.old_perm_cache_expiry = DefaultPermissionPolicy.CACHE_EXPIRY
DefaultPermissionPolicy.CACHE_EXPIRY = -1
# Hook up a dummy repository
self.repos = Mock(
get_node=lambda path, rev=None: Mock(get_history=lambda: [],
isdir=True),
normalize_path=lambda path: path,
sync=lambda: None
)
self.env.get_repository = lambda authname=None: self.repos
def environment_created(self):
# Create the required tables
db = self.env.get_db_cnx()
connector, _ = DatabaseManager(self.env)._get_connector()
cursor = db.cursor()
for table in schema:
for stmt in connector.to_sql(table):
cursor.execute(stmt)
# Insert a global version flag
cursor.execute("INSERT INTO system (name,value) "
"VALUES ('tt_version',%s)", (schema_version,))
# Create some default templates
now = int(time.time())
from default_templates import DEFAULT_TEMPLATES
for tt_name, tt_value in DEFAULT_TEMPLATES:
record = [
now,
SYSTEM_USER,
tt_name,
"description",
tt_value,
]
TT_Template.insert(self.env, record)
db.commit()
def _upgrade_db(self, db):
try:
try:
from trac.db import DatabaseManager
db_backend, _ = DatabaseManager(self.env)._get_connector()
except ImportError:
db_backend = self.env.get_db_cnx()
cursor = db.cursor()
for table in self.SCHEMA:
for stmt in db_backend.to_sql(table):
self.env.log.debug(stmt)
cursor.execute(stmt)
db.commit()
# Migrate old data
if self._need_migration(db):
cursor = db.cursor()
cursor.execute("INSERT INTO tags (tagspace, name, tag) SELECT 'wiki', name, namespace FROM wiki_namespace")
cursor.execute("DROP TABLE wiki_namespace")
db.commit()
except Exception, e:
db.rollback()
self.env.log.error(e, exc_info=1)
raise TracError(str(e))
def do_upgrade(env, ver, cursor):
"""Convert time values from integer seconds to integer microseconds."""
tables = [
('attachment', {'time': ('int', 'int64')}),
('wiki', {'time': ('int', 'int64')}),
('revision', {'time': ('int', 'int64')}),
('ticket', {'time': ('int', 'int64'),
'changetime': ('int', 'int64')}),
('ticket_change', {'time': ('int', 'int64')}),
('milestone', {'due': ('int', 'int64'),
'completed': ('int', 'int64')}),
('version', {'time': ('int', 'int64')}),
]
db_connector, _ = DatabaseManager(env).get_connector()
db = env.get_db_cnx()
for table, columns in tables:
# Alter column types
for sql in db_connector.alter_column_types(table, columns):
cursor.execute(sql)
# Convert timestamps to microseconds
cursor.execute("UPDATE %s SET %s" % (table,
', '.join("%s=%s*1000000" % (column, column)
for column in columns)))
# Convert comment edit timestamps to microseconds
cursor.execute("UPDATE ticket_change SET newvalue=%s*1000000 "
"WHERE field %s" % (db.cast('newvalue', 'int64'),
db.like()),
('_comment%',))
def do_upgrade(env, ver, cursor):
"""Convert time values from integer seconds to integer microseconds."""
tables = [
('attachment', {'time': ('int', 'int64')}),
('wiki', {'time': ('int', 'int64')}),
('revision', {'time': ('int', 'int64')}),
('ticket', {'time': ('int', 'int64'),
'changetime': ('int', 'int64')}),
('ticket_change', {'time': ('int', 'int64')}),
('milestone', {'due': ('int', 'int64'),
'completed': ('int', 'int64')}),
('version', {'time': ('int', 'int64')}),
]
db_connector, _ = DatabaseManager(env).get_connector()
for table, columns in tables:
# Alter column types
for sql in db_connector.alter_column_types(table, columns):
cursor.execute(sql)
# Convert timestamps to microseconds
cursor.execute("UPDATE %s SET %s" % (table,
', '.join("%s=%s*1000000" % (column, column)
for column in columns)))
# Convert comment edit timestamps to microseconds
with env.db_query as db:
cursor.execute("""
UPDATE ticket_change SET newvalue=%s*1000000
WHERE field %s""" % (db.cast('newvalue', 'int64'), db.like()),
('_comment%',))
def add_last_activity_to_build(env, db):
"""Add a column for storing the last activity to the build table."""
cursor = db.cursor()
build_table_schema_v12 = Table('bitten_build', key='id')[
Column('id', auto_increment=True), Column('config'), Column('rev'),
Column('rev_time', type='int'), Column('platform', type='int'),
Column('slave'), Column('started', type='int'),
Column('stopped', type='int'), Column('status', size=1),
Column('last_activity', type='int'),
Index(['config', 'rev', 'platform'], unique=True)
]
cursor.execute("CREATE TEMPORARY TABLE old_build_v11 AS "
"SELECT * FROM bitten_build")
cursor.execute("DROP TABLE bitten_build")
connector, _ = DatabaseManager(env)._get_connector()
for stmt in connector.to_sql(build_table_schema_v12):
cursor.execute(stmt)
# it's safe to make the last activity the stop time of the build
cursor.execute("INSERT INTO bitten_build (id,config,rev,rev_time,platform,"
"slave,started,stopped,last_activity,status) "
"SELECT id,config,rev,rev_time,platform,"
"slave,started,stopped,stopped,status FROM old_build_v11")
update_sequence(env, db, 'bitten_build', 'id')
def create_table(env, table, conn=None):
"""
Creates a the given table in the given environment. The Table
has to be of type trac.db.Table, and the Environment a
trac.env.Environment.
"""
assert isinstance(env, Environment), \
"[DB]: env should be an instance of trac.env.Environment, got %s" % type(env)
assert isinstance(table, Table), \
"[DB]: table should be an instance of trac.sb.Table, got %s" % type(table)
# Get The Databse Manager
dbm = DatabaseManager(env)
# Get the Connector Object for the current DB schema
connector, args = dbm._get_connector()
# Ask the connector to generate the proper DDL for the table
ddl_gen = connector.to_sql(table)
# Get a DB Connection from the pool, create a cursor and the table
conn, handle_ta = get_db_for_write(env, conn)
try:
cursor = conn.cursor()
for statement in ddl_gen:
debug(env, "[DB]: Table: %s\n%s" % (table.name, statement))
cursor.execute(statement)
if handle_ta:
conn.commit()
debug(env, "[DB]: Successfully Created Table %s" % table.name)
except Exception, e:
if handle_ta:
conn.rollback()
error(env, "[DB]: Unable to Create Table %s, an error occurred: %s" % \
(table.name, exception_to_unicode(e)))
raise
def upgrade_to_0_2_4(env, db, installed_version):
if installed_version>=[0,2,4]:
return True
milestones_shema_ext = [
Table('milestone_change', key=('milestone', 'time', 'field'))[
Column('milestone'),
Column('time', type='int'),
Column('author'),
Column('field'),
Column('oldvalue'),
Column('newvalue'),
Index(['milestone']),
Index(['time'])],
Table('milestone_custom', key=('milestone', 'name'))[
Column('milestone'),
Column('name'),
Column('value')]
]
db = db or env.get_db_cnx()
cursor = db.cursor()
db_backend, _ = DatabaseManager(env)._get_connector()
for table in milestones_shema_ext:
for stmt in db_backend.to_sql(table):
cursor.execute(stmt)
return True
def do_upgrade(env, ver, cursor):
"""Change `subscription_attribute` db table:
+ 'subscription_attribute.authenticated'
"""
cursor.execute("""
CREATE TEMPORARY TABLE subscription_attribute_old
AS SELECT * FROM subscription_attribute
""")
cursor.execute("DROP TABLE subscription_attribute")
connector = DatabaseManager(env)._get_connector()[0]
for table in schema:
for stmt in connector.to_sql(table):
cursor.execute(stmt)
cursor.execute("""
INSERT INTO subscription_attribute
(sid,authenticated,class,realm,target)
SELECT o.sid,s.authenticated,o.class,o.realm,o.target
FROM subscription_attribute_old AS o
LEFT JOIN session AS s
ON o.sid=s.sid
""")
cursor.execute("DROP TABLE subscription_attribute_old")
# DEVEL: Think that an old 'subscriptions' db table may still exist here.
cursor.execute("DROP TABLE IF EXISTS subscriptions")
def _initialize_db(self, db):
# pylint: disable=protected-access
self.log.debug("creating initial db schema for %s.", PLUGIN_NAME)
db_connector, dummy = DatabaseManager(self.env)._get_connector()
for table in db_default.SCHEMA:
for statement in db_connector.to_sql(table):
db(statement)
def test_missing_unique_fields(self):
"""ensure that that insert method works when _meta does not specify
unique fields when inserting more than one ProductResourceMap instances
"""
class TestModel(ModelBase):
"""A test model with no unique_fields"""
_meta = {
'table_name': 'bloodhound_testmodel',
'object_name': 'TestModelObject',
'key_fields': [
'id',
],
'non_key_fields': ['value'],
'unique_fields': [],
}
from trac.db import DatabaseManager
schema = [
TestModel._get_schema(),
]
with self.env.db_transaction as db:
db_connector, dummy = DatabaseManager(self.env)._get_connector()
for table in schema:
for statement in db_connector.to_sql(table):
db(statement)
structure = dict([(table.name, [col.name for col in table.columns])
for table in schema])
tm1 = TestModel(self.env)
tm1._data.update({'id': 1, 'value': 'value1'})
tm1.insert()
tm2 = TestModel(self.env)
tm2._data.update({'id': 2, 'value': 'value2'})
tm2.insert()
def setUp(self):
self.env = EnvironmentStub(enable=['trac.*', 'bitten.*'])
self.env.path = tempfile.mkdtemp()
# Create tables
db = self.env.get_db_cnx()
cursor = db.cursor()
connector, _ = DatabaseManager(self.env)._get_connector()
for table in schema:
for stmt in connector.to_sql(table):
cursor.execute(stmt)
# Set up permissions
self.env.config.set('trac', 'permission_store',
'DefaultPermissionStore')
# Hook up a dummy repository
self.repos = Mock(
get_node=lambda path, rev=None: Mock(get_history=lambda: [],
isdir=True),
normalize_path=lambda path: path,
sync=lambda: None,
)
self.repos.authz = Mock(has_permission=lambda path: True, assert_permission=lambda path: None)
self.env.get_repository = lambda authname=None: self.repos
def upgrade_environment(self, db):
if not type(self).__dict__.get('applySchema', False):
self.log.debug("""Not updating schema for \"%s\",
since applySchema is not defined or is False.
""" % type(self).__name__)
return
installed = self.get_installed_version(db)
if installed is None:
self.log.info('Installing TracForm plugin schema %s' % db_version)
db_connector, _ = DatabaseManager(self.env)._get_connector()
db = self._get_db(db)
cursor = db.cursor()
for table in schema:
for stmt in db_connector.to_sql(table):
cursor.execute(stmt)
self.set_installed_version(db, db_version)
self.log.info('Installation of %s successful.' % db_version)
return
self.log.debug('Upgrading schema for "%s".' % type(self).__name__)
for version, fn in self.get_schema_functions():
if version > installed:
self.log.info('Upgrading TracForm plugin schema to %s' %
version)
self.log.info('- %s: %s' % (fn.__name__, fn.__doc__))
db = self._get_db(db)
cursor = db.cursor()
fn(self.env, cursor)
self.set_installed_version(db, version)
installed = version
self.log.info('Upgrade to %s successful.' % version)
def test_missing_unique_fields(self):
"""ensure that that insert method works when _meta does not specify
unique fields when inserting more than one ProductResourceMap instances
"""
class TestModel(ModelBase):
"""A test model with no unique_fields"""
_meta = {'table_name': 'bloodhound_testmodel',
'object_name': 'TestModelObject',
'key_fields': ['id',],
'non_key_fields': ['value'],
'unique_fields': [],}
from trac.db import DatabaseManager
schema = [TestModel._get_schema(), ]
with self.env.db_transaction as db:
db_connector, dummy = DatabaseManager(self.env)._get_connector()
for table in schema:
for statement in db_connector.to_sql(table):
db(statement)
structure = dict([(table.name, [col.name for col in table.columns])
for table in schema])
tm1 = TestModel(self.env)
tm1._data.update({'id':1, 'value':'value1'})
tm1.insert()
tm2 = TestModel(self.env)
tm2._data.update({'id':2, 'value':'value2'})
tm2.insert()
def do_upgrade(env, ver, cursor):
"""Changes to subscription db table:
- 'subscriptions.destination', 'subscriptions.format'
+ 'subscriptions.authenticated', 'subscriptions.transport'
'subscriptions.managed' type='int' --> (default == char)
"""
cursor.execute("""
CREATE TEMPORARY TABLE subscriptions_old
AS SELECT * FROM subscriptions
""")
cursor.execute("DROP TABLE subscriptions")
connector = DatabaseManager(env)._get_connector()[0]
for table in schema:
for stmt in connector.to_sql(table):
cursor.execute(stmt)
cursor.execute("""
INSERT INTO subscriptions
(sid,authenticated,enabled,managed,
realm,category,rule,transport)
SELECT o.sid,s.authenticated,o.enabled,'watcher',
o.realm,o.category,rule,'email'
FROM subscriptions_old AS o
LEFT JOIN session AS s
ON o.sid=s.sid
""")
cursor.execute("DROP TABLE subscriptions_old")
def do_upgrade(env, ver, cursor):
"""Add the cache table."""
table = Table('cache', key='id')[Column('id'),
Column('generation', type='int')]
db_connector, _ = DatabaseManager(env).get_connector()
for stmt in db_connector.to_sql(table):
cursor.execute(stmt)
def setUp(self):
self.env = EnvironmentStub(enable=['trac.*', 'bitten.*'])
self.env.path = tempfile.mkdtemp()
# Create tables
db = self.env.get_db_cnx()
cursor = db.cursor()
connector, _ = DatabaseManager(self.env)._get_connector()
for table in schema:
for stmt in connector.to_sql(table):
cursor.execute(stmt)
# Set up permissions
self.env.config.set('trac', 'permission_store',
'DefaultPermissionStore')
# Hook up a dummy repository
self.repos = Mock(
get_node=lambda path, rev=None: Mock(get_history=lambda: [],
isdir=True),
normalize_path=lambda path: path,
normalize_rev=lambda rev: rev,
sync=lambda: None,
)
self.repos.authz = Mock(has_permission=lambda path: True,
assert_permission=lambda path: None)
self.env.get_repository = lambda authname=None: self.repos
def _upgrade_db(self, db):
try:
try:
from trac.db import DatabaseManager
db_backend, _ = DatabaseManager(self.env)._get_connector()
except ImportError:
db_backend = self.env.get_db_cnx()
cursor = db.cursor()
for table in self.SCHEMA:
for stmt in db_backend.to_sql(table):
self.env.log.debug(stmt)
cursor.execute(stmt)
db.commit()
# Migrate old data
if self._need_migration(db):
cursor = db.cursor()
cursor.execute("INSERT INTO tags (tagspace, name, tag) SELECT 'wiki', name, namespace FROM wiki_namespace")
cursor.execute("DROP TABLE wiki_namespace")
db.commit()
except Exception, e:
db.rollback()
self.env.log.error(e, exc_info=1)
raise TracError(str(e))
def setUp(self):
self.env = EnvironmentStub()
if hasattr(self.env, 'dburi'):
# Trac gained support for testing against different databases in 0.11.5
# If this support is available, we copy the test db uri configuration
# into the main test config so it can be picked up by
# upgrades.parse_scheme()
self.env.config.set('trac', 'database', self.env.dburi)
self.env.path = tempfile.mkdtemp()
logs_dir = self.env.config.get("bitten", "logs_dir", "log/bitten")
if os.path.isabs(logs_dir):
raise ValueError(
"Should not have absolute logs directory for temporary test")
logs_dir = os.path.join(self.env.path, logs_dir)
self.logs_dir = logs_dir
db = self.env.get_db_cnx()
cursor = db.cursor()
for table_name in self.other_tables:
cursor.execute("DROP TABLE IF EXISTS %s" % (table_name, ))
connector, _ = DatabaseManager(self.env)._get_connector()
for table in self.schema:
cursor.execute("DROP TABLE IF EXISTS %s" % (table.name, ))
for stmt in connector.to_sql(table):
cursor.execute(stmt)
db.commit()
def upgrade_environment(self, db):
old_version = current_version = self.get_version(db)
db_connector, dummy = DatabaseManager(self.env)._get_connector()
while current_version < self.version:
if current_version > 0:
db("CREATE TEMPORARY TABLE dummy_table_old AS "
"SELECT * FROM dummy_table")
db("DROP TABLE dummy_table")
table = self.construct_dummy_table(current_version+1)
for statement in db_connector.to_sql(table):
db(statement)
if current_version > 0:
cols = ['id'] + ['v%i' % (i+1)
for i in range(current_version+1)]
db("""INSERT INTO dummy_table (%s)
SELECT %s, '' FROM dummy_table_old
""" % (', '.join(cols), ', '.join(cols[:-1])))
db("DROP TABLE dummy_table_old")
current_version += 1
if current_version != old_version:
self.update_version(db, current_version)
def test_db_table():
# Create a EnvironmentStub
env = EnvironmentStub()
# Create a test table
table = Table('test', key=['id'])[Column('id', type='integer'),
Column(Key.NAME, type='text')]
# Get The Databse Manager
dbm = DatabaseManager(env)
# Get the Connector Object for the current DB schema
connector, args = dbm._get_connector()
# Ask the connector to generate the proper DDL for the table
ddl_gen = connector.to_sql(table)
# Get a DB Connection from the pool, create a cursor and the table
conn = dbm.get_connection()
try:
cursor = conn.cursor()
for statement in ddl_gen:
print "Table: %s\n%s" % (table.name, statement)
cursor.execute(statement)
conn.commit()
print "Successfully Created Table %s" % table.name
except Exception, e:
conn.rollback()
print "[ERROR]: Unable to Create Table %s, an error occurred: %s" % \
(table.name, str(e))
def upgrade_environment(self, db):
old_version = current_version = self.get_version(db)
db_connector, dummy = DatabaseManager(self.env)._get_connector()
while current_version < self.version:
if current_version > 0:
db("CREATE TEMPORARY TABLE dummy_table_old AS "
"SELECT * FROM dummy_table")
db("DROP TABLE dummy_table")
table = self.construct_dummy_table(current_version + 1)
for statement in db_connector.to_sql(table):
db(statement)
if current_version > 0:
cols = ['id'] + [
'v%i' % (i + 1) for i in range(current_version + 1)
]
db("""INSERT INTO dummy_table (%s)
SELECT %s, '' FROM dummy_table_old
""" % (', '.join(cols), ', '.join(cols[:-1])))
db("DROP TABLE dummy_table_old")
current_version += 1
if current_version != old_version:
self.update_version(db, current_version)
def _create_multiproduct_tables(self, db):
self.log.debug("Creating initial db tables for %s plugin." %
PLUGIN_NAME)
db_connector, dummy = DatabaseManager(self.env)._get_connector()
for table in self.SCHEMA:
for statement in db_connector.to_sql(table):
db(statement)
def do_upgrade(env, ver, cursor):
"""Add two more subscription db tables for a better normalized schema."""
connector = DatabaseManager(env)._get_connector()[0]
for table in schema:
for stmt in connector.to_sql(table):
cursor.execute(stmt)
def add_log_table(env, db):
"""Add a table for storing the builds logs."""
from bitten.model import BuildLog, BuildStep
cursor = db.cursor()
connector, _ = DatabaseManager(env)._get_connector()
for table in BuildLog._schema:
for stmt in connector.to_sql(table):
cursor.execute(stmt)
cursor.execute("SELECT build,name,log FROM bitten_step "
"WHERE log IS NOT NULL")
for build, step, log in cursor:
build_log = BuildLog(env, build, step)
build_log.messages = [(BuildLog.INFO, msg) for msg in log.splitlines()]
build_log.insert(db)
cursor.execute("CREATE TEMPORARY TABLE old_step AS SELECT * FROM bitten_step")
cursor.execute("DROP TABLE bitten_step")
for table in BuildStep._schema:
for stmt in connector.to_sql(table):
cursor.execute(stmt)
cursor.execute("INSERT INTO bitten_step (build,name,description,status,"
"started,stopped) SELECT build,name,description,status,"
"started,stopped FROM old_step")
def _create_multiproduct_tables(self, db):
self.log.debug("Creating initial db tables for %s plugin." %
PLUGIN_NAME)
db_connector, dummy = DatabaseManager(self.env)._get_connector()
for table in self.SCHEMA:
for statement in db_connector.to_sql(table):
db(statement)
def _upgrade_db(self, db):
"""Each schema version should have its own upgrade module, named
upgrades/dbN.py, where 'N' is the version number (int).
"""
dbm = DatabaseManager(self.env)
if dbm.needs_upgrade(db_default.version, db_default.name):
if not dbm.get_database_version(db_default.name):
dbm.create_tables(db_default.tables)
dbm.set_database_version(db_default.version, db_default.name)
else:
dbm.upgrade(db_default.version, db_default.name,
'crashdump.upgrades')
try:
Type(name='crashdump', env=self.env)
except ResourceNotFound:
crashdump_ticket_type = Type(env=self.env)
crashdump_ticket_type.name = 'crashdump'
crashdump_ticket_type.description = 'crashdump'
crashdump_ticket_type.insert()
custom = self.config['ticket-custom']
config_dirty = False
if 'linked_crash' not in custom:
custom.set('linked_crash', 'text')
custom.set('linked_crash.label', 'Linked crash')
config_dirty = True
if config_dirty:
self.config.save()
def do_upgrade(env, ver, cursor):
"""Rename the columns `kind` and `change` in the `node_change` table for
compatibity with MySQL.
"""
cursor.execute(
"CREATE TEMPORARY TABLE nc_old AS SELECT * FROM node_change")
cursor.execute("DROP TABLE node_change")
table = Table('node_change',
key=('rev', 'path',
'change_type'))[Column('rev'),
Column('path'),
Column('node_type', size=1),
Column('change_type', size=1),
Column('base_path'),
Column('base_rev'),
Index(['rev'])]
db_connector, _ = DatabaseManager(env).get_connector()
for stmt in db_connector.to_sql(table):
cursor.execute(stmt)
cursor.execute("INSERT INTO node_change (rev,path,node_type,change_type,"
"base_path,base_rev) SELECT rev,path,kind,change,"
"base_path,base_rev FROM nc_old")
cursor.execute("DROP TABLE nc_old")
def setUp(self):
self.env = EnvironmentStub()
if hasattr(self.env, 'dburi'):
# Trac gained support for testing against different databases in 0.11.5
# If this support is available, we copy the test db uri configuration
# into the main test config so it can be picked up by
# upgrades.parse_scheme()
self.env.config.set('trac', 'database', self.env.dburi)
self.env.path = tempfile.mkdtemp()
logs_dir = self.env.config.get("bitten", "logs_dir", "log/bitten")
if os.path.isabs(logs_dir):
raise ValueError("Should not have absolute logs directory for temporary test")
logs_dir = os.path.join(self.env.path, logs_dir)
self.logs_dir = logs_dir
db = self.env.get_db_cnx()
cursor = db.cursor()
for table_name in self.other_tables:
cursor.execute("DROP TABLE IF EXISTS %s" % (table_name,))
connector, _ = DatabaseManager(self.env)._get_connector()
for table in self.schema:
cursor.execute("DROP TABLE IF EXISTS %s" % (table.name,))
for stmt in connector.to_sql(table):
cursor.execute(stmt)
db.commit()
def do_upgrade(env, cursor):
db_connector, _ = DatabaseManager(env)._get_connector()
print "Upgrading forum database tables to version 3\n"
# Backup old topic table
cursor.execute("CREATE TEMPORARY TABLE topic_old AS SELECT * FROM topic")
cursor.execute("DROP TABLE topic")
# Create tables
for table in tables:
for statement in db_connector.to_sql(table):
cursor.execute(statement)
sql = "INSERT INTO topic (id, forum, time, author, subject, body, lastreply) "\
"SELECT id, forum, time, author, subject, body, time " \
"FROM topic_old"
# Copy old topics
print sql + "\n"
cursor.execute(sql)
# Set database schema version.
cursor.execute("UPDATE system SET value = '3' WHERE" \
" name = 'discussion_version'")
print "done.\n"
def do_upgrade(env, ver, cursor):
# Change repository key from reponame to a surrogate id
cursor.execute("SELECT id FROM repository "
"UNION SELECT repos AS id FROM revision "
"UNION SELECT repos AS id FROM node_change "
"ORDER BY id")
id_name_list = [(i + 1, name) for i, (name,) in enumerate(cursor)]
cursor.execute("CREATE TEMPORARY TABLE repo_old "
"AS SELECT * FROM repository")
cursor.execute("DROP TABLE repository")
cursor.execute("CREATE TEMPORARY TABLE rev_old "
"AS SELECT * FROM revision")
cursor.execute("DROP TABLE revision")
cursor.execute("CREATE TEMPORARY TABLE nc_old "
"AS SELECT * FROM node_change")
cursor.execute("DROP TABLE node_change")
tables = [Table('repository', key=('id', 'name'))[
Column('id', type='int'),
Column('name'),
Column('value')],
Table('revision', key=('repos', 'rev'))[
Column('repos', type='int'),
Column('rev', key_size=20),
Column('time', type='int'),
Column('author'),
Column('message'),
Index(['repos', 'time'])],
Table('node_change', key=('repos', 'rev', 'path', 'change_type'))[
Column('repos', type='int'),
Column('rev', key_size=20),
Column('path', key_size=255),
Column('node_type', size=1),
Column('change_type', size=1, key_size=2),
Column('base_path'),
Column('base_rev'),
Index(['repos', 'rev'])]]
db_connector, _ = DatabaseManager(env)._get_connector()
for table in tables:
for stmt in db_connector.to_sql(table):
cursor.execute(stmt)
cursor.executemany("INSERT INTO repository (id,name,value) "
"VALUES (%s,'name',%s)", id_name_list)
cursor.executemany("INSERT INTO repository (id,name,value) "
"SELECT %s,name,value FROM repo_old WHERE id=%s",
id_name_list)
cursor.execute("DROP TABLE repo_old")
cursor.executemany("INSERT INTO revision (repos,rev,time,author,message) "
"SELECT %s,rev,time,author,message FROM rev_old "
"WHERE repos=%s", id_name_list)
cursor.execute("DROP TABLE rev_old")
cursor.executemany("INSERT INTO node_change (repos,rev,path,node_type,"
" change_type,base_path,base_rev) "
"SELECT %s,rev,path,node_type,change_type,base_path,"
" base_rev FROM nc_old WHERE repos=%s", id_name_list)
cursor.execute("DROP TABLE nc_old")
def do_db_create(db):
db_manager, _ = DatabaseManager(self.env)._get_connector()
cursor = db.cursor()
for table in db_default.schema:
for sql in db_manager.to_sql(table):
cursor.execute(sql)
cursor.execute('INSERT INTO system (name, value) VALUES (%s, %s)',
(db_default.name, db_default.version))
def upgrade_environment(self, db):
db_backend, _ = DatabaseManager(self.env)._get_connector()
cursor = db.cursor()
for table in self.schema:
for stmt in db_backend.to_sql(table):
self.env.log.debug(stmt)
cursor.execute(stmt)
db.commit()