def applied(self):
"""
Check if the ssh key related tables already exist in trac_admin database.
:returns: True if the tables are there, False if they're not.
"""
conf = Configuration.instance()
with admin_query() as cursor:
cursor.execute('''
SELECT table_name FROM information_schema.tables
WHERE table_schema = '{0}'
AND table_name = 'ssh_keys'
'''.format(conf.db_admin_schema_name))
if cursor.rowcount != 1:
return False
cursor.execute('''
SELECT table_name FROM information_schema.tables
WHERE table_schema = '{0}'
AND table_name = 'ssh_key_update'
'''.format(conf.db_admin_schema_name))
if cursor.rowcount != 1:
return False
return True
def __init__(self, key_arg=None, key_args=None, key_attr=None, key_attrs=None, timeout=0):
"""
:param str key_arg: Single argument name to be used in memcached key construction
:param list key_args: List of key name arguments
:param key_attr: If used within class, use class attribute as part of the key construction
:param key_attrs: List of class attributes (see ``key_attr``)
:param timeout: Specify cache duration in seconds, zero (default) means forever
"""
if key_args and key_arg:
raise ValueError('key_arg and key_args are mutually exclusive')
if key_attr and key_attrs:
raise ValueError('key_attr and key_attrs are mutually exclusive')
self.timeout = timeout
self.key_args = []
self.key_attrs = []
self.mc = Configuration.instance().getMemcachedInstance()
if key_args:
if not isinstance(key_args, list):
raise ValueError('key_args must be a list')
self.key_args.extend(key_args)
if key_arg:
self.key_args.append(key_arg)
if key_attrs:
if not isinstance(key_attrs, list):
raise ValueError('key_attrs must be a list')
self.key_attrs.extend(key_attrs)
if key_attr:
self.key_attrs.append(key_attr)
def applied(self):
"""
Check if the ssh key related tables already exist in trac_admin database.
:returns: True if the tables are there, False if they're not.
"""
conf = Configuration.instance()
with admin_query() as cursor:
cursor.execute(
"""
SELECT table_name FROM information_schema.tables
WHERE table_schema = '{0}'
AND table_name = 'ssh_keys'
""".format(
conf.db_admin_schema_name
)
)
if cursor.rowcount != 1:
return False
cursor.execute(
"""
SELECT table_name FROM information_schema.tables
WHERE table_schema = '{0}'
AND table_name = 'ssh_key_update'
""".format(
conf.db_admin_schema_name
)
)
if cursor.rowcount != 1:
return False
return True
def main():
conf = Configuration.instance()
try:
module = ActivityCalculator()
module.update_project_activity()
except Exception:
conf.log.exception("Failed to update project activity")
def analytical_conn():
"""
Get a connection to ``trac_admin`` database.
.. NOTE::
DEPRECATED: Use either get_connection() or context managers analytical_query()
or analytical_transaction()
"""
from multiproject.core.configuration import Configuration
return get_connection(Configuration.instance().db_analytical_schema_name)
def analytical_conn():
"""
Get a connection to ``trac_admin`` database.
.. NOTE::
DEPRECATED: Use either get_connection() or context managers analytical_query()
or analytical_transaction()
"""
from multiproject.core.configuration import Configuration
return get_connection(Configuration.instance().db_analytical_schema_name)
def db_query(db_name='', cursor_type=None):
"""
Context manager for making a query into a specific database. This context manager
creates a connection to the specified database and yields a cursor to the database.
After user is done, the connection is closed automatically (or returned to pool).
:param db_name: Optional name of the database to get the cursor for reading
:param cursor_type: Optional cursor type
.. NOTE::
This context manager is meant for read-only operations into database. If you want
automatic commits and rollbacks, use db_transaction(), admin_transaction() or
analytical_transaction().
Example::
def some_function():
row = []
with db_query('trac_admin') as cursor:
query = "SELECT * FROM users WHERE username LIKE %s"
cursor.execute(query, 'mika%')
row = cursor.fetchall()
# Do something
"""
from multiproject.core.configuration import Configuration
conf = Configuration.instance()
conn = None
cursor = None
try:
conn = get_connection(db_name)
except Exception:
conf.log.error('Failed to get db connection for queries to %s' %
db_name)
raise
try:
cursor = conn.cursor(cursor_type)
# Uncomment for debugging
#cursor.execute = debug_db_execute(cursor.execute)
yield cursor
except Exception:
conf.log.exception('Exception in db_query(%s)' % db_name)
raise
finally:
cursor.close()
conn.close()
def applied(self):
"""
Check if column exists or not
:returns: True if exists, otherwise False
"""
conf = Configuration.instance()
with admin_query() as cursor:
cursor.execute('''
SELECT table_name FROM information_schema.tables
WHERE table_schema = '{0}'
AND table_name = 'tags'
'''.format(conf.db_admin_schema_name))
return cursor.rowcount != 1
def applied(self):
"""
Check if column exists or not
:returns: True if exists, otherwise False
"""
conf = Configuration.instance()
with admin_query() as cursor:
cursor.execute('''
SELECT table_name FROM information_schema.tables
WHERE table_schema = '{0}'
AND table_name = 'tags'
'''.format(conf.db_admin_schema_name))
return cursor.rowcount != 1
def applied(self):
"""
Checks from database if the the procedure exists
"""
conf = Configuration.instance()
with admin_query() as cursor:
cursor.execute('''
SHOW PROCEDURE STATUS
WHERE
Name = 'addactivityforproject' AND
Db = '{0}'
'''.format(conf.db_admin_schema_name))
return cursor.rowcount == 0
def applied(self):
"""
Checks from database if the the procedure exists
"""
conf = Configuration.instance()
with admin_query() as cursor:
cursor.execute('''
SHOW PROCEDURE STATUS
WHERE
Name = 'create_group_from_template' AND
Db = '{0}'
'''.format(conf.db_admin_schema_name))
return cursor.rowcount == 0
def db_transaction(db_name=''):
"""
Context manager for doing an update, insert, delete or some other transaction
into a database. This will take care of providing cursor and taking a connection for
the database for the calling code. After everything is done, the cursor and connection
are disposed. If no exception is returned, all changes to database are committed. If
an exception is raised, changes are rolled back.
:param db_name: Name of the database to get the cursor for modifying
Example::
def some_function():
data = (1, 2, 3)
with admin_transaction() as cursor:
try:
cursor.execute("DELETE ...", data)
except Exception:
conf.log.exception("Failed to delete...")
raise
return
"""
from multiproject.core.configuration import Configuration
conf = Configuration.instance()
conn = None
cursor = None
try:
conn = get_connection(db_name)
except Exception:
conf.log.error(
'Failed to get database connection for transactions to db %s' %
db_name)
raise
# Simple fallbacking from commit to rollback
try:
cursor = conn.cursor()
# Uncomment for debugging
#cursor.execute = debug_db_execute(cursor.execute)
yield cursor
conn.commit()
except Exception:
conf.log.error('Exception in db_transaction(%s), running rollback' %
db_name)
conn.rollback()
raise
finally:
cursor.close()
conn.close()
def db_query(db_name='', cursor_type=None):
"""
Context manager for making a query into a specific database. This context manager
creates a connection to the specified database and yields a cursor to the database.
After user is done, the connection is closed automatically (or returned to pool).
:param db_name: Optional name of the database to get the cursor for reading
:param cursor_type: Optional cursor type
.. NOTE::
This context manager is meant for read-only operations into database. If you want
automatic commits and rollbacks, use db_transaction(), admin_transaction() or
analytical_transaction().
Example::
def some_function():
row = []
with db_query('trac_admin') as cursor:
query = "SELECT * FROM users WHERE username LIKE %s"
cursor.execute(query, 'mika%')
row = cursor.fetchall()
# Do something
"""
from multiproject.core.configuration import Configuration
conf = Configuration.instance()
conn = None
cursor = None
try:
conn = get_connection(db_name)
except Exception:
conf.log.error('Failed to get db connection for queries to %s' % db_name)
raise
try:
cursor = conn.cursor(cursor_type)
# Uncomment for debugging
#cursor.execute = debug_db_execute(cursor.execute)
yield cursor
except Exception:
conf.log.exception('Exception in db_query(%s)' % db_name)
raise
finally:
cursor.close()
conn.close()
def debug_db_execute(fn):
"""
Simple debugger for database queries: use this function to patch the
original db cursor.execute::
def db_query(db_name='', cursor_type=None):
conn = get_connection(db_name)
cursor = conn.cursor(cursor_type)
# PATCH!
cursor.execute = debug_db_execute(cursor.execute)
yield cursor
As outcome, all the queries are logged for further analysis
.. important::
Do *not* enable this in production environment!
"""
import re
import traceback
from datetime import datetime
from multiproject.core.configuration import Configuration
stack_depth = 5
conf = Configuration.instance()
reprex = re.compile('\\n\s*')
def execute(*args, **kwargs):
query = [reprex.sub(' ', str(arg)).strip() for arg in args]
before = datetime.utcnow()
# Run the actual query
output = fn(*args, **kwargs)
# Log output:
diff = datetime.utcnow() - before
diffs = str(diff).rsplit(':', 1)[1]
# Raise exception to get execute information
try:
raise Exception('traceback')
except Exception:
trace = traceback.extract_stack(limit=stack_depth)
conf.log.error('SQL: %s, %s (took: %s sec, stack: %s)' %
(query, kwargs, diffs, trace[:-1]))
return output
return execute
def debug_db_execute(fn):
"""
Simple debugger for database queries: use this function to patch the
original db cursor.execute::
def db_query(db_name='', cursor_type=None):
conn = get_connection(db_name)
cursor = conn.cursor(cursor_type)
# PATCH!
cursor.execute = debug_db_execute(cursor.execute)
yield cursor
As outcome, all the queries are logged for further analysis
.. important::
Do *not* enable this in production environment!
"""
import re
import traceback
from datetime import datetime
from multiproject.core.configuration import Configuration
stack_depth = 5
conf = Configuration.instance()
reprex = re.compile('\\n\s*')
def execute(*args, **kwargs):
query = [reprex.sub(' ', str(arg)).strip() for arg in args]
before = datetime.utcnow()
# Run the actual query
output = fn(*args, **kwargs)
# Log output:
diff = datetime.utcnow() - before
diffs = str(diff).rsplit(':', 1)[1]
# Raise exception to get execute information
try:
raise Exception('traceback')
except Exception:
trace = traceback.extract_stack(limit=stack_depth)
conf.log.error('SQL: %s, %s (took: %s sec, stack: %s)' % (query, kwargs, diffs, trace[:-1]))
return output
return execute
def db_transaction(db_name=''):
"""
Context manager for doing an update, insert, delete or some other transaction
into a database. This will take care of providing cursor and taking a connection for
the database for the calling code. After everything is done, the cursor and connection
are disposed. If no exception is returned, all changes to database are committed. If
an exception is raised, changes are rolled back.
:param db_name: Name of the database to get the cursor for modifying
Example::
def some_function():
data = (1, 2, 3)
with admin_transaction() as cursor:
try:
cursor.execute("DELETE ...", data)
except Exception:
conf.log.exception("Failed to delete...")
raise
return
"""
from multiproject.core.configuration import Configuration
conf = Configuration.instance()
conn = None
cursor = None
try:
conn = get_connection(db_name)
except Exception:
conf.log.error('Failed to get database connection for transactions to db %s' % db_name)
raise
# Simple fallbacking from commit to rollback
try:
cursor = conn.cursor()
# Uncomment for debugging
#cursor.execute = debug_db_execute(cursor.execute)
yield cursor
conn.commit()
except Exception:
conf.log.error('Exception in db_transaction(%s), running rollback' % db_name)
conn.rollback()
raise
finally:
cursor.close()
conn.close()
def applied(self):
"""
Check if the global_announcements table already exist in trac_admin database.
:returns: True if the table is there, False if it's not.
"""
conf = Configuration.instance()
with admin_query() as cursor:
cursor.execute('''
SELECT table_name FROM information_schema.tables
WHERE table_schema = '{0}'
AND table_name = 'global_announcements'
'''.format(conf.db_admin_schema_name))
if cursor.rowcount != 1:
return False
return True
def analytical_transaction():
"""
Context manager for doign an update, insert, delete or some other transaction
into ``trac_analytical`` database. This will take care of providing cursor and taking
a connection for the database for the calling code. After everything is done, the
cursor and connection are disposed.
For example of use and more details, see db_transaction()
"""
from multiproject.core.configuration import Configuration
conf = Configuration.instance()
with db_transaction(conf.db_analytical_schema_name) as cursor:
try:
yield cursor
except Exception:
conf.log.error('Exception in analytics_transaction()')
raise
def applied(self):
"""
Check if the global_announcements table already exist in trac_admin database.
:returns: True if the table is there, False if it's not.
"""
conf = Configuration.instance()
with admin_query() as cursor:
cursor.execute('''
SELECT table_name FROM information_schema.tables
WHERE table_schema = '{0}'
AND table_name = 'global_announcements'
'''.format(conf.db_admin_schema_name))
if cursor.rowcount != 1:
return False
return True
def analytical_transaction():
"""
Context manager for doign an update, insert, delete or some other transaction
into ``trac_analytical`` database. This will take care of providing cursor and taking
a connection for the database for the calling code. After everything is done, the
cursor and connection are disposed.
For example of use and more details, see db_transaction()
"""
from multiproject.core.configuration import Configuration
conf = Configuration.instance()
with db_transaction(conf.db_analytical_schema_name) as cursor:
try:
yield cursor
except Exception:
conf.log.error('Exception in analytics_transaction()')
raise
def analytical_query(cursor_type=None):
"""
Context manager implementation for getting a configured ``trac_analytical`` database cursor
for executing a query into database. This handles exceptions for users and in the end
closes the connection.
:param cursor_type: Optional MySQLdb cursor type
For more example on how to use this, see db_query()
"""
from multiproject.core.configuration import Configuration
conf = Configuration.instance()
with db_query(conf.db_analytical_schema_name, cursor_type) as cursor:
try:
yield cursor
except Exception:
conf.log.error('Exception in analytics_query()')
raise
def analytical_query(cursor_type=None):
"""
Context manager implementation for getting a configured ``trac_analytical`` database cursor
for executing a query into database. This handles exceptions for users and in the end
closes the connection.
:param cursor_type: Optional MySQLdb cursor type
For more example on how to use this, see db_query()
"""
from multiproject.core.configuration import Configuration
conf = Configuration.instance()
with db_query(conf.db_analytical_schema_name, cursor_type) as cursor:
try:
yield cursor
except Exception:
conf.log.error('Exception in analytics_query()')
raise
def safe_int(value):
"""
Helper function to check and retrieve integer
.. NOTE:: Please use cursor escaping instead where possible
"""
real = None
try:
real = int(value)
except:
# Write a warning in log if this looks suspicious
# NOTE: this import must remain here or circular import will occur
from multiproject.core.configuration import Configuration
conf = Configuration.instance()
if value is not None:
conf.log.warning("Failed validating '%s' as int" % str(value))
return real
def safe_int(value):
"""
Helper function to check and retrieve integer
.. NOTE:: Please use cursor escaping instead where possible
"""
real = None
try:
real = int(value)
except:
# Write a warning in log if this looks suspicious
# NOTE: this import must remain here or circular import will occur
from multiproject.core.configuration import Configuration
conf = Configuration.instance()
if value is not None:
conf.log.warning("Failed validating '%s' as int" % str(value))
return real
def __init__(self):
self.section = 'multiproject-files'
# logic is a mapping from configuration_key to
# tuple (default_value, value_getter, strip_value)
_as_bool = lambda ignored_key, value: as_bool(value)
self.logic = {
'default_downloads_directory':
('downloads', self._get_directory, True),
'sys_dav_root': ('/var/www/trac/webdav', self._get_abspath, True),
'url_dav_path': ('dav', self._get_relative_url, True),
'downloads_dir_customizable': ('True', _as_bool, True),
}
values = {}
self.defaults = {}
value = None
options = {}
try:
items = Configuration.instance().config_parser.items(self.section)
for item in items:
options[item[0]] = item[1]
except NoSectionError:
options = {}
for key, logic in self.logic.items():
default_val, handler, do_strip = logic
self.defaults[key] = default_val
if key not in options:
value = handler(key, default_val)
else:
value = options[key].strip() if do_strip else options[key]
value = handler(key, value)
values[key] = value
self.default_downloads_directory = values[
'default_downloads_directory']
self.sys_dav_root = values['sys_dav_root']
self.url_dav_path = values['url_dav_path']
self.downloads_dir_customizable = values['downloads_dir_customizable']
def get_connection(db_name=''):
"""
Get a database connection from pool. Raises (and logs) an exception if connection fails.
Calling program needs to remember to close the connection after it is no longer needed.
:param db_name: Optional database name to connect into
:returns: Database connection, remember to call .close() for the connection when done
"""
# Importing this here, since importing it globally would be a circular dependency
# This use of configuration is bad, but better than importing the instanced global variable
# from the module itself.
from multiproject.core.configuration import Configuration
conf = Configuration.instance()
conn = None
try:
if not conf.use_alchemy_pool:
conn = MySQLdb.connect(host=conf.db_host,
user=conf.db_user,
passwd=conf.db_password,
port=int(conf.db_port),
db=db_name,
charset='utf8')
else:
conn = _get_pool().connect(host=conf.db_host,
user=conf.db_user,
passwd=conf.db_password,
port=int(conf.db_port),
db=db_name,
charset='utf8')
except Exception:
conf.log.exception("Failed to open database connection to db '%s'" %
db_name)
raise
return conn
def get_connection(db_name=''):
"""
Get a database connection from pool. Raises (and logs) an exception if connection fails.
Calling program needs to remember to close the connection after it is no longer needed.
:param db_name: Optional database name to connect into
:returns: Database connection, remember to call .close() for the connection when done
"""
# Importing this here, since importing it globally would be a circular dependency
# This use of configuration is bad, but better than importing the instanced global variable
# from the module itself.
from multiproject.core.configuration import Configuration
conf = Configuration.instance()
conn = None
try:
if not conf.use_alchemy_pool:
conn = MySQLdb.connect(host=conf.db_host,
user=conf.db_user,
passwd=conf.db_password,
port=int(conf.db_port),
db=db_name,
charset='utf8')
else:
conn = _get_pool().connect(host=conf.db_host,
user=conf.db_user,
passwd=conf.db_password,
port=int(conf.db_port),
db=db_name,
charset='utf8')
except Exception:
conf.log.exception("Failed to open database connection to db '%s'" % db_name)
raise
return conn
def __init__(self):
self.section = 'multiproject-files'
# logic is a mapping from configuration_key to
# tuple (default_value, value_getter, strip_value)
_as_bool = lambda ignored_key, value: as_bool(value)
self.logic = {
'default_downloads_directory': ('downloads', self._get_directory, True),
'sys_dav_root': ('/var/www/trac/webdav', self._get_abspath, True),
'url_dav_path': ('dav', self._get_relative_url, True),
'downloads_dir_customizable': ('True', _as_bool, True),
}
values = {}
self.defaults = {}
value = None
options = {}
try:
items = Configuration.instance().config_parser.items(self.section)
for item in items:
options[item[0]] = item[1]
except NoSectionError:
options = {}
for key, logic in self.logic.items():
default_val, handler, do_strip = logic
self.defaults[key] = default_val
if key not in options:
value = handler(key, default_val)
else:
value = options[key].strip() if do_strip else options[key]
value = handler(key, value)
values[key] = value
self.default_downloads_directory = values['default_downloads_directory']
self.sys_dav_root = values['sys_dav_root']
self.url_dav_path = values['url_dav_path']
self.downloads_dir_customizable = values['downloads_dir_customizable']
def __init__(self, env_name):
conf = Configuration.instance()
self.log = conf.log
self.news_forum_name = conf.news_forum_name
self.env_name = db.safe_string(env_name)
def __init__(self):
self.conf = Configuration.instance()
def __init__(self, env_name):
conf = Configuration.instance()
self.log = conf.log
self.news_forum_name = conf.news_forum_name
self.env_name = db.safe_string(env_name)
def __init__(self):
self.conf = Configuration.instance()
"""
def wrapper(*args, **kwargs):
import inspect
from multiproject.core.configuration import Configuration
caller_info = "<unknown>"
frames = None
try:
frames = inspect.stack()
if frames:
# Caller info is the second frame
caller = inspect.getframeinfo(frames[1][0])
module = inspect.getmodule(frames[1][0])
caller_info = "%s.%s():%d" % (module.__name__, caller[2], caller[1])
except Exception, e:
caller_info = str(e)
finally:
# Deleting the frames reference is apparently important. Otherwise this might affect stack
# dumps in cases where a real exception is raised
if frames:
del frames
conf = Configuration.instance()
conf.log.warning("Deprecated call %s.%s() from %s" %
(func.__module__, func.__name__, caller_info))
return func(*args, **kwargs)
return wrapper
def wrapper(*args, **kwargs):
import inspect
from multiproject.core.configuration import Configuration
caller_info = "<unknown>"
frames = None
try:
frames = inspect.stack()
if frames:
# Caller info is the second frame
caller = inspect.getframeinfo(frames[1][0])
module = inspect.getmodule(frames[1][0])
caller_info = "%s.%s():%d" % (module.__name__, caller[2],
caller[1])
except Exception, e:
caller_info = str(e)
finally:
# Deleting the frames reference is apparently important. Otherwise this might affect stack
# dumps in cases where a real exception is raised
if frames:
del frames
conf = Configuration.instance()
conf.log.warning("Deprecated call %s.%s() from %s" %
(func.__module__, func.__name__, caller_info))
return func(*args, **kwargs)
return wrapper
def __init__(self, fn):
self.fn = fn
self.mc = Configuration.instance().getMemcachedInstance()
