class GitwebProjectsRepositoryProvider(Component):
implements(IRepositoryProvider)
projects_list = PathOption('git',
'projects_list',
doc='Path to a gitweb-formatted projects.list')
projects_base = PathOption('git',
'projects_base',
doc='Path to the base of your git projects')
projects_url = Option(
'git',
'projects_url',
doc='Template for project URLs. %s will be replaced with the repo name'
)
def get_repositories(self):
if not self.projects_list:
return
for line in open(self.projects_list):
line = line.strip()
name = line
if name.endswith('.git'):
name = name[:-4]
repo = {
'dir': os.path.join(self.projects_base, line),
'type': 'git',
}
description_path = os.path.join(repo['dir'], 'description')
if os.path.exists(description_path):
repo['description'] = open(description_path).read().strip()
if self.projects_url:
repo['url'] = self.projects_url % name
yield name, repo
class GitwebProjectsRepositoryProvider(Component):
implements(IRepositoryProvider)
projects_list = PathOption(
'gitweb-repositories',
'projects_list',
doc="""Path to a gitweb-formatted projects.list""")
projects_base = PathOption('gitweb-repositories',
'projects_base',
doc="""Path to the base of your git projects""")
projects_url = Option(
'gitweb-repositories',
'projects_url',
doc="""Template for project URLs. `%s` will be replaced with the repo
name""")
sync_per_request = ListOption('gitweb-repositories',
'sync_per_request',
'',
doc="""Repositories to sync on every request
(not recommended).""")
def get_repositories(self):
"""Retrieve repositories specified in a `projects_list` file."""
if not self.projects_list:
return
if not os.path.exists(self.projects_list):
self.log.warning(
"The [git] projects_list file was not found at "
"'%s'", self.projects_list)
return
with open(self.projects_list, 'r') as fp:
for line in fp:
entries = line.strip().split()
if entries:
name = entries[0]
reponame = name.rstrip('.git')
info = {
'dir': os.path.join(self.projects_base, name),
'sync_per_request': reponame in self.sync_per_request,
'type': 'git',
}
description_path = \
os.path.join(info['dir'], 'description')
if os.path.exists(description_path):
with open(description_path, 'r') as fd:
info['description'] = fd.read().strip()
if self.projects_url:
info['url'] = self.projects_url % reponame
yield reponame, info
class ServiceProvider(Component):
implements(IGraphDatabaseProvider)
resource_uri = Option('neo4j', 'resource_uri', doc="""
""")
options = Options()
classpath = options.add(
ListOption('neo4j', 'classpath', sep=os.pathsep, doc="""
"""))
ext_dirs = options.add(
ListOption('neo4j', 'ext_dirs', sep=os.pathsep, doc="""
"""))
start_server = options.add(
BoolOption('neo4j', 'start_server', doc="""
"""))
server_path = options.add(
PathOption('neo4j', 'server_path', doc="""
"""))
username = options.add(Option('neo4j', 'username', doc="""
"""))
password = options.add(Option('neo4j', 'password', doc="""
"""))
jvm = options.add(PathOption('neo4j', 'jvm', doc="""
"""))
def start(self, resource_uri, params):
if resource_uri is None:
resource_uri = self.resource_uri
if not resource_uri:
resource_uri = os.path.join(self.env.path, 'neodb')
if resource_uri.startswith('file://'):
resource_uri = 'file://' + normalize_path(
self.env.path, resource_uri[7:])
elif '://' not in resource_uri:
resource_uri = normalize_path(self.env.path, resource_uri)
for option, filter in self.options:
if option not in params:
value = getattr(self, option)
if value is not None:
params[option] = filter(value)
return resource_uri, neo4j.GraphDatabase(resource_uri, **params)
instances = {}
def instance(self, resource_uri, params):
if resource_uri not in self.instances:
key = resource_uri
resource_uri, neo = self.start(resource_uri, params)
neo = TracNeo(self.env, neo)
if resource_uri != key:
self.instances[resource_uri] = neo
self.instances[key] = neo
return self.instances[resource_uri]
class GitConnector(Component):
implements(IRepositoryConnector, IWikiSyntaxProvider, IPropertyRenderer)
_persistent_cache = BoolOption('git', 'persistent_cache', 'false',
"Enable persistent caching of commit tree")
_cached_repository = BoolOption('git', 'cached_repository', 'false',
"Wrap `GitRepository` in `CachedRepository`")
_shortrev_len = IntOption('git', 'shortrev_len', 7,
"Length rev sha sums should be tried to be abbreviated to"
" (must be >= 4 and <= 40)")
_git_bin = PathOption('git', 'git_bin', '/usr/bin/git',
"Path to git executable (relative to trac project folder!)")
def __init__(self):
self._version = None
try:
self._version = PyGIT.Storage.git_version(git_bin=self._git_bin)
except PyGIT.GitError, e:
self.log.error("GitError: %s", e)
if self._version:
self.log.info("detected GIT version %s", self._version['v_str'])
self.env.systeminfo.append(('GIT', self._version['v_str']))
if not self._version['v_compatible']:
self.log.error("GIT version %s installed not compatible "
"(need >= %s)" , self._version['v_str'],
self._version['v_min_str'])
class ScreenshotsInit(Component):
"""
Init component initialises database and environment for screenshots
plugin.
"""
implements(IEnvironmentSetupParticipant)
# Configuration options.
path = PathOption('screenshots',
'path',
'../screenshots',
doc='Path where to store uploaded screenshots.')
# IEnvironmentSetupParticipanttr
def environment_created(self):
pass
def environment_needs_upgrade(self, db):
cursor = db.cursor()
# Is database up to date?
return (self._get_db_version(cursor) != last_db_version) or not \
os.path.isdir(self.path)
def upgrade_environment(self, db):
cursor = db.cursor()
# Create screenshots directory if not exists.
if not os.path.isdir(self.path):
os.mkdir(os.path.normpath(self.path))
# Get current database schema version
db_version = self._get_db_version(cursor)
# Is this clean installation?
if db_version == 0:
# Perform single upgrade.
module = __import__('tracscreenshots.db.db%s' % (last_db_version),
globals(), locals(), ['do_upgrade'])
module.do_upgrade(self.env, cursor, False)
else:
# Perform incremental upgrades
for I in range(db_version + 1, last_db_version + 1):
script_name = 'db%i' % (I)
module = __import__('tracscreenshots.db.%s' % (script_name),
globals(), locals(), ['do_upgrade'])
module.do_upgrade(self.env, cursor, True)
def _get_db_version(self, cursor):
try:
sql = ("""SELECT value
FROM system
WHERE name='screenshots_version'""")
self.log.debug(sql)
cursor.execute(sql)
for row in cursor:
return int(row[0])
return 0
except:
return 0
class GitwebProjectsRepositoryProvider(Component):
implements(IRepositoryProvider)
projects_list = PathOption(
'git',
'projects_list',
doc="""Path to a gitweb-formatted projects.list""")
projects_base = PathOption('git',
'projects_base',
doc="""Path to the base of your git projects""")
projects_url = Option(
'git',
'projects_url',
doc="""Template for project URLs. %s will be replaced with the repo
name""")
def get_repositories(self):
if not self.projects_list:
return
if not os.path.exists(self.projects_list):
self.log.warn(
"The [git] projects_list file was not found at "
"'%s'", self.projects_list)
return
with open(self.projects_list, 'r') as fp:
for line in fp:
entries = line.strip().split()
if entries:
reponame = entries[0]
repo = {
'dir': os.path.join(self.projects_base, reponame),
'type': 'git',
}
description_path = \
os.path.join(repo['dir'], 'description')
if os.path.exists(description_path):
with open(description_path, 'r') as fd:
repo['description'] = fd.read().strip()
name = reponame.rstrip('.git')
if self.projects_url:
repo['url'] = self.projects_url % name
yield name, repo
class SubversionConnector(Component):
"""Add support for creating and managing SVN repositories."""
implements(IAdministrativeRepositoryConnector)
svn_authz_file = PathOption('repository-manager', 'svn_authz_file',
'svn.authz',
doc="""The path where the svn auhz file for
repository access control via e.g.
Apache should be created for managed
repositories. If not set, no file will
be written.
""")
pre_commit_hook = Option('repository-manager', 'svn_pre_commit',
doc="""Path to an executable that should be run
before a change is committed to any SVN
repository managed via this plugin.
""")
post_commit_hook = Option('repository-manager', 'svn_post_commit',
doc="""Path to an executable that should be run
after a change was committed to any SVN
repository managed via this plugin.
""")
def get_supported_types(self):
yield ('svn', 0)
def can_fork(self, type):
return False
def can_delete_changesets(self, type):
return False
def can_ban_changesets(self, type):
return False
def create(self, repo):
try:
characters = string.ascii_lowercase + string.digits
layout = ''.join(random.choice(characters) for x in range(20))
layout = os.path.join('/tmp', layout)
os.makedirs(os.path.join(layout, 'trunk'))
os.makedirs(os.path.join(layout, 'branches'))
os.makedirs(os.path.join(layout, 'tags'))
config = { 'fs-type': 'fsfs' }
svn_repos_create(repo['dir'], '', '', None, config)
client = pysvn.Client()
client.set_default_username(repo['owner'])
client.import_(layout, 'file://' + repo['dir'],
'Initial repository layout')
shutil.rmtree(layout)
if self.pre_commit_hook:
os.symlink(os.path.join(self.env.path, self.pre_commit_hook),
os.path.join(repo['dir'], 'hooks/pre-commit'))
if self.post_commit_hook:
os.symlink(os.path.join(self.env.path, self.post_commit_hook),
os.path.join(repo['dir'], 'hooks/post-commit'))
except Exception, e:
raise TracError(_("Failed to initialize repository: ") + str(e))
class SimplecaptchaPlugin(Component):
implements(IRegistrationConfirmation)
dict_file = PathOption('simplecaptcha', 'dictionary_file',
default="http://java.sun.com/docs/books/tutorial/collections/interfaces/examples/dictionary.txt")
def __init__(self):
self.keys = {}
def pre_registration(self, req):
""" Returns the HTML to be added to the registration form """
msg = "Please enter the text below to prove you're not a machine."
key = random.Random().randint(0, sys.maxint)
word = self._random_word()
self.keys[key] = (word, time.time())
content = "<div><p>%s</p>%s" % (msg, self._get_captcha(word))
content += "<label>Enter Word: <input type='text' name='simplecaptcha_captcha' class='textwidget' size='20'></label>"
content += "<input type='hidden' name='simplecaptcha_key' value='%s' /></div>" % key
return content
def verify_registration(self, req):
"""Returns an error message if confirmation fails, or None on success
"""
# keys older than 10min get deleted
timeout = 600
[self.keys.pop(k) for k, (word, timestamp) in self.keys.items()
if time.time()-timestamp >= timeout]
key = int(req.args['simplecaptcha_key'])
if not self.keys.has_key(key): # timeout
return req.redirect(req.href.base + "/register")
correct_answer, timestamp = self.keys.pop(key)
if req.args['simplecaptcha_captcha'].lower() != correct_answer:
return "Sorry, the word you entered was incorrect. Please try again."
return None
def _random_word(self):
"""Returns a random word for use with the captcha"""
min_len = 5
if not globals().has_key('captcha_dict'):
if self.dict_file.startswith("http://"):
f = urllib.urlopen(self.dict_file)
else:
f = open(self.dict_file, "r")
_dict = f.read()
f.close()
_dict = _dict.lower().split()
_dict = [word for word in _dict if word.isalpha() and len(word) > min_len]
globals()['captcha_dict'] = _dict
return random.Random().choice(captcha_dict)
def _get_captcha(self, passphrase):
"""Returns HTML content of captcha as string"""
return skimpyAPI.Pre(passphrase).data()
class AuthzPolicy(Component):
"""Permission policy using an authz-like configuration file.
Refer to SVN documentation for syntax of the authz file. Groups are
supported.
As the fine-grained permissions brought by this permission policy are
often used in complement of the other permission policies (like the
`DefaultPermissionPolicy`), there's no need to redefine all the
permissions here. Only additional rights or restrictions should be added.
=== Installation ===
Enabling this policy requires listing it in `trac.ini`::
{{{
[trac]
permission_policies = AuthzPolicy, DefaultPermissionPolicy
[authz_policy]
authz_file = conf/authzpolicy.conf
}}}
This means that the `AuthzPolicy` permissions will be checked first, and
only if no rule is found will the `DefaultPermissionPolicy` be used.
=== Configuration ===
The `authzpolicy.conf` file is a `.ini` style configuration file.
- Each section of the config is a glob pattern used to match against a
Trac resource descriptor. These descriptors are in the form::
{{{
<realm>:<id>@<version>[/<realm>:<id>@<version> ...]
}}}
Resources are ordered left to right, from parent to child. If any
component is inapplicable, `*` is substituted. If the version pattern is
not specified explicitely, all versions (`@*`) is added implicitly
Example: Match the WikiStart page::
{{{
[wiki:*]
[wiki:WikiStart*]
[wiki:WikiStart@*]
[wiki:WikiStart]
}}}
Example: Match the attachment
``wiki:WikiStart@117/attachment/FOO.JPG@*`` on WikiStart::
{{{
[wiki:*]
[wiki:WikiStart*]
[wiki:WikiStart@*]
[wiki:WikiStart@*/attachment/*]
[wiki:WikiStart@117/attachment/FOO.JPG]
}}}
- Sections are checked against the current Trac resource '''IN ORDER''' of
appearance in the configuration file. '''ORDER IS CRITICAL'''.
- Once a section matches, the current username is matched, '''IN ORDER''',
against the keys of the section. If a key is prefixed with a `@`, it is
treated as a group. If a key is prefixed with a `!`, the permission is
denied rather than granted. The username will match any of 'anonymous',
'authenticated', <username> or '*', using normal Trac permission rules.
Example configuration::
{{{
[groups]
administrators = athomas
[*/attachment:*]
* = WIKI_VIEW, TICKET_VIEW
[wiki:WikiStart@*]
@administrators = WIKI_ADMIN
anonymous = WIKI_VIEW
* = WIKI_VIEW
# Deny access to page templates
[wiki:PageTemplates/*]
* =
# Match everything else
[*]
@administrators = TRAC_ADMIN
anonymous = BROWSER_VIEW, CHANGESET_VIEW, FILE_VIEW, LOG_VIEW,
MILESTONE_VIEW, POLL_VIEW, REPORT_SQL_VIEW, REPORT_VIEW,
ROADMAP_VIEW, SEARCH_VIEW, TICKET_CREATE, TICKET_MODIFY,
TICKET_VIEW, TIMELINE_VIEW,
WIKI_CREATE, WIKI_MODIFY, WIKI_VIEW
# Give authenticated users some extra permissions
authenticated = REPO_SEARCH, XML_RPC
}}}
"""
implements(IPermissionPolicy)
authz_file = PathOption('authz_policy', 'authz_file', '',
"Location of authz policy configuration file. "
"Non-absolute paths are relative to the "
"Environment `conf` directory.")
def __init__(self):
self.authz = None
self.authz_mtime = None
self.groups_by_user = {}
# IPermissionPolicy methods
def check_permission(self, action, username, resource, perm):
if not self.authz_mtime or \
os.path.getmtime(self.authz_file) != self.authz_mtime:
self.parse_authz()
resource_key = self.normalise_resource(resource)
self.log.debug('Checking %s on %s', action, resource_key)
permissions = self.authz_permissions(resource_key, username)
if permissions is None:
return None # no match, can't decide
elif permissions == []:
return False # all actions are denied
# FIXME: expand all permissions once for all
ps = PermissionSystem(self.env)
for deny, perms in groupby(permissions,
key=lambda p: p.startswith('!')):
if deny and action in ps.expand_actions(p[1:] for p in perms):
return False # action is explicitly denied
elif action in ps.expand_actions(perms):
return True # action is explicitly granted
return None # no match for action, can't decide
# Internal methods
def parse_authz(self):
self.log.debug("Parsing authz security policy %s",
self.authz_file)
if not self.authz_file:
self.log.error("The `[authz_policy] authz_file` configuration "
"option in trac.ini is empty or not defined.")
raise ConfigurationError()
try:
self.authz_mtime = os.path.getmtime(self.authz_file)
except OSError as e:
self.log.error("Error parsing authz permission policy file: %s",
exception_to_unicode(e))
raise ConfigurationError()
self.authz = UnicodeConfigParser(ignorecase_option=False)
try:
self.authz.read(self.authz_file)
except ParsingError as e:
self.log.error("Error parsing authz permission policy file: %s",
exception_to_unicode(e))
raise ConfigurationError()
groups = {}
if self.authz.has_section('groups'):
for group, users in self.authz.items('groups'):
groups[group] = to_list(users)
self.groups_by_user = {}
def add_items(group, items):
for item in items:
if item.startswith('@'):
add_items(group, groups[item[1:]])
else:
self.groups_by_user.setdefault(item, set()).add(group)
for group, users in groups.iteritems():
add_items('@' + group, users)
all_actions = set(PermissionSystem(self.env).get_actions())
authz_basename = os.path.basename(self.authz_file)
for section in self.authz.sections():
if section == 'groups':
continue
for _, actions in self.authz.items(section):
for action in to_list(actions):
if action.startswith('!'):
action = action[1:]
if action not in all_actions:
self.log.warning("The action %s in the [%s] section "
"of %s is not a valid action.",
action, section, authz_basename)
def normalise_resource(self, resource):
def to_descriptor(resource):
id = resource.id
return '%s:%s@%s' % (resource.realm or '*',
id if id is not None else '*',
resource.version or '*')
def flatten(resource):
if not resource:
return ['*:*@*']
descriptor = to_descriptor(resource)
if not resource.realm and resource.id is None:
return [descriptor]
# XXX Due to the mixed functionality in resource we can end up with
# ticket, ticket:1, ticket:1@10. This code naively collapses all
# subsets of the parent resource into one. eg. ticket:1@10
parent = resource.parent
while parent and resource.realm == parent.realm:
parent = parent.parent
if parent:
return flatten(parent) + [descriptor]
else:
return [descriptor]
return '/'.join(flatten(resource))
def authz_permissions(self, resource_key, username):
# TODO: Handle permission negation in sections. eg. "if in this
# ticket, remove TICKET_MODIFY"
if username and username != 'anonymous':
valid_users = ['*', 'authenticated', 'anonymous', username]
else:
valid_users = ['*', 'anonymous']
for resource_section in [a for a in self.authz.sections()
if a != 'groups']:
resource_glob = resource_section
if '@' not in resource_glob:
resource_glob += '@*'
if fnmatchcase(resource_key, resource_glob):
for who, permissions in self.authz.items(resource_section):
permissions = to_list(permissions)
if who in valid_users or \
who in self.groups_by_user.get(username, []):
self.log.debug("%s matched section %s for user %s",
resource_key, resource_glob, username)
if isinstance(permissions, basestring):
return [permissions]
else:
return permissions
return None
class AuthzSourcePolicy(Component):
"""Permission policy for `source:` and `changeset:` resources using a
Subversion authz file.
`FILE_VIEW` and `BROWSER_VIEW` permissions are granted as specified in the
authz file.
`CHANGESET_VIEW` permission is granted for changesets where `FILE_VIEW` is
granted on at least one modified file, as well as for empty changesets.
"""
implements(IPermissionPolicy)
authz_file = PathOption('svn',
'authz_file',
'',
"""The path to the Subversion
[%(svnbook)s authorization (authz) file].
To enable authz permission checking, the `AuthzSourcePolicy`
permission policy must be added to `[trac] permission_policies`.
Non-absolute paths are relative to the Environment `conf`
directory.
""",
doc_args={
'svnbook':
'http://svnbook.red-bean.com/en/1.7/'
'svn.serverconfig.pathbasedauthz.html'
})
authz_module_name = Option(
'svn', 'authz_module_name', '',
"""The module prefix used in the `authz_file` for the default
repository. If left empty, the global section is used.
""")
_handled_perms = frozenset([(None, 'BROWSER_VIEW'),
(None, 'CHANGESET_VIEW'), (None, 'FILE_VIEW'),
(None, 'LOG_VIEW'), ('source', 'BROWSER_VIEW'),
('source', 'FILE_VIEW'),
('source', 'LOG_VIEW'),
('changeset', 'CHANGESET_VIEW')])
def __init__(self):
self._mtime = 0
self._authz = {}
self._users = set()
# IPermissionPolicy methods
def check_permission(self, action, username, resource, perm):
realm = resource.realm if resource else None
if (realm, action) in self._handled_perms:
authz, users = self._get_authz_info()
if authz is None:
return False
if username == 'anonymous':
usernames = '$anonymous', '*'
else:
usernames = username, '$authenticated', '*'
if resource is None:
return True if users & set(usernames) else None
rm = RepositoryManager(self.env)
try:
repos = rm.get_repository(resource.parent.id)
except TracError:
return True # Allow error to be displayed in the repo index
if repos is None:
return True
modules = [resource.parent.id or self.authz_module_name]
if modules[0]:
modules.append('')
def check_path_0(spath):
sections = [
authz.get(module, {}).get(spath) for module in modules
]
sections = [section for section in sections if section]
denied = False
for user in usernames:
for section in sections:
if user in section:
if section[user]:
return True
denied = True
# Don't check section without module name
# because the section with module name defines
# the user's permissions.
break
if denied: # All users has no readable permission.
return False
def check_path(path):
path = '/' + pathjoin(repos.scope, path)
if path != '/':
path += '/'
# Allow access to parent directories of allowed resources
for spath in set(
sum((list(authz.get(module, {}))
for module in modules), [])):
if spath.startswith(path):
result = check_path_0(spath)
if result is True:
return True
# Walk from resource up parent directories
for spath in parent_iter(path):
result = check_path_0(spath)
if result is not None:
return result
if realm == 'source':
return check_path(resource.id)
elif realm == 'changeset':
changes = list(repos.get_changeset(resource.id).get_changes())
if not changes or any(
check_path(change[0]) for change in changes):
return True
def _get_authz_info(self):
if not self.authz_file:
self.log.error("The [svn] authz_file configuration option in "
"trac.ini is empty or not defined")
raise ConfigurationError()
try:
mtime = os.path.getmtime(self.authz_file)
except OSError as e:
self.log.error(
"Error accessing svn authz permission policy "
"file: %s", exception_to_unicode(e))
raise ConfigurationError()
if mtime != self._mtime:
self._mtime = mtime
rm = RepositoryManager(self.env)
modules = set(repos.reponame
for repos in rm.get_real_repositories())
if '' in modules and self.authz_module_name:
modules.add(self.authz_module_name)
modules.add('')
self.log.info("Parsing authz file: %s", self.authz_file)
try:
self._authz = parse(self.authz_file, modules)
except ParsingError as e:
self.log.error(
"Error parsing svn authz permission policy "
"file: %s", exception_to_unicode(e))
raise ConfigurationError()
else:
self._users = {
user
for paths in self._authz.itervalues()
for path in paths.itervalues()
for user, result in path.iteritems() if result
}
return self._authz, self._users
class Environment(Component, ComponentManager):
"""Trac environment manager.
Trac stores project information in a Trac environment. It consists
of a directory structure containing among other things:
* a configuration file,
* project-specific templates and plugins,
* the wiki and ticket attachments files,
* the SQLite database file (stores tickets, wiki pages...)
in case the database backend is sqlite
"""
implements(ISystemInfoProvider)
required = True
system_info_providers = ExtensionPoint(ISystemInfoProvider)
setup_participants = ExtensionPoint(IEnvironmentSetupParticipant)
components_section = ConfigSection('components',
"""This section is used to enable or disable components
provided by plugins, as well as by Trac itself. The component
to enable/disable is specified via the name of the
option. Whether its enabled is determined by the option value;
setting the value to `enabled` or `on` will enable the
component, any other value (typically `disabled` or `off`)
will disable the component.
The option name is either the fully qualified name of the
components or the module/package prefix of the component. The
former enables/disables a specific component, while the latter
enables/disables any component in the specified
package/module.
Consider the following configuration snippet:
{{{
[components]
trac.ticket.report.ReportModule = disabled
webadmin.* = enabled
}}}
The first option tells Trac to disable the
[wiki:TracReports report module].
The second option instructs Trac to enable all components in
the `webadmin` package. Note that the trailing wildcard is
required for module/package matching.
To view the list of active components, go to the ''Plugins''
page on ''About Trac'' (requires `CONFIG_VIEW`
[wiki:TracPermissions permissions]).
See also: TracPlugins
""")
shared_plugins_dir = PathOption('inherit', 'plugins_dir', '',
"""Path to the //shared plugins directory//.
Plugins in that directory are loaded in addition to those in
the directory of the environment `plugins`, with this one
taking precedence.
(''since 0.11'')""")
base_url = Option('trac', 'base_url', '',
"""Reference URL for the Trac deployment.
This is the base URL that will be used when producing
documents that will be used outside of the web browsing
context, like for example when inserting URLs pointing to Trac
resources in notification e-mails.""")
base_url_for_redirect = BoolOption('trac', 'use_base_url_for_redirect',
False,
"""Optionally use `[trac] base_url` for redirects.
In some configurations, usually involving running Trac behind
a HTTP proxy, Trac can't automatically reconstruct the URL
that is used to access it. You may need to use this option to
force Trac to use the `base_url` setting also for
redirects. This introduces the obvious limitation that this
environment will only be usable when accessible from that URL,
as redirects are frequently used. (''since 0.10.5'')""")
secure_cookies = BoolOption('trac', 'secure_cookies', False,
"""Restrict cookies to HTTPS connections.
When true, set the `secure` flag on all cookies so that they
are only sent to the server on HTTPS connections. Use this if
your Trac instance is only accessible through HTTPS. (''since
0.11.2'')""")
project_name = Option('project', 'name', 'My Project',
"""Name of the project.""")
project_description = Option('project', 'descr', 'My example project',
"""Short description of the project.""")
project_url = Option('project', 'url', '',
"""URL of the main project web site, usually the website in
which the `base_url` resides. This is used in notification
e-mails.""")
project_admin = Option('project', 'admin', '',
"""E-Mail address of the project's administrator.""")
project_admin_trac_url = Option('project', 'admin_trac_url', '.',
"""Base URL of a Trac instance where errors in this Trac
should be reported.
This can be an absolute or relative URL, or '.' to reference
this Trac instance. An empty value will disable the reporting
buttons. (''since 0.11.3'')""")
project_footer = Option('project', 'footer',
N_('Visit the Trac open source project at<br />'
'<a href="http://trac.edgewall.org/">'
'http://trac.edgewall.org/</a>'),
"""Page footer text (right-aligned).""")
project_icon = Option('project', 'icon', 'common/trac.ico',
"""URL of the icon of the project.""")
log_type = Option('logging', 'log_type', 'none',
"""Logging facility to use.
Should be one of (`none`, `file`, `stderr`, `syslog`, `winlog`).""")
log_file = Option('logging', 'log_file', 'trac.log',
"""If `log_type` is `file`, this should be a path to the
log-file. Relative paths are resolved relative to the `log`
directory of the environment.""")
log_level = Option('logging', 'log_level', 'DEBUG',
"""Level of verbosity in log.
Should be one of (`CRITICAL`, `ERROR`, `WARN`, `INFO`, `DEBUG`).""")
log_format = Option('logging', 'log_format', None,
"""Custom logging format.
If nothing is set, the following will be used:
Trac[$(module)s] $(levelname)s: $(message)s
In addition to regular key names supported by the Python
logger library (see
http://docs.python.org/library/logging.html), one could use:
- $(path)s the path for the current environment
- $(basename)s the last path component of the current environment
- $(project)s the project name
Note the usage of `$(...)s` instead of `%(...)s` as the latter form
would be interpreted by the ConfigParser itself.
Example:
`($(thread)d) Trac[$(basename)s:$(module)s] $(levelname)s: $(message)s`
(''since 0.10.5'')""")
def __init__(self, path, create=False, options=[]):
"""Initialize the Trac environment.
:param path: the absolute path to the Trac environment
:param create: if `True`, the environment is created and
populated with default data; otherwise, the
environment is expected to already exist.
:param options: A list of `(section, name, value)` tuples that
define configuration options
"""
ComponentManager.__init__(self)
self.path = path
self.log = None
self.config = None
# System info should be provided through ISystemInfoProvider rather
# than appending to systeminfo, which may be a private in a future
# release.
self.systeminfo = []
if create:
self.create(options)
else:
self.verify()
self.setup_config()
if create:
for setup_participant in self.setup_participants:
setup_participant.environment_created()
def get_systeminfo(self):
"""Return a list of `(name, version)` tuples describing the name
and version information of external packages used by Trac and plugins.
"""
info = self.systeminfo[:]
for provider in self.system_info_providers:
info.extend(provider.get_system_info() or [])
info.sort(key=lambda (name, version): (name != 'Trac', name.lower()))
return info
def get_configinfo(self):
"""Returns a list of dictionaries containing the `name` and `options`
of each configuration section. The value of `options` is a list of
dictionaries containing the `name`, `value` and `modified` state of
each configuration option. The `modified` value is True if the value
differs from its default.
:since: version 1.1.2
"""
defaults = self.config.defaults(self.compmgr)
sections = []
for section in self.config.sections(self.compmgr):
options = []
default_options = defaults.get(section, {})
for name, value in self.config.options(section, self.compmgr):
default = default_options.get(name) or ''
options.append({
'name': name, 'value': value,
'modified': unicode(value) != unicode(default)
})
options.sort(key=lambda o: o['name'])
sections.append({'name': section, 'options': options})
sections.sort(key=lambda s: s['name'])
return sections
# ISystemInfoProvider methods
def get_system_info(self):
from trac import core, __version__ as VERSION
yield 'Trac', get_pkginfo(core).get('version', VERSION)
yield 'Python', sys.version
yield 'setuptools', setuptools.__version__
from trac.util.datefmt import pytz
if pytz is not None:
yield 'pytz', pytz.__version__
if hasattr(self, 'webfrontend_version'):
yield self.webfrontend, self.webfrontend_version
def component_activated(self, component):
"""Initialize additional member variables for components.
Every component activated through the `Environment` object
gets three member variables: `env` (the environment object),
`config` (the environment configuration) and `log` (a logger
object)."""
component.env = self
component.config = self.config
component.log = self.log
def _component_name(self, name_or_class):
name = name_or_class
if not isinstance(name_or_class, basestring):
name = name_or_class.__module__ + '.' + name_or_class.__name__
return name.lower()
@lazy
def _component_rules(self):
_rules = {}
for name, value in self.components_section.options():
if name.endswith('.*'):
name = name[:-2]
_rules[name.lower()] = value.lower() in ('enabled', 'on')
return _rules
def is_component_enabled(self, cls):
"""Implemented to only allow activation of components that are
not disabled in the configuration.
This is called by the `ComponentManager` base class when a
component is about to be activated. If this method returns
`False`, the component does not get activated. If it returns
`None`, the component only gets activated if it is located in
the `plugins` directory of the environment.
"""
component_name = self._component_name(cls)
# Disable the pre-0.11 WebAdmin plugin
# Please note that there's no recommendation to uninstall the
# plugin because doing so would obviously break the backwards
# compatibility that the new integration administration
# interface tries to provide for old WebAdmin extensions
if component_name.startswith('webadmin.'):
self.log.info("The legacy TracWebAdmin plugin has been "
"automatically disabled, and the integrated "
"administration interface will be used "
"instead.")
return False
rules = self._component_rules
cname = component_name
while cname:
enabled = rules.get(cname)
if enabled is not None:
return enabled
idx = cname.rfind('.')
if idx < 0:
break
cname = cname[:idx]
# By default, all components in the trac package except
# trac.test are enabled
return component_name.startswith('trac.') and \
not component_name.startswith('trac.test.') or None
def enable_component(self, cls):
"""Enable a component or module."""
self._component_rules[self._component_name(cls)] = True
def verify(self):
"""Verify that the provided path points to a valid Trac environment
directory."""
try:
tag = read_file(os.path.join(self.path, 'VERSION')).splitlines()[0]
if tag != _VERSION:
raise Exception("Unknown Trac environment type '%s'" % tag)
except Exception as e:
raise TracError("No Trac environment found at %s\n%s"
% (self.path, e))
def get_db_cnx(self):
"""Return a database connection from the connection pool
:deprecated: Use :meth:`db_transaction` or :meth:`db_query` instead
`db_transaction` for obtaining the `db` database connection
which can be used for performing any query
(SELECT/INSERT/UPDATE/DELETE)::
with env.db_transaction as db:
...
Note that within the block, you don't need to (and shouldn't)
call ``commit()`` yourself, the context manager will take care
of it (if it's the outermost such context manager on the
stack).
`db_query` for obtaining a `db` database connection which can
be used for performing SELECT queries only::
with env.db_query as db:
...
"""
return DatabaseManager(self).get_connection()
@lazy
def db_exc(self):
"""Return an object (typically a module) containing all the
backend-specific exception types as attributes, named
according to the Python Database API
(http://www.python.org/dev/peps/pep-0249/).
To catch a database exception, use the following pattern::
try:
with env.db_transaction as db:
...
except env.db_exc.IntegrityError as e:
...
"""
return DatabaseManager(self).get_exceptions()
def with_transaction(self, db=None):
"""Decorator for transaction functions :deprecated:"""
return with_transaction(self, db)
def get_read_db(self):
"""Return a database connection for read purposes :deprecated:
See `trac.db.api.get_read_db` for detailed documentation."""
return DatabaseManager(self).get_connection(readonly=True)
@property
def db_query(self):
"""Return a context manager
(`~trac.db.api.QueryContextManager`) which can be used to
obtain a read-only database connection.
Example::
with env.db_query as db:
cursor = db.cursor()
cursor.execute("SELECT ...")
for row in cursor.fetchall():
...
Note that a connection retrieved this way can be "called"
directly in order to execute a query::
with env.db_query as db:
for row in db("SELECT ..."):
...
:warning: after a `with env.db_query as db` block, though the
`db` variable is still defined, you shouldn't use it as it
might have been closed when exiting the context, if this
context was the outermost context (`db_query` or
`db_transaction`).
If you don't need to manipulate the connection itself, this
can even be simplified to::
for row in env.db_query("SELECT ..."):
...
"""
return QueryContextManager(self)
@property
def db_transaction(self):
"""Return a context manager
(`~trac.db.api.TransactionContextManager`) which can be used
to obtain a writable database connection.
Example::
with env.db_transaction as db:
cursor = db.cursor()
cursor.execute("UPDATE ...")
Upon successful exit of the context, the context manager will
commit the transaction. In case of nested contexts, only the
outermost context performs a commit. However, should an
exception happen, any context manager will perform a rollback.
You should *not* call `commit()` yourself within such block,
as this will force a commit even if that transaction is part
of a larger transaction.
Like for its read-only counterpart, you can directly execute a
DML query on the `db`::
with env.db_transaction as db:
db("UPDATE ...")
:warning: after a `with env.db_transaction` as db` block,
though the `db` variable is still available, you shouldn't
use it as it might have been closed when exiting the
context, if this context was the outermost context
(`db_query` or `db_transaction`).
If you don't need to manipulate the connection itself, this
can also be simplified to::
env.db_transaction("UPDATE ...")
"""
return TransactionContextManager(self)
def shutdown(self, tid=None):
"""Close the environment."""
RepositoryManager(self).shutdown(tid)
DatabaseManager(self).shutdown(tid)
if tid is None:
self.log.removeHandler(self._log_handler)
self._log_handler.flush()
self._log_handler.close()
del self._log_handler
def get_repository(self, reponame=None, authname=None):
"""Return the version control repository with the given name,
or the default repository if `None`.
The standard way of retrieving repositories is to use the
methods of `RepositoryManager`. This method is retained here
for backward compatibility.
:param reponame: the name of the repository
:param authname: the user name for authorization (not used
anymore, left here for compatibility with
0.11)
"""
return RepositoryManager(self).get_repository(reponame)
def create(self, options=[]):
"""Create the basic directory structure of the environment,
initialize the database and populate the configuration file
with default values.
If options contains ('inherit', 'file'), default values will
not be loaded; they are expected to be provided by that file
or other options.
"""
# Create the directory structure
if not os.path.exists(self.path):
os.mkdir(self.path)
os.mkdir(self.get_log_dir())
os.mkdir(self.get_htdocs_dir())
os.mkdir(os.path.join(self.path, 'plugins'))
# Create a few files
create_file(os.path.join(self.path, 'VERSION'), _VERSION + '\n')
create_file(os.path.join(self.path, 'README'),
'This directory contains a Trac environment.\n'
'Visit http://trac.edgewall.org/ for more information.\n')
# Setup the default configuration
os.mkdir(os.path.join(self.path, 'conf'))
create_file(os.path.join(self.path, 'conf', 'trac.ini.sample'))
config = Configuration(os.path.join(self.path, 'conf', 'trac.ini'))
for section, name, value in options:
config.set(section, name, value)
config.save()
self.setup_config()
if not any((section, option) == ('inherit', 'file')
for section, option, value in options):
self.config.set_defaults(self)
self.config.save()
# Create the database
DatabaseManager(self).init_db()
def get_version(self, initial=False):
"""Return the current version of the database. If the
optional argument `initial` is set to `True`, the version of
the database used at the time of creation will be returned.
In practice, for database created before 0.11, this will
return `False` which is "older" than any db version number.
:since: 0.11
"""
rows = self.db_query("""
SELECT value FROM system WHERE name='%sdatabase_version'
""" % ('initial_' if initial else ''))
return int(rows[0][0]) if rows else False
def setup_config(self):
"""Load the configuration file."""
self.config = Configuration(os.path.join(self.path, 'conf',
'trac.ini'),
{'envname': os.path.basename(self.path)})
self.setup_log()
from trac.loader import load_components
plugins_dir = self.shared_plugins_dir
load_components(self, plugins_dir and (plugins_dir,))
def get_templates_dir(self):
"""Return absolute path to the templates directory."""
return os.path.join(self.path, 'templates')
def get_htdocs_dir(self):
"""Return absolute path to the htdocs directory."""
return os.path.join(self.path, 'htdocs')
def get_log_dir(self):
"""Return absolute path to the log directory."""
return os.path.join(self.path, 'log')
def setup_log(self):
"""Initialize the logging sub-system."""
from trac.log import logger_handler_factory
logtype = self.log_type
logfile = self.log_file
if logtype == 'file' and not os.path.isabs(logfile):
logfile = os.path.join(self.get_log_dir(), logfile)
format = self.log_format
logid = 'Trac.%s' % sha1(self.path).hexdigest()
if format:
format = format.replace('$(', '%(') \
.replace('%(path)s', self.path) \
.replace('%(basename)s', os.path.basename(self.path)) \
.replace('%(project)s', self.project_name)
self.log, self._log_handler = logger_handler_factory(
logtype, logfile, self.log_level, logid, format=format)
from trac import core, __version__ as VERSION
self.log.info('-' * 32 + ' environment startup [Trac %s] ' + '-' * 32,
get_pkginfo(core).get('version', VERSION))
def get_known_users(self):
"""Generator that yields information about all known users,
i.e. users that have logged in to this Trac environment and
possibly set their name and email.
This function generates one tuple for every user, of the form
(username, name, email) ordered alpha-numerically by username.
"""
for username, name, email in self.db_query("""
SELECT DISTINCT s.sid, n.value, e.value
FROM session AS s
LEFT JOIN session_attribute AS n ON (n.sid=s.sid
and n.authenticated=1 AND n.name = 'name')
LEFT JOIN session_attribute AS e ON (e.sid=s.sid
AND e.authenticated=1 AND e.name = 'email')
WHERE s.authenticated=1 ORDER BY s.sid
"""):
yield username, name, email
def backup(self, dest=None):
"""Create a backup of the database.
:param dest: Destination file; if not specified, the backup is
stored in a file called db_name.trac_version.bak
"""
return DatabaseManager(self).backup(dest)
def needs_upgrade(self):
"""Return whether the environment needs to be upgraded."""
for participant in self.setup_participants:
args = ()
with self.db_query as db:
if arity(participant.environment_needs_upgrade) == 1:
args = (db,)
if participant.environment_needs_upgrade(*args):
self.log.warn("Component %s requires environment upgrade",
participant)
return True
return False
def upgrade(self, backup=False, backup_dest=None):
"""Upgrade database.
:param backup: whether or not to backup before upgrading
:param backup_dest: name of the backup file
:return: whether the upgrade was performed
"""
upgraders = []
for participant in self.setup_participants:
args = ()
with self.db_query as db:
if arity(participant.environment_needs_upgrade) == 1:
args = (db,)
if participant.environment_needs_upgrade(*args):
upgraders.append(participant)
if not upgraders:
return
if backup:
try:
self.backup(backup_dest)
except Exception as e:
raise BackupError(e)
for participant in upgraders:
self.log.info("%s.%s upgrading...", participant.__module__,
participant.__class__.__name__)
args = ()
with self.db_transaction as db:
if arity(participant.upgrade_environment) == 1:
args = (db,)
participant.upgrade_environment(*args)
# Database schema may have changed, so close all connections
DatabaseManager(self).shutdown()
return True
@lazy
def href(self):
"""The application root path"""
return Href(urlsplit(self.abs_href.base).path)
@lazy
def abs_href(self):
"""The application URL"""
if not self.base_url:
self.log.warn("base_url option not set in configuration, "
"generated links may be incorrect")
_abs_href = Href('')
else:
_abs_href = Href(self.base_url)
return _abs_href
class FilesCoreComponent(Component):
"""
Helper component for the file configurations etc.
"""
default_downloads_directory = Option('multiproject-files', 'default_downloads_directory',
default='downloads',
doc='Default name of the directory which is created for project. ')
sys_dav_root = PathOption('multiproject-files', 'sys_dav_root',
default='/var/www/trac/webdav',
doc='Path to the root directory of the webdav directory. '
'For example, "/path/to/multiproject/root/webdav". ')
url_dav_path = Option('multiproject-files', 'url_dav_path',
default='dav',
doc='Relative url path to the dav directory after [multiproject] url_projects_path. '
'For example, "dav". ')
downloads_dir_customizable = BoolOption('multiproject-files', 'downloads_dir_customizable',
default='True',
doc="Whether or not projects can configure their downloads directory, "
"or whether to use the MultiProject-wide configurations. "
"Setting this to False improves performance. ")
def base_url(self, context_name='files'):
env_name = self.env.project_identifier
if context_name == 'files':
return '/files'
elif context_name == 'webdav':
return '/'.join([self.config.get('multiproject',
'url_projects_path', ''), self.url_dav_path, env_name])
else:
raise InvalidOperationError("context name was wrong")
def files_download_config(self, context_name='files', req=None):
"""
:param str context_name: either 'files' or 'webdav'
:return: FilesDownloadConfig
"""
ctx = None
ctx_key = 'files.download_config.' + context_name
if req:
ctx = get_context(req)
try:
return ctx[ctx_key]
except KeyError:
pass
env_name = self.env.project_identifier
base_url = self.base_url(context_name=context_name)
download_config = FilesDownloadConfig(env_name, base_url=base_url)
if ctx:
ctx[ctx_key] = download_config
return download_config
def files_node_factory_and_config(self, req=None, context_name='files'):
"""
:param req: Request object
:param str context_name: either 'files' or 'webdav'
:return:
"""
ctx = None
ctx_key = 'files.node_factory.' + context_name
if req:
ctx = get_context(req)
try:
return ctx[ctx_key], self.files_download_config(context_name, req=req)
except KeyError:
pass
download_config = self.files_download_config(context_name=context_name, req=req)
project = Project.get(self.env)
project_id = project.id
node_factory = MappedFileNode(project_id,
download_config.base_path, download_config.base_url, download_config.downloads_dir)
if ctx:
ctx[ctx_key] = node_factory
return node_factory, download_config
def order_files_and_dirs(self, req, is_download, dirs=None, files=None):
order_by = req.args.get('files_order_by', 'filename')
order_reversed = req.args.get('files_order_in', 'asc') == 'desc'
dir_key = None
if order_by == 'filename':
key = lambda node: node.filename
elif order_by == 'count' and is_download:
key = lambda node: (node.download().count, node.filename)
dir_key = lambda node: node.filename
elif order_by == 'featured' and is_download:
key = lambda node: (node.download().is_featured() and 1 or 0, node.filename)
dir_key = lambda node: node.filename
elif order_by == 'modified':
if is_download:
key = lambda node: (node.download().created, node.filename)
dir_key = lambda node: (node.time_changed, node.filename)
else:
key = lambda node: (node.time_changed, node.filename)
elif order_by == 'size':
key = lambda node: (node.size, node.filename)
dir_key = lambda node: node.filename
else:
key = lambda node: node.filename
if files:
files.sort(key=key, reverse=order_reversed)
if dirs:
if not dir_key:
dirs.sort(key=key, reverse=order_reversed)
else:
dirs.sort(key=dir_key, reverse=order_reversed)
class DownloadsApi(Component):
# Download change listeners.
change_listeners = ExtensionPoint(IDownloadChangeListener)
# Configuration options.
title = Option('downloads',
'title',
'Downloads',
doc='Main navigation bar button title.')
path = PathOption('downloads',
'path',
'../downloads',
doc='Path where to store uploaded downloads.')
ext = ListOption('downloads', 'ext', 'zip,gz,bz2,rar',
doc = 'List of file extensions allowed to upload. Set to ''all'' ' \
'to specify that any file extensions is allowed.')
max_size = IntOption(
'downloads', 'max_size', 268697600,
'Maximum allowed file size (in bytes) for downloads. Default is 256 MB.'
)
visible_fields = ListOption(
'downloads',
'visible_fields',
'id,file,description,size,time,count,author,tags,component,version,'
'architecture,platform,type',
doc='List of downloads table fields that'
' should be visible to users on Downloads section.')
download_sort = Option(
'downloads', 'download_sort', 'time', 'Column by'
' which downloads list will be sorted. Possible values are: id, file,'
' description, size, time, count, author, tags, component, version,'
' architecture, platform, type. Default value is: time.')
download_sort_direction = Option(
'downloads', 'download_sort_direction', 'desc',
'Direction of downloads list sorting. Possible values are: asc,'
' desc. Default value is: desc.')
architecture_sort = Option(
'downloads', 'architecture_sort', 'name',
'Column by which architectures list will be sorted. Possible values are:'
' id, name, description. Default value is: name.')
architecture_sort_direction = Option(
'downloads', 'architecture_sort_direction', 'asc',
'Direction of architectures list'
' sorting. Possible values are: asc, desc. Default value is: asc.')
platform_sort = Option(
'downloads', 'platform_sort', 'name', 'Column by'
' which platforms list will be sorted. Possible values are: id, name,'
' description. Default value is: name.')
platform_sort_direction = Option(
'downloads', 'platform_sort_direction', 'asc',
'Direction of platforms list sorting. Possible values are: asc,'
' desc. Default value is: asc.')
type_sort = Option(
'downloads', 'type_sort', 'name', 'Column by which types'
' list will be sorted. Possible values are: id, name, description.'
' Default value is: name.')
type_sort_direction = Option(
'downloads', 'type_sort_direction', 'asc',
'Direction of types list sorting. Possible values are: asc, desc. Default'
' value is: asc.')
unique_filename = BoolOption(
'downloads',
'unique_filename',
False,
doc='If enabled checks if uploaded file has unique name.')
# Get list functions.
def _get_items(self,
context,
table,
columns,
where='',
values=(),
order_by='',
desc=False):
sql = 'SELECT ' + ', '.join(columns) + ' FROM ' + table + (
where and (' WHERE ' + where)
or '') + (order_by and (' ORDER BY ' + order_by +
(' ASC', ' DESC')[bool(desc)]) or '')
self.log.debug(sql % values)
context.cursor.execute(sql, values)
items = []
for row in context.cursor:
row = dict(zip(columns, row))
items.append(row)
return items
def get_versions(self, context, order_by='name', desc=False):
# Get versions from table.
versions = self._get_items(context,
'version', ('name', 'description'),
order_by=order_by,
desc=desc)
# Add IDs to versions according to selected sorting.
id = 0
for version in versions:
id = id + 1
version['id'] = id
return versions
def get_components(self, context, order_by='', desc=False):
# Get components from table.
components = self._get_items(context,
'component', ('name', 'description'),
order_by=order_by,
desc=desc)
# Add IDs to versions according to selected sorting.
id = 0
for component in components:
id = id + 1
component['id'] = id
return components
def get_downloads(self, context, order_by='id', desc=False):
# Get downloads from table.
downloads = self._get_items(
context,
'download', ('id', 'file', 'description', 'size', 'time', 'count',
'author', 'tags', 'component', 'version',
'architecture', 'platform', 'type'),
order_by=order_by,
desc=desc)
# Replace field IDs with apropriate objects.
for download in downloads:
download['architecture'] = self.get_architecture(
context, download['architecture'])
download['platform'] = self.get_platform(context,
download['platform'])
download['type'] = self.get_type(context, download['type'])
return downloads
def get_new_downloads(self,
context,
start,
stop,
order_by='time',
desc=False):
return self._get_items(context,
'download',
('id', 'file', 'description', 'size', 'time',
'count', 'author', 'tags', 'component',
'version', 'architecture', 'platform', 'type'),
'time BETWEEN %s AND'
' %s', (start, stop),
order_by=order_by,
desc=desc)
def get_architectures(self, context, order_by='id', desc=False):
return self._get_items(context,
'architecture', ('id', 'name', 'description'),
order_by=order_by,
desc=desc)
def get_platforms(self, context, order_by='id', desc=False):
return self._get_items(context,
'platform', ('id', 'name', 'description'),
order_by=order_by,
desc=desc)
def get_types(self, context, order_by='id', desc=False):
return self._get_items(context,
'download_type', ('id', 'name', 'description'),
order_by=order_by,
desc=desc)
# Get one item functions.
def _get_item(self, context, table, columns, where='', values=()):
sql = 'SELECT ' + ', '.join(columns) + ' FROM ' + table + (
where and (' WHERE ' + where) or '')
self.log.debug(sql % values)
context.cursor.execute(sql, values)
for row in context.cursor:
row = dict(zip(columns, row))
return row
return None
def get_download(self, context, id):
return self._get_item(context, 'download',
('id', 'file', 'description', 'size', 'time',
'count', 'author', 'tags', 'component',
'version', 'architecture', 'platform', 'type'),
'id = %s', (id, ))
def get_download_by_time(self, context, time):
return self._get_item(context, 'download',
('id', 'file', 'description', 'size', 'time',
'count', 'author', 'tags', 'component',
'version', 'architecture', 'platform', 'type'),
'time = %s', (time, ))
def get_download_by_file(self, context, file):
return self._get_item(context, 'download',
('id', 'file', 'description', 'size', 'time',
'count', 'author', 'tags', 'component',
'version', 'architecture', 'platform', 'type'),
'file = %s', (file, ))
def get_architecture(self, context, id):
architecture = self._get_item(context, 'architecture',
('id', 'name', 'description'), 'id = %s',
(id, ))
if not architecture:
architecture = {'id': 0, 'name': '', 'description': ''}
return architecture
def get_architecture_by_name(self, context, name):
architecture = self._get_item(context, 'architecture',
('id', 'name', 'description'),
'name = %s', (name, ))
if not architecture:
architecture = {'id': 0, 'name': '', 'description': ''}
return architecture
def get_platform(self, context, id):
platform = self._get_item(context, 'platform',
('id', 'name', 'description'), 'id = %s',
(id, ))
if not platform:
platform = {'id': 0, 'name': '', 'description': ''}
return platform
def get_platform_by_name(self, context, name):
platform = self._get_item(context, 'platform',
('id', 'name', 'description'), 'name = %s',
(name, ))
if not platform:
platform = {'id': 0, 'name': '', 'description': ''}
return platform
def get_type(self, context, id):
type = self._get_item(context, 'download_type',
('id', 'name', 'description'), 'id = %s', (id, ))
if not type:
type = {'id': 0, 'name': '', 'description': ''}
return type
def get_type_by_name(self, context, name):
type = self._get_item(context, 'download_type',
('id', 'name', 'description'), 'name = %s',
(name, ))
if not type:
type = {'id': 0, 'name': '', 'description': ''}
return type
def get_description(self, context):
sql = "SELECT value FROM system WHERE name = 'downloads_description'"
self.log.debug(sql)
context.cursor.execute(sql)
for row in context.cursor:
return row[0]
# Add item functions.
def _add_item(self, context, table, item):
fields = item.keys()
values = item.values()
sql = "INSERT INTO %s (" % (table,) + ", ".join(fields) + ") VALUES (" \
+ ", ".join(["%s" for I in xrange(len(fields))]) + ")"
self.log.debug(sql % tuple(values))
context.cursor.execute(sql, tuple(values))
def add_download(self, context, download):
self._add_item(context, 'download', download)
def add_architecture(self, context, architecture):
self._add_item(context, 'architecture', architecture)
def add_platform(self, context, platform):
self._add_item(context, 'platform', platform)
def add_type(self, context, type):
self._add_item(context, 'download_type', type)
# Edit item functions.
def _edit_item(self, context, table, id, item):
fields = item.keys()
values = item.values()
sql = "UPDATE %s SET " % (table, ) + ", ".join(
[("%s = %%s" % (field)) for field in fields]) + " WHERE id = %s"
self.log.debug(sql % tuple(values + [id]))
context.cursor.execute(sql, tuple(values + [id]))
def edit_download(self, context, id, download):
self._edit_item(context, 'download', id, download)
def edit_architecture(self, context, id, architecture):
self._edit_item(context, 'architecture', id, architecture)
def edit_platform(self, context, id, platform):
self._edit_item(context, 'platform', id, platform)
def edit_type(self, context, id, type):
self._edit_item(context, 'download_type', id, type)
def edit_description(self, context, description):
sql = "UPDATE system SET value = %s WHERE name = 'downloads_description'"
self.log.debug(sql % (description, ))
context.cursor.execute(sql, (description, ))
# Delete item functions.
def _delete_item(self, context, table, id):
sql = "DELETE FROM " + table + " WHERE id = %s"
self.log.debug(sql % (id, ))
context.cursor.execute(sql, (id, ))
def _delete_item_ref(self, context, table, column, id):
sql = "UPDATE " + table + " SET " + column + " = NULL WHERE " + column + " = %s"
self.log.debug(sql % (id, ))
context.cursor.execute(sql, (id, ))
def delete_download(self, context, id):
self._delete_item(context, 'download', id)
def delete_architecture(self, context, id):
self._delete_item(context, 'architecture', id)
self._delete_item_ref(context, 'download', 'architecture', id)
def delete_platform(self, context, id):
self._delete_item(context, 'platform', id)
self._delete_item_ref(context, 'download', 'platform', id)
def delete_type(self, context, id):
self._delete_item(context, 'download_type', id)
self._delete_item_ref(context, 'download', 'type', id)
# Misc database access functions.
def _get_attribute(self, context, table, column, where='', values=()):
sql = 'SELECT ' + column + ' FROM ' + table + (where and
(' WHERE ' + where)
or '')
self.log.debug(sql % values)
context.cursor.execute(sql, values)
for row in context.cursor:
return row[0]
return None
def get_download_id_from_file(self, context, file):
return self._get_attribute(context, 'download', 'file', 'id = %s',
(file, ))
def get_number_of_downloads(self, context, download_ids=None):
sql = 'SELECT SUM(count) FROM download' + (
download_ids and (' WHERE id in (' + ', '.join(
[to_unicode(download_id)
for download_id in download_ids]) + ')') or '')
self.log.debug(sql)
context.cursor.execute(sql)
for row in context.cursor:
return row[0]
return None
# Proces request functions.
def process_downloads(self, context):
# Clear data for next request.
self.data = {}
# Get database access.
db = self.env.get_db_cnx()
context.cursor = db.cursor()
# Get request mode
modes = self._get_modes(context)
self.log.debug('modes: %s' % modes)
# Perform mode actions
self._do_actions(context, modes)
# Fill up the template data.
self.data['authname'] = context.req.authname
self.data['time'] = format_datetime(datetime.now(utc))
self.data['realm'] = context.resource.realm
# Add CSS styles
add_stylesheet(context.req, 'common/css/wiki.css')
add_stylesheet(context.req, 'downloads/css/downloads.css')
add_stylesheet(context.req, 'downloads/css/admin.css')
# Add JavaScripts
add_script(context.req, 'common/js/trac.js')
add_script(context.req, 'common/js/wikitoolbar.js')
# Commit database changes and return template and data.
db.commit()
self.env.log.debug('data: %s' % (self.data, ))
return modes[-1] + '.html', {'downloads': self.data}
# Internal functions.
def _get_modes(self, context):
# Get request arguments.
page = context.req.args.get('page')
action = context.req.args.get('action')
self.log.debug('context: %s page: %s action: %s' %
(context, page, action))
# Determine mode.
if context.resource.realm == 'downloads-admin':
if page == 'downloads':
if action == 'post-add':
return ['downloads-post-add', 'admin-downloads-list']
elif action == 'post-edit':
return ['downloads-post-edit', 'admin-downloads-list']
elif action == 'delete':
return ['downloads-delete', 'admin-downloads-list']
else:
return ['admin-downloads-list']
elif page == 'architectures':
if action == 'post-add':
return [
'architectures-post-add', 'admin-architectures-list'
]
elif action == 'post-edit':
return [
'architectures-post-edit', 'admin-architectures-list'
]
elif action == 'delete':
return ['architectures-delete', 'admin-architectures-list']
else:
return ['admin-architectures-list']
elif page == 'platforms':
if action == 'post-add':
return ['platforms-post-add', 'admin-platforms-list']
elif action == 'post-edit':
return ['platforms-post-edit', 'admin-platforms-list']
elif action == 'delete':
return ['platforms-delete', 'admin-platforms-list']
else:
return ['admin-platforms-list']
elif page == 'types':
if action == 'post-add':
return ['types-post-add', 'admin-types-list']
elif action == 'post-edit':
return ['types-post-edit', 'admin-types-list']
elif action == 'delete':
return ['types-delete', 'admin-types-list']
else:
return ['admin-types-list']
elif context.resource.realm == 'downloads-core':
if action == 'get-file':
return ['get-file']
elif context.resource.realm == 'downloads-downloads':
if action == 'post-add':
return ['downloads-post-add', 'downloads-list']
elif action == 'edit':
return ['description-edit', 'downloads-list']
elif action == 'post-edit':
return ['description-post-edit', 'downloads-list']
else:
return ['downloads-list']
else:
pass
def _do_actions(self, context, actions):
for action in actions:
if action == 'get-file':
context.req.perm.require('DOWNLOADS_VIEW')
# Get request arguments.
download_id = context.req.args.get('id') or 0
download_file = context.req.args.get('file')
# Get download.
if download_id:
download = self.get_download(context, download_id)
else:
download = self.get_download_by_file(
context, download_file)
# Check if requested download exists.
if not download:
raise TracError('File not found.')
# Check resource based permission.
context.req.perm.require('DOWNLOADS_VIEW',
Resource('downloads', download['id']))
# Get download file path.
path = os.path.normpath(
os.path.join(self.path, to_unicode(download['id']),
download['file']))
self.log.debug('path: %s' % (path, ))
# Increase downloads count.
new_download = {'count': download['count'] + 1}
# Edit download.
self.edit_download(context, download['id'], new_download)
# Notify change listeners.
for listener in self.change_listeners:
listener.download_changed(context, new_download, download)
# Commit DB before file send.
db = self.env.get_db_cnx()
db.commit()
# Guess mime type.
file = open(path.encode('utf-8'), "r")
file_data = file.read(1000)
file.close()
mimeview = Mimeview(self.env)
mime_type = mimeview.get_mimetype(path, file_data)
if not mime_type:
mime_type = 'application/octet-stream'
if 'charset=' not in mime_type:
charset = mimeview.get_charset(file_data, mime_type)
mime_type = mime_type + '; charset=' + charset
# Return uploaded file to request.
context.req.send_header(
'Content-Disposition', 'attachment;filename="%s"' %
(os.path.normpath(download['file'])))
context.req.send_header('Content-Description',
download['description'])
context.req.send_file(path.encode('utf-8'), mime_type)
elif action == 'downloads-list':
context.req.perm.require('DOWNLOADS_VIEW')
self.log.debug('visible_fields: %s' % (self.visible_fields, ))
# Get form values.
order = context.req.args.get('order') or self.download_sort
if context.req.args.has_key('desc'):
desc = context.req.args.get('desc') == '1'
else:
desc = self.download_sort_direction == 'desc'
self.data['order'] = order
self.data['desc'] = desc
self.data['has_tags'] = self.env.is_component_enabled(
'tractags.api.TagEngine')
self.data['visible_fields'] = self.visible_fields
self.data['title'] = self.title
self.data['description'] = self.get_description(context)
self.data['downloads'] = self.get_downloads(
context, order, desc)
self.data['visible_fields'] = [
visible_field for visible_field in self.visible_fields
]
# Component, versions, etc. are needed only for new download
# add form.
if context.req.perm.has_permission('DOWNLOADS_ADD'):
self.data['components'] = self.get_components(context)
self.data['versions'] = self.get_versions(context)
self.data['architectures'] = self.get_architectures(
context)
self.data['platforms'] = self.get_platforms(context)
self.data['types'] = self.get_types(context)
elif action == 'admin-downloads-list':
context.req.perm.require('DOWNLOADS_ADMIN')
# Get form values
order = context.req.args.get('order') or self.download_sort
if context.req.args.has_key('desc'):
desc = context.req.args.get('desc') == '1'
else:
desc = self.download_sort_direction == 'desc'
download_id = int(context.req.args.get('download') or 0)
self.data['order'] = order
self.data['desc'] = desc
self.data['has_tags'] = self.env.is_component_enabled(
'tractags.api.TagEngine')
self.data['download'] = self.get_download(context, download_id)
self.data['downloads'] = self.get_downloads(
context, order, desc)
self.data['components'] = self.get_components(context)
self.data['versions'] = self.get_versions(context)
self.data['architectures'] = self.get_architectures(context)
self.data['platforms'] = self.get_platforms(context)
self.data['types'] = self.get_types(context)
elif action == 'description-edit':
context.req.perm.require('DOWNLOADS_ADMIN')
elif action == 'description-post-edit':
context.req.perm.require('DOWNLOADS_ADMIN')
# Get form values.
description = context.req.args.get('description')
# Set new description.
self.edit_description(context, description)
elif action == 'downloads-post-add':
context.req.perm.require('DOWNLOADS_ADD')
# Get form values.
file, filename, file_size = self._get_file_from_req(context)
download = {
'file': filename,
'description': context.req.args.get('description'),
'size': file_size,
'time': to_timestamp(datetime.now(utc)),
'count': 0,
'author': context.req.authname,
'tags': context.req.args.get('tags'),
'component': context.req.args.get('component'),
'version': context.req.args.get('version'),
'architecture': context.req.args.get('architecture'),
'platform': context.req.args.get('platform'),
'type': context.req.args.get('type')
}
# Upload file to DB and file storage.
self._add_download(context, download, file)
# Close input file.
file.close()
elif action == 'downloads-post-edit':
context.req.perm.require('DOWNLOADS_ADMIN')
# Get form values.
download_id = context.req.args.get('id')
old_download = self.get_download(context, download_id)
download = {
'description': context.req.args.get('description'),
'tags': context.req.args.get('tags'),
'component': context.req.args.get('component'),
'version': context.req.args.get('version'),
'architecture': context.req.args.get('architecture'),
'platform': context.req.args.get('platform'),
'type': context.req.args.get('type')
}
# Edit Download.
self.edit_download(context, download_id, download)
# Notify change listeners.
for listener in self.change_listeners:
listener.download_changed(context, download, old_download)
elif action == 'downloads-delete':
context.req.perm.require('DOWNLOADS_ADMIN')
# Get selected downloads.
selection = context.req.args.get('selection')
if isinstance(selection, (str, unicode)):
selection = [selection]
# Delete download.
if selection:
for download_id in selection:
download = self.get_download(context, download_id)
self.log.debug('download: %s' % (download, ))
self._delete_download(context, download)
elif action == 'admin-architectures-list':
context.req.perm.require('DOWNLOADS_ADMIN')
# Get form values
order = context.req.args.get('order') or self.architecture_sort
if context.req.args.has_key('desc'):
desc = context.req.args.get('desc') == '1'
else:
desc = self.architecture_sort_direction == 'desc'
architecture_id = int(
context.req.args.get('architecture') or 0)
# Display architectures.
self.data['order'] = order
self.data['desc'] = desc
self.data['architecture'] = self.get_architecture(
context, architecture_id)
self.data['architectures'] = self.get_architectures(
context, order, desc)
elif action == 'architectures-post-add':
context.req.perm.require('DOWNLOADS_ADMIN')
# Get form values.
architecture = {
'name': context.req.args.get('name'),
'description': context.req.args.get('description')
}
# Add architecture.
self.add_architecture(context, architecture)
elif action == 'architectures-post-edit':
context.req.perm.require('DOWNLOADS_ADMIN')
# Get form values.
architecture_id = context.req.args.get('id')
architecture = {
'name': context.req.args.get('name'),
'description': context.req.args.get('description')
}
# Add architecture.
self.edit_architecture(context, architecture_id, architecture)
elif action == 'architectures-delete':
context.req.perm.require('DOWNLOADS_ADMIN')
# Get selected architectures.
selection = context.req.args.get('selection')
if isinstance(selection, (str, unicode)):
selection = [selection]
# Delete architectures.
if selection:
for architecture_id in selection:
self.delete_architecture(context, architecture_id)
elif action == 'admin-platforms-list':
context.req.perm.require('DOWNLOADS_ADMIN')
# Get form values.
order = context.req.args.get('order') or self.platform_sort
if context.req.args.has_key('desc'):
desc = context.req.args.get('desc') == '1'
else:
desc = self.platform_sort_direction == 'desc'
platform_id = int(context.req.args.get('platform') or 0)
# Display platforms.
self.data['order'] = order
self.data['desc'] = desc
self.data['platform'] = self.get_platform(context, platform_id)
self.data['platforms'] = self.get_platforms(
context, order, desc)
elif action == 'platforms-post-add':
context.req.perm.require('DOWNLOADS_ADMIN')
# Get form values.
platform = {
'name': context.req.args.get('name'),
'description': context.req.args.get('description')
}
# Add platform.
self.add_platform(context, platform)
elif action == 'platforms-post-edit':
context.req.perm.require('DOWNLOADS_ADMIN')
# Get form values.
platform_id = context.req.args.get('id')
platform = {
'name': context.req.args.get('name'),
'description': context.req.args.get('description')
}
# Add platform.
self.edit_platform(context, platform_id, platform)
elif action == 'platforms-delete':
context.req.perm.require('DOWNLOADS_ADMIN')
# Get selected platforms.
selection = context.req.args.get('selection')
if isinstance(selection, (str, unicode)):
selection = [selection]
# Delete platforms.
if selection:
for platform_id in selection:
self.delete_platform(context, platform_id)
elif action == 'admin-types-list':
context.req.perm.require('DOWNLOADS_ADMIN')
# Get form values
order = context.req.args.get('order') or self.type_sort
if context.req.args.has_key('desc'):
desc = context.req.args.get('desc') == '1'
else:
desc = self.type_sort_direction == 'desc'
platform_id = int(context.req.args.get('type') or 0)
# Display platforms.
self.data['order'] = order
self.data['desc'] = desc
self.data['type'] = self.get_type(context, platform_id)
self.data['types'] = self.get_types(context, order, desc)
elif action == 'types-post-add':
context.req.perm.require('DOWNLOADS_ADMIN')
# Get form values.
type = {
'name': context.req.args.get('name'),
'description': context.req.args.get('description')
}
# Add type.
self.add_type(context, type)
elif action == 'types-post-edit':
context.req.perm.require('DOWNLOADS_ADMIN')
# Get form values.
type_id = context.req.args.get('id')
type = {
'name': context.req.args.get('name'),
'description': context.req.args.get('description')
}
# Add platform.
self.edit_type(context, type_id, type)
elif action == 'types-delete':
context.req.perm.require('DOWNLOADS_ADMIN')
# Get selected types.
selection = context.req.args.get('selection')
if isinstance(selection, (str, unicode)):
selection = [selection]
# Delete types.
if selection:
for type_id in selection:
self.delete_type(context, type_id)
""" Full implementation of download addition. It creates DB entry for
download <download> and stores download file <file> to file system. """
def _add_download(self, context, download, file):
# Check for file name uniqueness.
if self.unique_filename:
if self.get_download_by_file(context, download['file']):
raise TracError('File with same name is already uploaded and'
' unique file names are enabled.')
# Check correct file type.
name, ext = os.path.splitext(download['file'])
self.log.debug('file_ext: %s ext: %s' % (ext, self.ext))
if not (ext[1:].lower() in self.ext) and not ('all' in self.ext):
raise TracError('Unsupported file type.')
# Check for maximum file size.
if self.max_size >= 0 and download['size'] > self.max_size:
raise TracError('Maximum file size: %s bytes' % (self.max_size),
'Upload failed')
# Add new download to DB.
self.add_download(context, download)
# Get inserted download by time to get its ID.
download = self.get_download_by_time(context, download['time'])
# Prepare file paths.
path = os.path.normpath(
os.path.join(self.path, to_unicode(download['id'])))
filepath = os.path.normpath(os.path.join(path, download['file']))
self.log.debug('path: %s' % ((path, )))
self.log.debug('filepath: %s' % ((filepath, )))
# Store uploaded image.
try:
os.mkdir(path.encode('utf-8'))
out_file = open(filepath.encode('utf-8'), "wb+")
file.seek(0)
shutil.copyfileobj(file, out_file)
out_file.close()
except Exception, error:
self.delete_download(context, download['id'])
self.log.debug(error)
try:
os.remove(filepath.encode('utf-8'))
except:
pass
try:
os.rmdir(path.encode('utf-8'))
except:
pass
raise TracError('Error storing file %s! Is directory specified in' \
' path config option in [downloads] section of trac.ini' \
' existing?' % (download['file'],))
# Notify change listeners.
for listener in self.change_listeners:
listener.download_created(context, download)
class CrashDumpSubmit(Component):
"""Upload/Submit new crash dumps"""
implements(IRequestHandler, IRequestFilter, INavigationContributor, ITemplateProvider)
dumpdata_dir = PathOption('crashdump', 'dumpdata_dir', default='../dumpdata',
doc='Path to the crash dump data directory relative to the environment conf directory.')
default_priority = Option('crashdump', 'default_priority', default='major',
doc='Default priority for submitted crash reports.')
default_milestone = Option('crashdump', 'default_milestone', '< default >',
"""Default milestone for submitted crash reports.""")
default_version = Option('crashdump', 'default_version', '< default >',
"""Default version for submitted crash reports.""")
default_component = Option('crashdump', 'default_component', '< default >',
"""Default component for submitted crash reports.""")
default_severity = Option('crashdump', 'default_severity', '',
"""Default severity for submitted crash reports.""")
default_summary = Option('crashdump', 'default_summary', '',
"""Default summary (title) for submitted crash reports.""")
default_description = Option('crashdump', 'default_description', '',
"""Default description for submitted crash reports.""")
default_keywords = Option('crashdump', 'default_keywords', '',
"""Default keywords for submitted crash reports.""")
default_reporter = Option('crashdump', 'default_reporter', '< default >',
"""Default reporter for submitted crash reports.""")
default_owner = Option('crashdump', 'default_owner', '< default >',
"""Default owner for submitted crash reports.""")
default_ticket_type = Option('crashdump', 'ticket_type', 'defect',
"""Default ticket type for linked tickets.""")
ignored_modules = Option('crashdump', 'ignore_modules', 'libc, kernel32, ntdll, user32, gdi32',
"""List of modules to ignore for component matching.""")
replace_usernames = Option('crashdump', 'replace_usernames', '',
"""List of username replacements applied when a new crash is uploaded (format username=myrealname; multiple values separated by comma).""")
max_upload_size = IntOption('crashdump', 'max_upload_size', default=16 * 1024 * 1024,
doc="""Maximum allowed upload size. If set to zero the upload limit is disabled and all uploads will be accepted.""")
upload_disabled = BoolOption('crashdump', 'upload_disabled', 'false',
doc="""Disable upload. No further crashdumps can be submitted.""")
disable_manual_upload = BoolOption('crashdump', 'manual_upload_disabled', 'false',
doc="""Disable manual upload function. Crashes can only be uploaded automatically via the crash handler.""")
# INavigationContributor methods
def get_active_navigation_item(self, req):
self.log.debug('get_active_navigation_item %s' % req.path_info)
if not self.disable_manual_upload:
val = re.search('/crash_upload$', req.path_info)
if val and val.start() == 0:
return 'upload_crash'
def get_navigation_items(self, req):
if not self.disable_manual_upload:
yield ('mainnav', 'upload_crash',
tag.a('Upload Crash', href=req.href('crash_upload')))
# IRequestHandler methods
def match_request(self, req):
if req.method == 'POST' and (req.path_info == '/crashdump/submit' or req.path_info == '/submit'):
self.log.debug('match_request: %s %s', req.method, req.path_info)
return True
elif req.method == 'GET' and (req.path_info == '/crashdump/submit/crashlist' or req.path_info == '/submit/crashlist'):
self.log.debug('match_request: %s %s', req.method, req.path_info)
return True
elif req.method == 'GET' and (req.path_info == '/crashdump/submit/capabilities' or req.path_info == '/submit/capabilities'):
self.log.debug('match_request: %s %s', req.method, req.path_info)
return True
elif req.method == 'GET' and (req.path_info == '/crashdump/list' or req.path_info == '/crashlist'):
self.log.debug('match_request: %s %s', req.method, req.path_info)
return True
elif req.method == 'GET' and (req.path_info == '/crashdump/capabilities' or req.path_info == '/capabilities'):
self.log.debug('match_request: %s %s', req.method, req.path_info)
return True
elif (req.method == 'GET' or req.method == 'POST') and (req.path_info == '/crashdump/crash_upload' or req.path_info == '/crash_upload'):
return True
else:
self.log.debug('match_request: %s %s', req.method, req.path_info)
return False
def _error_response(self, req, status, body=None, content_type='text/plain', headers=None):
self.log.debug('_error_response: %s %s -> %i: %s', req.method, req.path_info, status, body)
if isinstance(body, unicode):
body = body.encode('utf-8')
req.send_response(status)
req._outheaders = []
req.send_header('Cache-Control', 'must-revalidate')
req.send_header('Expires', 'Fri, 01 Jan 1999 00:00:00 GMT')
req.send_header('Content-Type', content_type + ';charset=utf-8')
req.send_header('Content-Length', len(body))
if headers:
for k,v in headers.items():
req.send_header(k, v)
req._send_cookie_headers()
if req.method != 'HEAD':
req.write(body)
raise RequestDone
def _success_response(self, req, body=None, content_type='text/plain', status=200, headers=None):
if isinstance(body, unicode):
body = body.encode('utf-8')
req.send_response(status)
req.send_header('Cache-Control', 'must-revalidate')
req.send_header('Expires', 'Fri, 01 Jan 1999 00:00:00 GMT')
req.send_header('Content-Type', content_type + ';charset=utf-8')
req.send_header('Content-Length', len(body))
if headers:
for k,v in headers.items():
req.send_header(k, v)
req.end_headers()
if req.method != 'HEAD':
req.write(body)
raise RequestDone
def _manual_upload_result(self, req, error=None):
data = {}
action = 'upload'
params = None
submit_href = req.href + '/submit'
data.update({'action': action,
'params': params,
'submit_href': submit_href,
'upload_error': error,
})
if crashdump_use_jinja2:
metadata = {'content_type': 'text/html'}
else:
add_script(req, 'common/js/folding.js')
metadata = None
add_script(req, 'crashdump/crashdump.js')
add_stylesheet(req, 'crashdump/crashdump.css')
return 'upload.html', data, metadata
def _find_first_component_from_list(self, possible_components):
ret = None
for compname in possible_components:
try:
component = TicketComponent(self.env, compname)
ret = component.name
break
except ResourceNotFound:
# No such component exists
pass
return ret
def _find_first_milestone_from_list(self, possible_milestones):
#print('_find_first_milestone_from_list %s' % str(possible_milestones))
ret = None
for ms_name in possible_milestones:
try:
milestone = Milestone(self.env, ms_name)
ret = milestone.name
break
except ResourceNotFound:
# No such component exists
pass
return ret
def _find_first_version_from_list(self, possible_versions):
#print('_find_first_version_from_list %s' % str(possible_versions))
ret = None
for v_name in possible_versions:
try:
ver = Version(self.env, v_name)
ret = ver.name
break
except ResourceNotFound:
# No such component exists
pass
return ret
def _find_component_from_involved_modules(self, module_list, buildpostfix):
possible_components = []
for m in module_list:
module_base = os.path.basename(m) if '/' in m else m
module_name, module_ext = os.path.splitext(module_base)
if buildpostfix and module_name.endswith(buildpostfix):
module_name = module_name[:-len(buildpostfix)]
if '-' in module_name:
(prefix, name) = module_name.split('-', 1)
name_is_version = True
for c in name:
if (c >= '0' and c <= '9') or c == '.':
pass
else:
name_is_version = False
if name_is_version:
# name is a version number so check the prefix instead of the name
# and to not check the full module name since it would check for
# a matching version number as well.
if prefix not in self.ignored_modules:
possible_components.append(prefix)
else:
# add the entire module name
if module_name not in self.ignored_modules:
possible_components.append(module_name)
# ... and the shorten name (without prefix) to the list
if name not in self.ignored_modules:
possible_components.append(name)
else:
if module_name not in self.ignored_modules:
possible_components.append(module_name)
return self._find_first_component_from_list(possible_components)
def _find_component_for_application(self, applicationname):
if applicationname is None:
return None
possible_components = [applicationname]
if '-' in applicationname:
(prefix, name) = applicationname.split('-', 1)
possible_components.append(name)
return self._find_first_component_from_list(possible_components)
def _find_milestone(self, productversion, producttargetversion):
if producttargetversion is None:
return None
possible_versions = []
v_elems = producttargetversion.split('.')
while len(v_elems) < 4:
v_elems.append('0')
for i in range(4, 0, -1):
possible_versions.append('v' + '.'.join(v_elems[0:i]))
possible_versions.append('.'.join(v_elems[0:i]))
return self._find_first_milestone_from_list(possible_versions)
def _find_version(self, productversion, producttargetversion):
if productversion is None:
return None
possible_versions = []
v_elems = productversion.split('.')
while len(v_elems) < 4:
v_elems.append('0')
for i in range(4, 2, -1):
possible_versions.append('v' + '.'.join(v_elems[0:i]))
possible_versions.append('.'.join(v_elems[0:i]))
if v_elems[i - 1] != '0':
v_elems[i - 1] = '0'
possible_versions.append('v' + '.'.join(v_elems[0:i]))
possible_versions.append('.'.join(v_elems[0:i]))
return self._find_first_version_from_list(possible_versions)
def _apply_username_replacements(self, username):
if username is None:
return None
self.log.debug('CrashDumpSubmit _apply_username_replacements in=\'%s\'' % username)
ret = username
ret_lower = username.lower()
for pattern in self.replace_usernames.split(','):
pattern = pattern.strip()
self.log.debug('CrashDumpSubmit _apply_username_replacements pattern=\'%s\'' % pattern)
if '=' in pattern:
(find, replace) = pattern.split('=', 1)
find = find.strip().lower()
replace = replace.strip()
self.log.debug('CrashDumpSubmit _apply_username_replacements find=\'%s\' -> replace=\'%s\'' % (find, replace))
if ret_lower == find:
ret = replace
ret_lower = replace.lower()
self.log.debug('CrashDumpSubmit _apply_username_replacements out=\'%s\'' % ret)
return ret
def pre_process_request(self, req, handler):
if req.path_info != '/crashdump/submit' and req.path_info != '/submit' and \
req.path_info != '/crashdump/crash_upload' and req.path_info != '/crash_upload':
return handler
self.log.debug('CrashDumpSubmit pre_process_request: %s %s %s', req.method, req.path_info, handler)
if req.method == "POST":
user_agent = req.get_header('User-Agent')
if user_agent is not None and '/' in user_agent:
user_agent, agent_ver = user_agent.split('/', 1)
if user_agent == 'terra3d-crashuploader':
# copy the requested form token from into the args to pass the CSRF test
req.args['__FORM_TOKEN' ] = req.form_token
manual_upload = req.args.as_int('manual_upload', 0)
# for testing
if manual_upload:
# copy the requested form token from into the args to pass the CSRF test
req.args['__FORM_TOKEN' ] = req.form_token
return handler
def post_process_request(self, req, template, data, content_type, method=None):
return template, data, content_type, method
def process_request(self, req):
self.log.debug('CrashDumpSubmit process_request: %s %s', req.method, req.path_info)
if req.path_info == '/crashdump/submit' or req.path_info == '/submit':
self.log.debug('CrashDumpSubmit process_request_submit: %s %s', req.method, req.path_info)
return self.process_request_submit(req)
elif req.path_info == '/crashdump/crash_upload' or req.path_info == '/crash_upload':
return self.process_request_crash_upload(req)
elif req.path_info == '/crashdump/list' or req.path_info == '/crashlist' or req.path_info == '/crashdump/submit/crashlist' or req.path_info == '/submit/crashlist':
return self.process_request_crashlist(req)
elif req.path_info == '/crashdump/capabilities' or req.path_info == '/capabilities' or req.path_info == '/crashdump/submit/capabilities' or req.path_info == '/submit/capabilities':
return self.process_request_capabilities(req)
else:
return self._error_response(req, status=HTTPMethodNotAllowed.code, body='Invalid request path %s.' % req.path_info)
def process_request_capabilities(self, req):
if req.method != "GET":
return self._error_response(req, status=HTTPMethodNotAllowed.code, body='Method %s not allowed' % req.method)
user_agent = req.get_header('User-Agent')
if user_agent is None:
return self._error_response(req, status=HTTPForbidden.code, body='No user-agent specified.')
headers = {}
headers['Max-Upload-Size'] = self.max_upload_size
headers['Upload-Disabled'] = '1' if self.upload_disabled else '0'
# This is a plain Python source file, not an egg
dist = get_distribution('TracCrashDump')
if dist:
headers['Crashdump-Plugin-Version'] = dist.version
headers['Upload-Disabled'] = '1' if self.upload_disabled else '0'
if self.upload_disabled:
body = 'Disabled'
else:
body = 'OK'
return self._success_response(req, body=body.encode('utf-8'), headers=headers)
def process_request_crash_upload(self, req):
return self._manual_upload_result(req, error=None)
def escape_ticket_values(self, values):
ret = {}
for k,v in values.items():
if isinstance(v, str) or isinstance(v, basestring):
ret[k] = v.replace('#', '!#')
else:
ret[k] = v
return ret
def process_request_submit(self, req):
if req.method != "POST":
return self._error_response(req, status=HTTPMethodNotAllowed.code, body='Method %s not allowed' % req.method)
manual_upload = req.args.as_int('manual_upload', 0)
if manual_upload == 0:
user_agent_full = req.get_header('User-Agent')
if user_agent_full is None:
return self._error_response(req, status=HTTPForbidden.code, body='No user-agent specified.')
if '/' in user_agent_full:
user_agent, agent_ver = user_agent_full.split('/', 1)
else:
user_agent = user_agent_full
if user_agent != 'terra3d-crashuploader':
return self._error_response(req, status=HTTPForbidden.code, body='User-agent %s not allowed' % user_agent_full)
headers = {}
headers['Max-Upload-Size'] = self.max_upload_size
headers['Upload-Disabled'] = '1' if self.upload_disabled else '0'
if self.upload_disabled:
return self._error_response(req, status=HTTPInternalServerError.code, body='Crashdump upload has been disabled by the administrator.', headers=headers)
id_str = req.args.get('id')
if not manual_upload:
if not id_str or not CrashDump.uuid_is_valid(id_str):
return self._error_response(req, status=HTTPInternalServerError.code, body='Invalid crash identifier %s specified.' % id_str)
total_upload_size = self._get_total_upload_size(req)
if self.max_upload_size > 0 and total_upload_size > self.max_upload_size:
self.log.debug('total_upload_size %i > max_upload_size %i' % (total_upload_size, self.max_upload_size) )
return self._error_response(req, status=HTTPInternalServerError.code, body='Upload size %i bytes exceed the upload limit of %i bytes' % (total_upload_size, self.max_upload_size), headers=headers)
else:
self.log.debug('total_upload_size %i <= max_upload_size %i' % (total_upload_size, self.max_upload_size) )
if manual_upload:
self.log.debug('manual_upload')
files = req.args.getlist('files')
if len(files) == 0:
return self._error_response(req, status=HTTPInternalServerError.code, body='No files uploaded.')
import re
id_str = None
minidump = None
minidumpreportxml = None
for file in files:
if isinstance(file, cgi.FieldStorage):
filename = os.path.basename(file.filename)
self.log.debug('got file %s' % filename)
match = re.match(r'^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})\.([0-9a-zA-Z\.]+)$', filename)
if match:
new_id_str = match.groups()[0]
ext = match.groups()[1]
self.log.debug('got file match %s' % new_id_str)
if id_str is None:
id_str = new_id_str
elif id_str == new_id_str:
pass
else:
return self._error_response(req, status=HTTPInternalServerError.code, body='At the moment uploading multiples crashes is not supported.')
if ext == 'dmp':
minidump = file
elif ext == 'dmp.xml':
minidumpreportxml = file
self.log.debug('got id %s, ext %s' % (id_str, ext))
else:
self.log.debug('skip file field %s-%s' % (type(file), file) )
if not id_str:
return self._manual_upload_result(req, error='Cannot determine crash identifier from file upload. The files uploaded must have a UUID in its name and the extentsion must either be .dmp or .dmp.xml.')
elif minidump is None and minidumpreportxml is None:
return self._manual_upload_result(req, error='Uploaded files do not contain a valid crash dump information.')
self.log.debug('got crashid %s' % id_str)
if minidump is not None:
req.args['minidump'] = minidump
if minidumpreportxml is not None:
req.args['minidumpreportxml'] = minidumpreportxml
uuid = UUID(id_str)
crashid = None
crashobj = CrashDump.find_by_uuid(self.env, uuid)
if not crashobj:
crashobj = CrashDump(uuid=uuid, env=self.env, must_exist=False)
else:
crashid = crashobj.id
force_str = req.args.get('force') or 'false'
force = True if force_str.lower() == 'true' else False
if crashid is not None and not force and not manual_upload:
headers = {}
headers['Crash-URL'] = req.abs_href('crash', str(uuid))
headers['CrashId'] = str(crashid)
self.log.debug('crash %s already uploaded %s' % (uuid, headers['Crash-URL']) )
return self._error_response(req, status=HTTPInternalServerError.code, body='Crash identifier %s already uploaded.' % id_str, headers=headers)
ticket_str = req.args.get('ticket') or 'no'
linked_tickets = set()
ticketobjs = []
new_ticket = None
if ticket_str == 'no':
pass
elif '#' in ticket_str:
ticket_ids = []
for t in ticket_str.split(','):
if t[0] == '#':
ticket_ids.append(int(t[1:]))
ticketobjs = []
for tkt_id in ticket_ids:
try:
ticketobjs.append(Ticket(env=self.env, tkt_id=tkt_id))
except ResourceNotFound:
return self._error_response(req, status=HTTPNotFound.code, body='Ticket %i not found. Cannot link crash %s to the requested ticket.' % (tkt_id, str(uuid)))
elif ticket_str == 'auto':
if crashid is None:
new_ticket = Ticket(env=self.env)
ticketobjs = [ new_ticket ]
else:
for tkt_id in crashobj.linked_tickets:
try:
ticketobjs.append( Ticket(env=self.env, tkt_id=tkt_id) )
break
except ResourceNotFound:
pass
if len(ticketobjs) == 0:
new_ticket = Ticket(env=self.env)
ticketobjs = [ new_ticket ]
elif ticket_str == 'new':
new_ticket = Ticket(env=self.env)
ticketobjs = [ new_ticket ]
else:
return self._error_response(req, status=HTTPInternalServerError.code, body='Unrecognized ticket string %s for crash %s.' % (ticket_str, str(uuid)))
#print('ticket_str=%s' % ticket_str)
#print('ticketobjs=%s' % str(ticketobjs))
# we require at least one crash dump file (either minidump or coredump)
# and any number of report files
failure_message = None
result = False
ok, new_minidumpfile, errmsg = self._store_dump_file(uuid, req, 'minidump', force)
if ok:
result = True
elif failure_message is None:
failure_message = errmsg
ok, new_minidumpreporttextfile, errmsg = self._store_dump_file(uuid, req, 'minidumpreport', force)
ok, new_minidumpreportxmlfile, errmsg = self._store_dump_file(uuid, req, 'minidumpreportxml', force)
# accept XML crash upload only for manual uploads
if manual_upload and ok:
result = True
elif failure_message is None:
failure_message = errmsg
ok, new_minidumpreporthtmlfile, errmsg = self._store_dump_file(uuid, req, 'minidumpreporthtml', force)
ok, new_coredumpfile, errmsg = self._store_dump_file(uuid, req, 'coredump', force)
if ok:
result = True
elif failure_message is None:
failure_message = errmsg
ok, new_coredumpreporttextfile, errmsg = self._store_dump_file(uuid, req, 'coredumpreport', force)
ok, new_coredumpreportxmlfile, errmsg = self._store_dump_file(uuid, req, 'coredumpreportxml', force)
ok, new_coredumpreporthtmlfile, errmsg = self._store_dump_file(uuid, req, 'coredumpreporthtml', force)
self.log.debug('new_minidumpfile \'%s\'' % new_minidumpfile)
self.log.debug('new_minidumpreportxmlfile \'%s\'' % new_minidumpreportxmlfile)
self.log.debug('before crashobj[minidumpfile] \'%s\'' % crashobj['minidumpfile'])
self.log.debug('before crashobj[minidumpreportxmlfile] \'%s\'' % crashobj['minidumpreportxmlfile'])
if manual_upload:
if not crashobj['minidumpfile'] or force:
crashobj['minidumpfile'] = new_minidumpfile
if not crashobj['minidumpreporttextfile'] or force:
crashobj['minidumpreporttextfile'] = new_minidumpreporttextfile
if not crashobj['minidumpreportxmlfile'] or force:
crashobj['minidumpreportxmlfile'] = new_minidumpreportxmlfile
if not crashobj['minidumpreporthtmlfile'] or force:
crashobj['minidumpreporthtmlfile'] = new_minidumpreporthtmlfile
if not crashobj['coredumpfile'] or force:
crashobj['coredumpfile'] = new_coredumpfile
if not crashobj['coredumpreporttextfile'] or force:
crashobj['coredumpreporttextfile'] = new_coredumpreporttextfile
if not crashobj['coredumpreportxmlfile'] or force:
crashobj['coredumpreportxmlfile'] = new_coredumpreportxmlfile
if not crashobj['coredumpreporthtmlfile'] or force:
crashobj['coredumpreporthtmlfile'] = new_coredumpreporthtmlfile
else:
crashobj['minidumpfile'] = new_minidumpfile
crashobj['minidumpreporttextfile'] = new_minidumpreporttextfile
crashobj['minidumpreportxmlfile'] = new_minidumpreportxmlfile
crashobj['minidumpreporthtmlfile'] = new_minidumpreporthtmlfile
crashobj['coredumpfile'] = new_coredumpfile
crashobj['coredumpreporttextfile'] = new_coredumpreporttextfile
crashobj['coredumpreportxmlfile'] = new_coredumpreportxmlfile
crashobj['coredumpreporthtmlfile'] = new_coredumpreporthtmlfile
self.log.debug('after crashobj[minidumpfile] \'%s\'' % crashobj['minidumpfile'])
self.log.debug('after crashobj[minidumpreportxmlfile] \'%s\'' % crashobj['minidumpreportxmlfile'])
new_applicationfile = req.args.get('applicationfile')
if not crashobj['applicationfile']:
crashobj['applicationfile'] = new_applicationfile
self.log.debug('crashtimestamp from http form \'%s\'' % req.args.get('crashtimestamp'))
self.log.debug('reporttimestamp from http form \'%s\'' % req.args.get('reporttimestamp'))
try:
crashtimestamp = parse_date(req.args.get('crashtimestamp', ''), hint='iso8601' )
except TracError:
crashtimestamp = None
self.log.warn('invalid crash timestamp \'%s\'' % (req.args.get('crashtimestamp')))
try:
reporttimestamp = parse_date(req.args.get('reporttimestamp', ''), hint='iso8601' )
except TracError:
reporttimestamp = None
self.log.warn('invalid crash report timestamp \'%s\'' % (req.args.get('reporttimestamp')))
crashobj['crashtime'] = crashtimestamp if crashtimestamp else None
crashobj['reporttime'] = reporttimestamp if reporttimestamp else None
crashobj['uploadtime'] = datetime.datetime.now(utc)
self.log.debug('crashtimestamp %s' % (crashobj['crashtime']))
self.log.debug('reporttimestamp %s' % (crashobj['reporttime']))
self.log.debug('uploadtime %s' % (crashobj['uploadtime']))
if not manual_upload:
crashobj['productname'] = req.args.get('productname')
crashobj['productcodename'] = req.args.get('productcodename')
crashobj['productversion'] = req.args.get('productversion')
crashobj['producttargetversion'] = req.args.get('producttargetversion')
crashobj['uploadhostname'] = req.args.get('fqdn')
crashobj['uploadusername'] = req.args.get('username')
crashobj['crashhostname'] = req.args.get('crashfqdn')
crashobj['crashusername'] = req.args.get('crashusername')
crashobj['buildtype'] = req.args.get('buildtype')
crashobj['buildpostfix'] = req.args.get('buildpostfix')
crashobj['machinetype'] = req.args.get('machinetype')
crashobj['systemname'] = req.args.get('systemname')
crashobj['osversion'] = req.args.get('osversion')
crashobj['osrelease'] = req.args.get('osrelease')
crashobj['osmachine'] = req.args.get('osmachine')
if result:
xmlreport = None
try:
if crashobj['minidumpreportxmlfile']:
xmlfile = self._get_dump_filename(crashobj, 'minidumpreportxmlfile')
xmlreport = XMLReport(xmlfile)
elif crashobj['coredumpreportxmlfile']:
xmlfile = self._get_dump_filename(crashobj, 'coredumpreportxmlfile')
xmlreport = XMLReport(xmlfile)
except XMLReport.XMLReportException as e:
return self._error_response(req, status=HTTPInternalServerError.code, body='Failed to process crash dump %s: %s' % (uuid, str(e)))
if xmlreport and manual_upload:
if xmlreport.crash_info:
crashobj['crashtime'] = xmlreport.crash_info.crash_timestamp
crashobj['reporttime'] = xmlreport.crash_info.report_time
crashobj['uploadhostname'] = req.remote_addr
crashobj['uploadusername'] = req.remote_user
crashobj['applicationfile'] = xmlreport.crash_info.application
if xmlreport.fast_protect_version_info:
crashobj['productname'] = xmlreport.fast_protect_version_info.product_name
crashobj['productcodename'] = xmlreport.fast_protect_version_info.product_code_name
crashobj['productversion'] = xmlreport.fast_protect_version_info.product_version
crashobj['producttargetversion'] = xmlreport.fast_protect_version_info.product_target_version
crashobj['buildtype'] = xmlreport.fast_protect_version_info.product_build_type
crashobj['buildpostfix'] = xmlreport.fast_protect_version_info.product_build_postfix
if xmlreport.fast_protect_system_info:
crashobj['crashhostname'] = xmlreport.fast_protect_system_info.fqdn
crashobj['crashusername'] = xmlreport.fast_protect_system_info.username
crashobj['machinetype'] = xmlreport.fast_protect_system_info.machine_type
if xmlreport.system_info:
crashobj['systemname'] = xmlreport.system_info.platform_type
crashobj['osversion'] = xmlreport.system_info.os_version
crashobj['osrelease'] = xmlreport.system_info.os_build_number
crashobj['osmachine'] = xmlreport.system_info.cpu_type
# get the application name from the application file
if crashobj['applicationfile']:
appfile = crashobj['applicationfile']
if '/' in appfile:
appbase = appfile.split('/')[-1]
elif '\\' in appfile:
appbase = appfile.split('\\')[-1]
else:
appbase = os.path.basename(appfile)
(appbase, ext) = os.path.splitext(appbase)
if crashobj['buildpostfix'] and appbase.endswith(crashobj['buildpostfix']):
appbase = appbase[:-len(crashobj['buildpostfix'])]
crashobj['applicationname'] = appbase
new_crash = True if crashid is None else False
if new_crash:
crashobj['status'] = 'new'
crashobj['type'] = 'crash'
crashobj['priority'] = self.default_priority
if self.default_milestone == '< default >':
crashobj['milestone'] = self._find_milestone(crashobj['productversion'], crashobj['producttargetversion'])
else:
crashobj['milestone'] = self.default_milestone
if self.default_version == '< default >':
crashobj['version'] = self._find_version(crashobj['productversion'], crashobj['producttargetversion'])
else:
crashobj['version'] = self.default_version
if self.default_component == '< default >':
if xmlreport is not None and xmlreport.exception is not None and xmlreport.exception.involved_modules:
crashobj['component'] = self._find_component_from_involved_modules(xmlreport.exception.involved_modules, crashobj['buildpostfix'])
if not crashobj['component']:
crashobj['component'] = self._find_component_for_application(crashobj['applicationname'])
else:
crashobj['component'] = self.default_component
crashobj['severity'] = self.default_severity
crashobj['summary'] = self.default_summary
crashobj['description'] = self.default_description
crashobj['keywords'] = self.default_keywords
if self.default_owner == '< default >':
default_to_owner = ''
if crashobj['component']:
try:
component = TicketComponent(self.env, crashobj['component'])
default_to_owner = component.owner # even if it's empty
except ResourceNotFound:
# No such component exists
pass
if default_to_owner:
crashobj['owner'] = default_to_owner
else:
# If the current owner is "< default >", we need to set it to
# _something_ else, even if that something else is blank.
crashobj['owner'] = crashobj['crashusername']
else:
crashobj['owner'] = self.default_owner
if self.default_reporter == '< default >':
crashobj['reporter'] = crashobj['crashusername']
else:
crashobj['reporter'] = self.default_reporter
# apply replacements on usernames in owner and reporter field
crashobj['owner'] = self._apply_username_replacements(crashobj['owner'])
crashobj['reporter'] = self._apply_username_replacements(crashobj['reporter'])
crashid = crashobj.insert()
result = True if crashid else False
if result:
if xmlreport is not None and xmlreport.exception is not None:
ex_thread = xmlreport.exception.thread
else:
ex_thread = None
if ex_thread is not None:
threadid = ex_thread.id
stackdump = ex_thread.simplified_stackdump if ex_thread.simplified_stackdump is not None else ex_thread.stackdump
if stackdump:
for frameno, frm in enumerate(stackdump.callstack):
frameobj = CrashDumpStackFrame(crashid, threadid,frameno, env=self.env)
frameobj['module'] = frm.module
frameobj['function'] = frm.function
frameobj['funcoff'] = frm.funcoff
frameobj['source'] = frm.source
frameobj['line'] = frm.line
frameobj['lineoff'] = frm.lineoff
frameobj.insert()
else:
#print('update crash %s' % crashobj)
result = crashobj.save_changes(author=crashobj['crashusername'])
if result:
values = crashobj.values
values['crashtimestamp'] = crashtimestamp
values['reporttimestamp'] = reporttimestamp
values['crashid'] = crashid
values['uuid'] = crashobj.uuid
values['app'] = crashobj['applicationname'] if crashobj['applicationname'] else crashobj['applicationfile']
# Update all already linked tickets
for tkt_id in crashobj.linked_tickets:
try:
new_linked_ticketobj = Ticket(env=self.env, tkt_id=tkt_id)
comment = """The crash [[/crash/%(uuid)s|CrashId#%(crashid)s - %(uuid)s]] has been updated by **%(uploadusername)s**
from **%(uploadhostname)s** is already linked to this ticket.
""" % values
new_linked_ticketobj.save_changes(author=crashobj['reporter'], comment=comment)
# Only add valid tickets to the linked_tickets set
linked_tickets.add(tkt_id)
except ResourceNotFound:
pass
if new_ticket is not None:
new_ticket['type'] = self.default_ticket_type
new_ticket['summary'] = "Crash %(uuid)s in %(app)s" % values
comment = """The crash [[/crash/%(uuid)s|CrashId#%(crashid)s - %(uuid)s]] has been uploaded by **%(uploadusername)s**
from **%(uploadhostname)s** and this ticket has been automatically created to track the progress in finding and resolving the cause of the crash.
""" % values
new_ticket['description'] = comment
# copy over some fields from the crash itself
for field in ['status', 'priority', 'milestone', 'component',
'severity', 'keywords']:
new_ticket[field] = crashobj[field]
# apply replacements on usernames in owner and reporter field
new_ticket['owner'] = self._apply_username_replacements(crashobj['owner'])
new_ticket['reporter'] = self._apply_username_replacements(crashobj['reporter'])
new_ticket['linked_crash'] = str(crashid)
new_ticket.insert()
# Now add the newly linked tickets as well
for tkt_obj in ticketobjs:
if tkt_obj.id not in crashobj.linked_tickets:
ticket_values = self.escape_ticket_values(values)
#self.log.debug('ticket_values=%s' % str(ticket_values))
comment = """The crash [[/crash/%(uuid)s|CrashId#%(crashid)s - %(uuid)s]] has been uploaded by **%(uploadusername)s**
from **%(uploadhostname)s** and linked to this ticket.
The crash occured at //%(crashtimestamp)s UTC// on **%(crashhostname)s** with user **%(crashusername)s** while running `%(applicationfile)s`. The
application was running as part of %(productname)s (%(productcodename)s) version %(productversion)s (%(producttargetversion)s, %(buildtype)s) on a
%(systemname)s/%(machinetype)s with %(osversion)s (%(osrelease)s/%(osmachine)s).
""" % ticket_values
linked_crashes = tkt_obj['linked_crash'] if tkt_obj['linked_crash'] else ''
linked_crashes = set([int(x.strip()) for x in linked_crashes.split(',') if x.strip()])
#print('crashid=%s' % crashid)
#print('linked_crashes=%s' % linked_crashes)
linked_crashes.add(crashid)
#print('linked_crashes=%s' % linked_crashes)
tkt_obj['linked_crash'] = ', '.join(str(x) for x in sorted(linked_crashes))
tkt_obj.save_changes(author=crashobj['reporter'], comment=comment)
linked_tickets.add(tkt_obj.id)
with self.env.db_transaction as db:
links = CrashDumpTicketLinks(self.env, tkt=tkt_obj, db=db)
links.crashes.add(crashid)
links.save(author=crashobj['reporter'], db=db)
if result:
if manual_upload:
req.redirect(req.abs_href('crash', str(uuid)))
else:
headers = {}
linked_ticket_header = []
for tkt_id in linked_tickets:
linked_ticket_header.append('#%i:%s' % (tkt_id, req.abs_href.ticket(tkt_id)))
if linked_ticket_header:
headers['Linked-Tickets'] = ';'.join(linked_ticket_header)
headers['Crash-URL'] = req.abs_href('crash', str(uuid))
headers['CrashId'] = str(crashid)
return self._success_response(req, body='Crash dump %s uploaded successfully.' % uuid, headers=headers)
elif new_crash:
return self._error_response(req, status=HTTPInternalServerError.code, body='Failed to add crash dump %s to database' % uuid)
else:
headers = {}
headers['Crash-URL'] = req.abs_href('crash', str(uuid))
headers['CrashId'] = str(crashid)
return self._error_response(req, status=HTTPInternalServerError.code, body='Failed to update crash dump %s to database' % uuid, headers=headers)
else:
if failure_message is None:
body = 'Failed to process crash dump %s' % uuid
else:
body = 'The following occured while processing the crash dump %s: %s' % (uuid, failure_message)
return self._error_response(req, status=HTTPInternalServerError.code, body=body)
def process_request_crashlist(self, req):
if req.method != "GET":
return self._error_response(req, status=HTTPMethodNotAllowed.code, body='Method %s not allowed' % req.method)
user_agent = req.get_header('User-Agent')
if user_agent is None:
return self._error_response(req, status=HTTPForbidden.code, body='No user-agent specified.')
if '/' in user_agent:
user_agent, agent_ver = user_agent.split('/', 1)
#if user_agent != 'terra3d-crashuploader':
#return self._error_response(req, status=HTTPForbidden.code, body='User-agent %s not allowed' % user_agent)
req_status = req.args.get('status') or 'active'
headers = {}
body = ''
body = body + '<?xml version="1.0" encoding="utf-8"?>\r\n<crashlist>\r\n'
for crashobj in CrashDump.query(env=self.env, status=req_status):
body = body + '<crash id=\"%i\" uuid=\"%s\" url=\"%s\" xmlreport=\"%s\" rawfile=\"%s\">\r\n' % \
(crashobj.id, crashobj['uuid'],
req.href('crash', crashobj['uuid']),
req.href('crash', crashobj['uuid'], 'xml'),
req.href('crash', crashobj['uuid'], 'raw'),
)
for field in crashobj.fields:
field_name = field['name']
if field_name == 'uuid':
continue
field_type = field['type']
field_value = crashobj[field_name]
if field_type == 'time':
field_value = str(to_utimestamp(field_value))
body = body + '<%s type=\"%s\">' % (field_name, field_type)
if field_value is not None:
#print('%s=%s' % (field_name, field_value))
body = body + escape(field_value)
body = body + '</%s>\r\n' % (field_name)
body = body + '<linked_tickets>\r\n'
for tkt in crashobj.linked_tickets:
body = body + '<ticket id=\"%i\" url=\"%s\">\r\n' % (tkt, req.href.ticket(tkt))
body = body + '</ticket>\r\n'
body = body + '</linked_tickets>\r\n'
body = body + '</crash>\r\n'
body = body + '</crashlist>\r\n'
return self._success_response(req, body=body.encode('utf-8'), headers=headers)
# ITemplateProvider methods
def get_htdocs_dirs(self):
"""Return the absolute path of a directory containing additional
static resources (such as images, style sheets, etc).
"""
return [('crashdump', resource_filename(__name__, 'htdocs'))]
def get_templates_dirs(self):
"""Return the absolute path of the directory containing the provided
ClearSilver templates.
"""
return [resource_filename(__name__, 'templates')]
@property
def path(self):
return self._get_path(self.env.path, self.parent_realm, self.parent_id,
self.filename)
def _create_crash_file(self, filename, force):
flags = os.O_CREAT + os.O_WRONLY
if force:
flags += os.O_TRUNC
else:
if os.path.isfile(filename):
return None
flags += os.O_EXCL
if hasattr(os, 'O_BINARY'):
flags += os.O_BINARY
return os.fdopen(os.open(filename, flags, 0660), 'w')
def _get_total_upload_size(self, req):
ret = 0
files_fields = ['minidump', 'minidumpreport', 'minidumpreportxml', 'minidumpreporthtml',
'coredump', 'coredumpreport', 'coredumpreportxml', 'coredumpreporthtml']
for name in files_fields:
file = req.args.get(name) if name in req.args else None
if file is None:
continue
if hasattr(file, 'fileno'):
size = os.fstat(file.fileno())[6]
else:
file.file.seek(0, 2) # seek to end of file
size = file.file.tell()
file.file.seek(0)
self.log.debug('found file name %s, size %i' % (name, size))
ret = ret + size
return ret
def _store_dump_file(self, uuid, req, name, force):
item_name = None
ret = False
file = req.args.get(name) if name in req.args else None
errmsg = None
if file is None:
errmsg = 'Field %s not available' % name
else:
filename = file.filename
fileobj = file.file
item_name = os.path.join(str(uuid), filename)
crash_dir = os.path.join(self.env.path, self.dumpdata_dir, str(uuid))
crash_file = os.path.join(crash_dir, filename)
self.log.debug('_store_dump_file env.path %s' % (self.env.path))
self.log.debug('_store_dump_file self.dumpdata_dir %s' % (self.dumpdata_dir))
self.log.debug('_store_dump_file item_name %s' % (item_name))
self.log.debug('_store_dump_file crash_dir %s' % (crash_dir))
self.log.debug('_store_dump_file crash_file %s' % (crash_file))
if not os.path.isdir(crash_dir):
os.makedirs(crash_dir)
flags = os.O_CREAT + os.O_WRONLY
flags += os.O_TRUNC
#if force:
#flags += os.O_TRUNC
#else:
#if os.path.isfile(crash_file):
#errmsg = 'File %s already exists.' % crash_file
#return (False, item_name, errmsg)
#flags += os.O_EXCL
if hasattr(os, 'O_BINARY'):
flags += os.O_BINARY
targetfileobj = None
try:
targetfileobj = os.fdopen(os.open(crash_file, flags, 0660), 'w')
except OSError as e:
errmsg = str(e)
except IOError as e:
errmsg = str(e)
if targetfileobj is None:
ret = False
if errmsg is None:
errmsg = 'Cannot open file %s.' % crash_file
else:
try:
shutil.copyfileobj(fileobj, targetfileobj)
ret = True
except OSError as e:
errmsg = str(e)
except IOError as e:
errmsg = str(e)
return (ret, item_name, errmsg)
def _get_dump_filename(self, crashobj, name):
item_name = crashobj[name]
crash_file = os.path.join(self.env.path, self.dumpdata_dir, item_name)
return crash_file
class AuthzSourcePolicy(Component):
"""Permission policy for `source:` and `changeset:` resources using a
Subversion authz file.
`FILE_VIEW` and `BROWSER_VIEW` permissions are granted as specified in the
authz file.
`CHANGESET_VIEW` permission is granted for changesets where `FILE_VIEW` is
granted on at least one modified file, as well as for empty changesets.
"""
implements(IPermissionPolicy)
authz_file = PathOption(
'trac', 'authz_file', '', """The path to the Subversion
[http://svnbook.red-bean.com/en/1.5/svn.serverconfig.pathbasedauthz.html authorization (authz) file].
To enable authz permission checking, the `AuthzSourcePolicy` permission
policy must be added to `[trac] permission_policies`.
""")
authz_module_name = Option(
'trac', 'authz_module_name', '',
"""The module prefix used in the `authz_file` for the default
repository. If left empty, the global section is used.
""")
_mtime = 0
_authz = {}
_users = set()
_handled_perms = frozenset([(None, 'BROWSER_VIEW'),
(None, 'CHANGESET_VIEW'), (None, 'FILE_VIEW'),
(None, 'LOG_VIEW'), ('source', 'BROWSER_VIEW'),
('source', 'FILE_VIEW'),
('source', 'LOG_VIEW'),
('changeset', 'CHANGESET_VIEW')])
# IPermissionPolicy methods
def check_permission(self, action, username, resource, perm):
realm = resource.realm if resource else None
if (realm, action) in self._handled_perms:
authz, users = self._get_authz_info()
if authz is None:
return False
if username == 'anonymous':
usernames = ('$anonymous', '*')
else:
usernames = (username, '$authenticated', '*')
if resource is None:
return True if users & set(usernames) else None
rm = RepositoryManager(self.env)
try:
repos = rm.get_repository(resource.parent.id)
except TracError:
return True # Allow error to be displayed in the repo index
if repos is None:
return True
modules = [resource.parent.id or self.authz_module_name]
if modules[0]:
modules.append('')
def check_path(path):
path = '/' + join(repos.scope, path)
if path != '/':
path += '/'
# Allow access to parent directories of allowed resources
if any(
section.get(user) is True
for module in modules for spath, section in authz.get(
module, {}).iteritems() if spath.startswith(path)
for user in usernames):
return True
# Walk from resource up parent directories
for spath in parent_iter(path):
for module in modules:
section = authz.get(module, {}).get(spath)
if section:
for user in usernames:
result = section.get(user)
if result is not None:
return result
if realm == 'source':
return check_path(resource.id)
elif realm == 'changeset':
changes = list(repos.get_changeset(resource.id).get_changes())
if not changes or any(
check_path(change[0]) for change in changes):
return True
def _get_authz_info(self):
try:
mtime = os.path.getmtime(self.authz_file)
except OSError, e:
if self._authz is not None:
self.log.error('Error accessing authz file: %s',
exception_to_unicode(e))
self._mtime = mtime = 0
self._authz = None
self._users = set()
if mtime > self._mtime:
self._mtime = mtime
rm = RepositoryManager(self.env)
modules = set(repos.reponame
for repos in rm.get_real_repositories())
if '' in modules and self.authz_module_name:
modules.add(self.authz_module_name)
modules.add('')
self.log.info('Parsing authz file: %s' % self.authz_file)
try:
self._authz = parse(read_file(self.authz_file), modules)
self._users = set(user for paths in self._authz.itervalues()
for path in paths.itervalues()
for user, result in path.iteritems()
if result)
except Exception, e:
self._authz = None
self._users = set()
self.log.error('Error parsing authz file: %s',
exception_to_unicode(e))
class DownloadPlugin(Component):
implements(INavigationContributor, IRequestHandler, IAdminPanelProvider,
ITemplateProvider, IPermissionRequestor)
path = PathOption('downloads',
'path',
'../downloads',
doc="Path where to store uploaded downloads.")
ext = ListOption(
'downloads',
'ext',
'zip,gz,bz2,rar',
doc="""List of file extensions allowed to upload. Set to 'all'
to specify that any file extensions is allowed.
""")
max_size = IntOption(
'downloads', 'max_size', 268697600,
"""Maximum allowed file size (in bytes) for downloads. Default
is 256 MB.
""")
def __init__(self):
self.env.log.debug('Download Initial')
# INavigationContributor methods
def get_active_navigation_item(self, req):
if 'DOWNLOAD_VIEW' in req.perm:
return 'downloads'
def get_navigation_items(self, req):
if 'DOWNLOAD_VIEW' in req.perm:
yield ('mainnav', 'downloads',
html.a('Downloads', href=req.href.downloads()))
# IRequestHandler methods
def match_request(self, req):
return req.path_info.find('/downloads') == 0
def process_request(self, req):
data = {}
cursor = self.env.db_query(
"SELECT id, file, description FROM download ORDER BY id")
data['downloads'] = [(row[0], row[1], row[2]) for row in cursor]
return ('downloads.html', data, None)
# IAdminPageProvider methods
def get_admin_panels(self, req):
if req.perm.has_permission('DOWNLOAD_ADMIN'):
yield ('general', 'General', 'download', 'Download')
def render_admin_panel(self, req, cat, page, version):
# here comes the page content, handling, etc.
data = {}
if req.method == "POST":
submit = req.args.get('submit').strip()
if submit == 'Add':
# Get form values.
#file = req.args['download']
# Test if file is uploaded.
#if hasattr(file, 'filename'):
# self.log.debug("Filename:" + file.filename)
file, filename, file_size = self.get_file_from_req(req)
download = {
'file': filename,
'description': req.args.get('description'),
'size': file_size,
'time': to_timestamp(datetime.datetime.now(utc)),
'count': 0,
'author': req.authname
}
self.log.debug("FileUpload filename:" + download['file'])
self.log.debug("FileUpload description:" +
download['description'])
self.log.debug("FileUpload size:", download['size'])
self.log.debug("FileUpload time:", download['time'])
self.log.debug("FileUpload author:" + download['author'])
# Upload file to DB and file storage.
add_download(download, file)
file.close()
add_notice(req, 'Download has been added.')
elif submit == 'Remove':
ids = req.args.getlist('sels')
if ids is not None and len(ids) > 0:
for id in ids:
sql = "DELETE FROM download WHERE id ={}".format(
int(id))
self.env.db_transaction(sql)
add_notice(req, 'Download has been deleted.')
else:
# Get download.
download_id = req.args.get('sel') or 0
if download_id > 0:
sql = "SELECT file, description FROM download where id={}".format(
download_id)
cursor = self.env.db_query(sql)
if len(cursor) > 0:
fn = cursor[0][0]
description = cursor[0][1]
else:
raise TracError("File not found.")
# Get download file path.
filename = os.path.basename(fn)
filepath = os.path.join(self.path, to_unicode(download_id),
filename)
filepath = os.path.normpath(filepath)
# Increase downloads count.
sql = "UPDATE download SET count=count+1 WHERE id ={}".format(
download_id)
self.env.db_transaction(sql)
# Guess mime type.
with open(filepath.encode('utf-8'), 'r') as fileobj:
file_data = fileobj.read(1000)
mimeview = Mimeview(self.env)
mime_type = mimeview.get_mimetype(filepath, file_data)
if not mime_type:
mime_type = 'application/octet-stream'
if 'charset=' not in mime_type:
charset = mimeview.get_charset(file_data, mime_type)
mime_type = mime_type + '; charset=' + charset
# Return uploaded file to request.
req.send_header(
'Content-Disposition',
'attachment;filename="%s"' % os.path.normpath(fn))
req.send_header('Content-Description', description)
req.send_file(filepath.encode('utf-8'), mime_type)
cursor = self.env.db_query(
"SELECT id, file, description, size, time, author FROM download ORDER BY id"
)
data['downloads'] = [(row[0], row[1], row[2]) for row in cursor]
return ('admin_download.html', data, None)
# ITemplateProvider
def get_htdocs_dirs(self):
"""Return the absolute path of a directory containing additional
static resources (such as images, style sheets, etc).
"""
from pkg_resources import resource_filename
return [resource_filename(__name__, 'htdocs')]
def get_templates_dirs(self):
"""Return the absolute path of the directory containing the provided
ClearSilver/Genshi templates.
"""
from pkg_resources import resource_filename
return [resource_filename(__name__, 'templates')]
# IPermissionRequestor methods.
def get_permission_actions(self):
view = 'DOWNLOAD_VIEW'
add = ('DOWNLOAD_ADD', ['DOWNLOAD_VIEW'])
admin = ('DOWNLOAD_ADMIN', ['DOWNLOAD_VIEW', 'DOWNLOAD_ADD'])
return [view, add, admin]
def get_download_id_by_time(self, time):
cursor = self.env.db_query(
"SELECT id, file, description, size, time, author FROM download where time={}"
.format(time))
for row in cursor:
return row[0]
return None
def get_file_from_req(self, req):
file = req.args['file']
# Test if file is uploaded.
if not hasattr(file, 'filename') or not file.filename:
raise TracError("No file uploaded.")
# Get file size.
if hasattr(file.file, 'fileno'):
size = os.fstat(file.file.fileno())[6]
else:
# Seek to end of file to get its size.
file.file.seek(0, 2)
size = file.file.tell()
file.file.seek(0)
if size == 0:
raise TracError("Can't upload empty file.")
# Try to normalize the filename to unicode NFC if we can.
# Files uploaded from OS X might be in NFD.
self.log.debug("input filename: %s", file.filename)
filename = unicodedata.normalize('NFC',
to_unicode(file.filename, 'utf-8'))
filename = filename.replace('\\', '/').replace(':', '/')
filename = os.path.basename(filename)
self.log.debug("output filename: %s", filename)
return file.file, filename, size
def add_download(self, download, file):
# Check for file name uniqueness.
#if self.unique_filename:
# if self.get_download_by_file(download['file']):
# raise TracError("File with same name is already uploaded "
# "and unique file names are enabled.")
# Check correct file type.
#name, ext = os.path.splitext(download['file'])
#if not (ext[1:].lower() in self.ext) and not ('all' in self.ext):
# raise TracError("Unsupported file type.")
# Check for maximum file size.
if 0 <= self.max_size < download['size']:
raise TracError("Maximum file size: %s bytes" % self.max_size,
"Upload failed")
# Add new download to DB.
#self.add_download(download)
sql = "INSERT INTO download (file,description,size,time,author) " \
" VALUES(%s,%s,%s,%s,%s)"
args = (download['file'], download['description'], download['size'],
download['time'], download['author'])
self.env.db_transaction(sql, args)
self.log.debug("FileUpload SQL: %s", sql)
# Get inserted download by time to get its ID.
id = self.get_download_id_by_time(download['time'])
self.log.debug("FileUpload id: %s", id)
# Prepare file paths.
path = os.path.normpath(os.path.join(self.path, to_unicode(id)))
filepath = os.path.normpath(os.path.join(path, download['file']))
self.log.debug("FileUpload path: %s", path)
self.log.debug("FileUpload filepath: %s", filepath)
# Store uploaded image.
try:
os.mkdir(path.encode('utf-8'))
with open(filepath.encode('utf-8'), 'wb+') as fileobj:
file.seek(0)
shutil.copyfileobj(file, fileobj)
except Exception as error:
self._delete_download(id)
self.log.debug(error)
try:
os.remove(filepath.encode('utf-8'))
except:
pass
try:
os.rmdir(path.encode('utf-8'))
except:
pass
raise TracError("Error storing file %s. Does the directory "
"specified in path config option of [downloads] "
"section of trac.ini exist?" % download['file'])
"""Use git-committer id instead of git-author id for the
changeset ''Author'' field.
""")
use_committer_time = BoolOption(
'git', 'use_committer_time', 'true',
"""Use git-committer timestamp instead of git-author timestamp
for the changeset ''Timestamp'' field.
""")
git_fs_encoding = Option(
'git', 'git_fs_encoding', 'utf-8',
"""Define charset encoding of paths within git repositories.""")
git_bin = PathOption(
'git', 'git_bin', '/usr/bin/git',
"""Path to git executable (relative to the Trac configuration folder,
so better use an absolute path here).""")
def get_supported_types(self):
yield ('git', 8)
def get_repository(self, type, dir, params):
"""GitRepository factory method"""
assert type == 'git'
if not (4 <= self.shortrev_len <= 40):
raise TracError(
"[git] shortrev_len setting must be within [4..40]")
if not (4 <= self.wiki_shortrev_len <= 40):
raise TracError("[git] wikishortrev_len must be within [4..40]")
class MultiProductSystem(Component):
"""Creates the database tables and template directories"""
implements(IEnvironmentSetupParticipant, IExternalResourceConnector,
IPermissionRequestor, IResourceChangeListener, IResourceManager,
ISupportMultiProductEnvironment, ITemplateProvider,
ITicketFieldProvider, IWikiSyntaxProvider, ITicketManipulator)
default_product_prefix = Option(
'multiproduct',
'default_product_prefix',
default='@',
doc="""Prefix used for default product when migrating single-product
installations to multi-product.""", doc_domain='multiproduct')
default_product = Option('ticket', 'default_product', '',
"""Default product for newly created tickets.""")
product_base_url = Option('multiproduct', 'product_base_url', '',
"""A pattern used to generate the base URL of product environments,
e.g. the use cases listed in bh:wiki:/Proposals/BEP-0003#url-mapping .
Both absolute as well as relative URLs are supported. The later
will be resolved with respect to the base URL of the parent global
environment. The pattern may contain references to $(envname)s,
$(prefix)s and $(name)s placeholders representing the environment name,
product prefix and product name respectively . If nothing is set the
following will be used `products/$(prefix)s`
Note the usage of `$(...)s` instead of `%(...)s` as the later form
would be interpreted by the ConfigParser itself. """,
doc_domain='multiproduct')
product_config_parent = PathOption('inherit', 'multiproduct', '',
"""The path to the configuration file containing the settings shared
by sibling product environments. By default will inherit
global environment configuration.
""", doc_domain='multiproduct')
SCHEMA = [mcls._get_schema()
for mcls in (Product, ProductResourceMap)]
# Tables which should be migrated (extended with 'product' column)
MIGRATE_TABLES = ['component',
'milestone',
'version',
'enum',
'permission',
'wiki',
'report',
]
PRODUCT_POPULATE_TABLES = list(set(MIGRATE_TABLES) - set(['wiki']))
def __init__(self, *args, **kwargs):
import pkg_resources
locale_dir = pkg_resources.resource_filename(__name__, 'locale')
add_domain(self.env.path, locale_dir)
super(MultiProductSystem, self).__init__(*args, **kwargs)
def get_version(self):
"""Finds the current version of the bloodhound database schema"""
rows = self.env.db_direct_query("""
SELECT value FROM system WHERE name = %s
""", (DB_SYSTEM_KEY,))
return int(rows[0][0]) if rows else -1
# IEnvironmentSetupParticipant methods
def environment_created(self):
"""Insertion of any default data into the database."""
self.log.debug("creating environment for %s plugin." % PLUGIN_NAME)
def environment_needs_upgrade(self, db_dummy=None):
"""Detects if the installed db version matches the running system"""
db_installed_version = self.get_version()
if db_installed_version > DB_VERSION:
raise TracError('''Current db version (%d) newer than supported by
this version of the %s (%d).''' % (db_installed_version,
PLUGIN_NAME,
DB_VERSION))
needs_upgrade = db_installed_version < DB_VERSION
if not needs_upgrade:
self.env.enable_multiproduct_schema(True)
return needs_upgrade
def _update_db_version(self, db, version):
old_version = self.get_version()
if old_version != -1:
self.log.info("Updating multiproduct database schema from version %d"
" to %d" % (old_version, version))
db("""UPDATE system SET value=%s
WHERE name=%s""", (version, DB_SYSTEM_KEY))
else:
self.log.info("Initial multiproduct database schema set to version %d" % version)
db("""
INSERT INTO system (name, value) VALUES ('%s','%s')
""" % (DB_SYSTEM_KEY, version))
return version
_system_wiki_list = None
@property
def system_wiki_list(self):
if MultiProductSystem._system_wiki_list is None:
MultiProductSystem._system_wiki_list = self._get_system_wiki_list()
return MultiProductSystem._system_wiki_list
def _get_system_wiki_list(self):
"""Helper function that enumerates all 'system' wikis. The
list is combined of default wiki pages and pages that are
bundled with Bloodhound dashboard and search plugins"""
from bhdashboard import wiki
paths = [resource_filename('trac.wiki',
'default-pages')] + \
[resource_filename('bhdashboard',
'default-pages')] + \
[resource_filename('bhsearch',
'default-pages')]
pages = []
original_pages = []
for path in paths:
for page in os.listdir(path):
filename = os.path.join(path, page)
page = unicode_unquote(page.encode('utf-8'))
if os.path.isfile(filename):
original_pages.append(page)
for original_name in original_pages:
if original_name.startswith('Trac'):
new_name = wiki.new_name(original_name)
if not new_name:
continue
if new_name in original_pages:
continue
name = new_name
# original trac wikis should also be included in the list
pages.append(original_name)
else:
name = original_name
pages.append(name)
return pages
def upgrade_environment(self, db_dummy=None):
"""Installs or updates tables to current version"""
self.log.debug("upgrading existing environment for %s plugin." %
PLUGIN_NAME)
db_installed_version = self.get_version()
with self.env.db_direct_transaction as db:
if db_installed_version < 1:
self._add_column_product_to_ticket(db)
self._create_multiproduct_tables(db)
db_installed_version = self._update_db_version(db, 1)
if db_installed_version < 2:
self._replace_product_on_ticket_with_product_prefix(db)
db_installed_version = self._update_db_version(db, 2)
if db_installed_version < 3:
SYSTEM_TABLES = ['system']
TICKET_TABLES = [
'ticket_change', 'ticket_custom', 'attachment',
]
table_defs = self._add_product_column_to_tables(
self.MIGRATE_TABLES + TICKET_TABLES + SYSTEM_TABLES,
db_installed_version)
table_columns = self._get_table_columns(table_defs)
create_temp_table = lambda table: self._create_temp_table(
db, table, table_columns, table_defs)
self._insert_default_product(db)
self._upgrade_tickets(db, TICKET_TABLES, create_temp_table)
self._upgrade_wikis(db, create_temp_table)
self._upgrade_system_tables(db, create_temp_table)
self._soft_link_repositories_to_default_product(db)
self._upgrade_table_system(SYSTEM_TABLES, create_temp_table, db)
self._enable_multiproduct_hooks()
db_installed_version = self._update_db_version(db, 3)
if db_installed_version < 4:
self._create_product_tables_for_plugins(db)
db_installed_version = self._update_db_version(db, 4)
if db_installed_version < 5:
table_defs = self._add_product_column_to_tables(
['ticket'], db_installed_version)
self._modify_ticket_pk(db, table_defs)
db_installed_version = self._update_db_version(db, 5)
self.env.enable_multiproduct_schema(True)
def _add_column_product_to_ticket(self, db):
self.log.debug("Adding field product to ticket table")
db("ALTER TABLE ticket ADD COLUMN product TEXT")
def _create_multiproduct_tables(self, db):
self.log.debug("Creating initial db tables for %s plugin." %
PLUGIN_NAME)
db_connector, dummy = DatabaseManager(self.env)._get_connector()
for table in self.SCHEMA:
for statement in db_connector.to_sql(table):
db(statement)
def _replace_product_on_ticket_with_product_prefix(self, db):
for prod in Product.select(self.env):
db("""UPDATE ticket SET product=%s
WHERE product=%s""", (prod.prefix, prod.name))
def _create_temp_table(self, db, table, table_columns, table_defs):
"""creates temporary table with the new schema and
drops original table"""
table_temp_name = '%s_temp' % table
if table == 'report':
cols = ','.join([c for c in table_columns[table] if c != 'id'])
else:
cols = ','.join(table_columns[table])
self.log.info("Migrating table '%s' to a new schema", table)
db("""CREATE TABLE %s AS SELECT %s FROM %s""" %
(table_temp_name, cols, table))
db("""DROP TABLE %s""" % table)
db_connector, _ = DatabaseManager(self.env)._get_connector()
table_schema = [t for t in table_defs if t.name == table][0]
for sql in db_connector.to_sql(table_schema):
db(sql)
return table_temp_name, cols
def _drop_temp_table(self, db, table):
db("""DROP TABLE %s""" % table)
def _add_product_column_to_tables(self, tables, current_version):
"""Extend trac default schema by adding product column
and extending key with product.
"""
table_defs = [copy.deepcopy(t) for t in trac.db_default.schema
if
t.name in tables]
for t in table_defs:
t.columns.append(Column('product'))
if isinstance(t.key, list):
t.key = tuple(t.key) + tuple(['product'])
elif isinstance(t.key, tuple):
t.key = t.key + tuple(['product'])
else:
raise TracError(
"Invalid table '%s' schema key '%s' while upgrading "
"plugin '%s' from version %d to %d'" %
(t.name, t.key, PLUGIN_NAME, current_version, 3))
return table_defs
def _get_table_columns(self, table_defs, all_columns=False):
table_columns = dict()
for table in table_defs:
table_definition = \
[t for t in table_defs if t.name == table.name][0]
column_names = \
[column.name for column in table_definition.columns]
table_columns[table.name] = \
[c for c in column_names if all_columns or c != 'product']
return table_columns
def _insert_default_product(self, db):
self.log.info("Creating default product")
db("""INSERT INTO bloodhound_product (prefix, name, description, owner)
VALUES ('%s', '%s', '%s', '')
""" % (self.default_product_prefix, 'Default', 'Default product'))
def _upgrade_tickets(self, db, TICKET_TABLES, create_temp_table):
# migrate tickets that don't have product assigned to default product
# - update ticket table product column
# - update ticket related tables by:
# - upgrading schema
# - update product column to match ticket's product
self.log.info("Migrating tickets w/o product to default product")
db("""UPDATE ticket SET product='%s'
WHERE (product IS NULL OR product='')
""" % self.default_product_prefix)
self._migrate_attachments(
db("""SELECT a.type, a.id, a.filename
FROM attachment a
INNER JOIN ticket t ON a.id = %(t.id)s
WHERE a.type='ticket'
""" % {'t.id': db.cast('t.id', 'text')}),
to_product=self.default_product_prefix
)
self.log.info("Migrating ticket tables to a new schema")
for table in TICKET_TABLES:
temp_table_name, cols = create_temp_table(table)
db("""INSERT INTO %s (%s, product)
SELECT %s, '' FROM %s""" %
(table, cols, cols, temp_table_name))
self._drop_temp_table(db, temp_table_name)
if table == 'attachment':
db("""UPDATE attachment
SET product=(SELECT ticket.product
FROM ticket
WHERE %(ticket.id)s=attachment.id
LIMIT 1)
WHERE attachment.type='ticket'
AND EXISTS(SELECT ticket.product
FROM ticket
WHERE %(ticket.id)s=attachment.id)
""" % {'ticket.id': db.cast('ticket.id', 'text')})
else:
db("""UPDATE %(table)s
SET product=(SELECT ticket.product
FROM ticket
WHERE ticket.id=%(table)s.ticket)
""" % {'table': table})
def _upgrade_system_tables(self, db, create_temp_table):
# migrate system table (except wiki which is handled separately)
# to a new schema
# - create tables with the new schema
# - populate system tables with global configuration for each product
# - exception is permission table where permissions
# are also populated in global scope
#
# permission table specifics: 'anonymous' and 'authenticated' users
# should by default have a PRODUCT_VIEW permission for all products
self.log.info("Migrating system tables to a new schema")
for table in self.MIGRATE_TABLES:
if table == 'wiki':
continue
temp_table_name, cols = create_temp_table(table)
for product in Product.select(self.env):
self.log.info("Populating table '%s' for product '%s' ('%s')",
table, product.name, product.prefix)
db("""INSERT INTO %s (%s, product) SELECT %s,'%s' FROM %s""" %
(table, cols, cols, product.prefix, temp_table_name))
if table == 'permission':
db.executemany(
"""INSERT INTO permission (username, action, product)
VALUES (%s, %s, %s)""",
[('anonymous', 'PRODUCT_VIEW', product.prefix),
('authenticated', 'PRODUCT_VIEW', product.prefix)])
if table == 'permission':
self.log.info("Populating table '%s' for global scope", table)
db("""INSERT INTO %s (%s, product) SELECT %s,'%s' FROM %s""" %
(table, cols, cols, '', temp_table_name))
self._drop_temp_table(db, temp_table_name)
db.executemany(
"""INSERT INTO permission (username, action, product)
VALUES (%s, %s, %s)""",
[('anonymous', 'PRODUCT_VIEW', ''),
('authenticated', 'PRODUCT_VIEW', '')])
def _upgrade_wikis(self, db, create_temp_table):
# migrate wiki table
# - populate system wikis to all products + global scope
# - update wiki attachment product to match wiki product
table = 'wiki'
temp_table_name, cols = create_temp_table(table)
self.log.info("Migrating wikis to default product")
db("""INSERT INTO %(table)s (%(cols)s, product)
SELECT %(cols)s, '%(default_product)s' FROM %(temp_table)s
""" % dict(table=table,
temp_table=temp_table_name,
cols=cols,
default_product=self.default_product_prefix,))
db("""UPDATE attachment
SET product='%s'
WHERE attachment.type='wiki'
""" % self.default_product_prefix)
self._migrate_attachments(
db("""SELECT type, id, filename
FROM attachment
WHERE type='wiki'
AND product='%s'
""" % (self.default_product_prefix)),
to_product=self.default_product_prefix,
)
self._drop_temp_table(db, temp_table_name)
def _migrate_attachments(self, attachments, to_product=None, copy=False):
for type, id, filename in attachments:
old_path = Attachment._get_path(self.env.path, type, id, filename)
new_path = self.env.path
if to_product:
new_path = os.path.join(new_path, 'products', to_product)
new_path = Attachment._get_path(new_path, type, id, filename)
dirname = os.path.dirname(new_path)
if not os.path.exists(old_path):
self.log.warning(
"Missing attachment files for %s:%s/%s",
type, id, filename)
continue
if os.path.exists(new_path):
# TODO: Do we want to overwrite?
continue
try:
if not os.path.exists(dirname):
os.makedirs(dirname)
if copy:
if hasattr(os, 'link'):
# TODO: It this safe?
os.link(old_path, new_path)
else:
shutil.copy(old_path, new_path)
else:
os.rename(old_path, new_path)
except OSError as err:
self.log.warning(
"Could not move attachment %s from %s %s to"
"product @ (%s)",
filename, type, id, str(err)
)
def _soft_link_repositories_to_default_product(self, db):
# soft link existing repositories to default product
repositories_linked = []
for id, name in db("""SELECT id, value FROM repository
WHERE name='name'"""):
if id in repositories_linked:
continue
db("""INSERT INTO repository (id, name, value)
VALUES (%s, 'product', '%s')""" %
(id, self.default_product_prefix))
repositories_linked.append(id)
self.log.info("Repository '%s' (%s) soft linked to default product",
name, id)
def _upgrade_table_system(self, SYSTEM_TABLES, create_temp_table, db):
# Update system tables
# Upgrade schema
self.log.info("Migrating system tables to a new schema")
for table in SYSTEM_TABLES:
temp_table_name, cols = create_temp_table(table)
db("""INSERT INTO %s (%s, product)
SELECT %s,'' FROM %s""" %
(table, cols, cols, temp_table_name))
self._drop_temp_table(db, temp_table_name)
def _enable_multiproduct_hooks(self):
# enable multi product hooks in environment configuration
config_update = False
if not 'environment_factory' in self.env.config['trac']:
self.env.config['trac'].set('environment_factory',
'multiproduct.hooks.MultiProductEnvironmentFactory')
config_update = True
if not 'request_factory' in self.env.config['trac']:
self.env.config['trac'].set('request_factory',
'multiproduct.hooks.ProductRequestFactory')
config_update = True
if config_update:
self.log.info(
"Enabling multi product hooks in environment configuration")
self.env.config.save()
def _create_product_tables_for_plugins(self, db):
self.log.debug("creating additional db tables for %s plugin." %
PLUGIN_NAME)
db_connector, dummy = DatabaseManager(self.env)._get_connector()
for statement in db_connector.to_sql(ProductSetting._get_schema()):
db(statement)
def _modify_ticket_pk(self, db, table_defs):
self.log.debug("Modifying ticket primary key: id -> uid")
table_columns = self._get_table_columns(table_defs, True)
db_connector, _ = DatabaseManager(self.env)._get_connector()
def rename_id_to_uid(table):
for c in table.columns:
if c.name == 'id':
c.name = 'uid'
break
table.key = ['uid']
def add_new_id_column(table):
id_column = Column('id', type='int', auto_increment=True)
if using_sqlite_backend(self.env) or using_mysql_backend(self.env):
# sqlite and mysql don't support multiple auto increment columns
id_column.auto_increment = False
table.columns.append(id_column)
table.indices.append(Index(['product', 'id'], unique=True))
for t in table_defs:
rename_id_to_uid(t)
add_new_id_column(t)
temp_table_name, cols = self._create_temp_table(
db, t.name, table_columns, table_defs)
db("""INSERT INTO ticket (%s, uid)
SELECT %s, id FROM ticket_temp""" %
(cols, cols))
self._drop_temp_table(db, temp_table_name)
db.update_sequence(db.cursor(), 'ticket', 'id')
db.update_sequence(db.cursor(), 'ticket', 'uid')
# IResourceChangeListener methods
def match_resource(self, resource):
return isinstance(resource, Product)
def resource_created(self, resource, context):
import trac.db_default
from multiproduct.env import EnvironmentStub
# Don't populate product database when running from within test
# environment stub as test cases really don't expect that ...
if isinstance(self.env, EnvironmentStub):
return
product = resource
self.log.debug("Adding product info (%s) to tables:" % product.prefix)
with self.env.db_direct_transaction as db:
# create the default entries for this Product from defaults
for table in trac.db_default.get_data(db):
if not table[0] in self.PRODUCT_POPULATE_TABLES:
continue
self.log.debug(" -> %s" % table[0])
cols = table[1] + ('product', )
rows = [p + (product.prefix, ) for p in table[2]]
db.executemany(
"INSERT INTO %s (%s) VALUES (%s)" %
(table[0], ','.join(cols), ','.join(['%s' for c in cols])),
rows)
# Import default pages in product wiki
wikiadmin = WikiAdmin(ProductEnvironment(self.env, product.prefix))
pages = ('TitleIndex', 'RecentChanges', 'InterTrac', 'InterWiki')
for page in pages:
filename = resource_filename('trac.wiki', 'default-pages/' + page)
wikiadmin.import_page(filename, page)
def resource_changed(self, resource, old_values, context):
return
def resource_deleted(self, resource, context):
return
def resource_version_deleted(self, resource, context):
return
# ITemplateProvider methods
def get_templates_dirs(self):
"""provide the plugin templates"""
return [resource_filename(__name__, 'templates')]
def get_htdocs_dirs(self):
"""proved the plugin htdocs"""
return []
# IPermissionRequestor methods
def get_permission_actions(self):
acts = ['PRODUCT_CREATE', 'PRODUCT_DELETE', 'PRODUCT_MODIFY',
'PRODUCT_VIEW']
if not isinstance(self.env, ProductEnvironment):
return acts + [('PRODUCT_ADMIN', acts)] + [('ROADMAP_ADMIN', acts)]
else:
# In product context PRODUCT_ADMIN will be provided by product env
# to ensure it will always be handy
return acts
# ITicketFieldProvider methods
def get_select_fields(self):
"""Product select fields"""
return [(35, {'name': 'product', 'label': _('Product'),
'cls': Product, 'pk': 'prefix', 'optional': False,
'value': self.default_product})]
def get_radio_fields(self):
"""Product radio fields"""
return []
# IResourceManager methods
def get_resource_realms(self):
"""Manage 'product' realm.
"""
yield 'product'
def get_resource_description(self, resource, format='default', context=None,
**kwargs):
"""Describe product resource.
"""
desc = resource.id
if format != 'compact':
desc = _('Product %(name)s', name=resource.id)
if context:
return self._render_link(context, resource.id, desc)
else:
return desc
def resource_exists(self, resource):
"""Check whether product exists physically.
"""
products = Product.select(self.env, where={'name' : resource.id})
return bool(products)
# IExternalResourceConnector methods
def get_supported_neighborhoods(self):
"""Neighborhoods for `product` and `global` environments.
"""
yield 'product'
yield 'global'
def load_manager(self, neighborhood):
"""Load global environment or product environment given its prefix
"""
if neighborhood._realm == 'global':
# FIXME: ResourceNotFound if neighborhood ID != None ?
prefix = GLOBAL_PRODUCT
elif neighborhood._realm == 'product':
prefix = neighborhood._id
else:
raise ResourceNotFound(_(u'Unsupported neighborhood %(realm)s',
realm=neighborhood._realm))
try:
return lookup_product_env(self.env, prefix)
except LookupError:
raise ResourceNotFound(_(u'Unknown product prefix %(prefix)s',
prefix=prefix))
def manager_exists(self, neighborhood):
"""Check whether the target environment exists physically.
"""
if neighborhood._realm == 'global':
# Global environment
return isinstance(self.env, (Environment, ProductEnvironment))
elif neighborhood._realm == 'product':
prefix = neighborhood._id
if not prefix:
# Global environment
return True
return Product(lookup_product_env(self.env, GLOBAL_PRODUCT),
{'prefix' : prefix})._exists
# IWikiSyntaxProvider methods
short_syntax_delimiter = u'->'
def get_wiki_syntax(self):
yield (r'(?<!\S)!?(?P<pid>%s)%s(?P<ptarget>%s:(?:%s)|%s|%s(?:%s*%s)?)' %
(IDENTIFIER,
PRODUCT_SYNTAX_DELIMITER_RE,
WikiParser.LINK_SCHEME, WikiParser.QUOTED_STRING,
WikiParser.QUOTED_STRING, WikiParser.SHREF_TARGET_FIRST,
WikiParser.SHREF_TARGET_MIDDLE, WikiParser.SHREF_TARGET_LAST),
lambda f, m, fm :
self._format_link(f, 'product',
'%s:%s' % (fm.group('pid'),
unquote_label(fm.group('ptarget'))),
fm.group(0), fm))
if self.env[ProductTicketModule] is not None:
yield (r"(?<!\S)!?(?P<jtp>%s)-(?P<jtt>\d+)(?P<jtf>[?#]\S+)?" %
(IDENTIFIER,),
lambda f, m, fm :
self._format_link(f, 'product',
'%s:ticket:%s' %
(fm.group('jtp'),
fm.group('jtt') +
(fm.group('jtf') or '')),
m, fm))
def get_link_resolvers(self):
yield ('global', self._format_link)
yield ('product', self._format_link)
# ITicketManipulator methods
def validate_ticket(self, req, ticket):
# check whether the owner exists in db, add a warning if not
if req.args.get('action') == 'reassign' and \
ticket['owner'] != self.env.config.get('ticket', 'default_owner'):
owner = self.env.db_direct_query(
"SELECT sid FROM session WHERE sid=%s",
(ticket['owner'], ))
if not owner:
# Note: add_warning() is used intead of returning a list of
# error tuples, since the latter results in trac rendering
# errors (ticket's change.date is not populated)
add_warning(req, _('The user "%s" does not exist.') %
ticket['owner'])
return []
# Internal methods
def _render_link(self, context, name, label, extra='', prefix=None):
"""Render link to product page.
"""
product_env = product = None
env = self.env
if isinstance(env, ProductEnvironment):
if (prefix is not None and env.product.prefix == prefix) \
or (prefix is None and env.name == name):
product_env = env
env = env.parent
try:
if product_env is None:
if prefix is not None:
product_env = ProductEnvironment(env, to_unicode(prefix))
else:
product = Product.select(env,
where={'name' : to_unicode(name)})
if not product:
raise LookupError("Missing product")
product_env = ProductEnvironment(env,
to_unicode(product[0]))
except LookupError:
pass
if product_env is not None:
product = product_env.product
href = resolve_product_href(to_env=product_env, at_env=self.env)
if 'PRODUCT_VIEW' in context.perm(product.resource):
return tag.a(label, class_='product', href=href() + extra,
title=product.name)
if 'PRODUCT_CREATE' in context.perm('product', name):
params = [('action', 'new')]
if prefix:
params.append( ('prefix', prefix) )
if name:
params.append( ('name', name) )
return tag.a(label, class_='missing product',
href=env.href('products', params),
rel='nofollow')
return tag.a(label, class_='missing product')
def _format_link(self, formatter, ns, target, label, fullmatch):
link, params, fragment = formatter.split_link(target)
expr = link.split(':', 1)
if ns == 'product' and len(expr) == 1:
# product:prefix form
return self._render_link(formatter.context, None, label,
params + fragment, expr[0])
elif ns == 'global' or (ns == 'product' and expr[0] == ''):
# global scope
sublink = link if ns == 'global' else expr[1]
target_env = self.env.parent \
if isinstance(self.env, ProductEnvironment) \
else self.env
return self._make_sublink(target_env, sublink, formatter, ns,
target, label, fullmatch,
extra=params + fragment)
else:
# product:prefix:realm:id:...
prefix, sublink = expr
try:
target_env = lookup_product_env(self.env, prefix)
except LookupError:
return tag.a(label, class_='missing product')
# TODO: Check for nested product links
# e.g. product:p1:product:p2:ticket:1
return self._make_sublink(target_env, sublink, formatter, ns,
target, label, fullmatch,
extra=params + fragment)
FakePermClass = FakePerm
def _make_sublink(self, env, sublink, formatter, ns, target, label,
fullmatch, extra=''):
parent_match = {'ns' : ns,
'target' : target,
'label': Markup(escape(unescape(label)
if isinstance(label, Markup)
else label)),
'fullmatch' : fullmatch,
}
# Tweak nested context to work in target product/global scope
subctx = formatter.context.child()
subctx.href = resolve_product_href(to_env=env, at_env=self.env)
try:
req = formatter.context.req
except AttributeError:
pass
else:
# Authenticate in local context but use foreign permissions
subctx.perm = self.FakePermClass() \
if isinstance(req.session, FakeSession) \
else PermissionCache(env, req.authname)
subctx.req = req
subformatter = EmbeddedLinkFormatter(env, subctx, parent_match)
subformatter.auto_quote = True
ctxtag = '[%s] ' % (env.product.prefix,) \
if isinstance(env, ProductEnvironment) \
else '<global> '
subformatter.enhance_link = lambda link : (
link(title=ctxtag + link.attrib.get('title'))
if isinstance(link, Element)
and 'title' in link.attrib
else link)
link = subformatter.match(sublink + extra)
if link:
return link
else:
# Return outermost match unchanged like if it was !-escaped
for itype, match in fullmatch.groupdict().items():
if match and not itype in formatter.wikiparser.helper_patterns:
return escape(match)
class ShortcutIconRequestPlugin(Component):
"""Implements the /favicon.ico handler."""
implements(IRequestHandler, IRequestFilter)
iconpath = PathOption('shortcuticon', 'iconpath', None,
"Filesystem path of shortcut icon")
_mimetype = Option('shortcuticon', 'mimetype', None,
"Mimetype of shortcut icon")
ishandler = BoolOption('shortcuticon', 'handler', True,
"Handler for '/favicon.ico'")
isfilter = BoolOption('shortcuticon', 'linkheader', ishandler,
"Add 'link' tags for icon into HTML pages")
path = r'/favicon.ico'
exttypes = {
'.ico': 'image/x-icon',
'.png': 'image/png',
'.jpg': 'image/jpg',
'.gif': 'image/gif',
}
def __init__(self):
if self._mimetype:
self.mimetype = self._mimetype
else:
try:
iconpath = self.iconpath
idx = iconpath.rindex('.', -4)
self.mimetype = self.exttypes[iconpath[idx:]]
self.mimetype = 'image/x-icon'
except:
self.mimetype = 'image/x-icon'
# IRequestHandler methods
def match_request(self, req):
if not self.ishandler:
return False
return req.path_info == self.path \
or req.path_info == req.base_path + self.path
def process_request(self, req):
iconpath = self.iconpath
iconok = False
if iconpath:
if os.path.isfile(iconpath) and os.access(iconpath, R_OK):
iconok = True
else:
self.env.log.warning("Icon '%s' isn't a readable file!" %
iconpath)
else:
self.env.log.warning("No icon file configured!")
if iconok:
req.send_file(self.iconpath, self.mimetype)
else:
req.send_response(404)
req.end_headers()
raise RequestDone
def pre_process_request(self, req, handler):
return handler
def post_process_request(self, req, template, data, content_type):
if self.isfilter:
path = req.base_path + self.path
add_link(req, 'shortcut icon', path, None, self.mimetype)
add_link(req, 'icon', path, None, self.mimetype)
return (template, data, content_type)
class CrashDumpModule(Component):
"""UI for crash dumps."""
implements(IRequestHandler, IRequestFilter, ITemplateStreamFilter,
INavigationContributor, ITemplateProvider, IAdminPanelProvider)
dumpdata_dir = PathOption(
'crashdump',
'dumpdata_dir',
default='../dumpdata',
doc=
'Path to the crash dump data directory relative to the environment conf directory.'
)
crashlink_query = Option(
'crashdump',
'crashlink_query',
default='?status=!closed',
doc="""The base query to be used when linkifying values of ticket
fields. The query is a URL query
string starting with `?` as used in `query:`
[TracQuery#UsingTracLinks Trac links].
(''since 0.12'')""")
nav_url = Option(
'crashdump', 'main_page', 'crash/list',
'The url of the crashes main page to which the trac nav '
'entry should link; if empty, no entry is created in '
'the nav bar. This may be a relative url.')
items_per_page = IntOption(
'crashdump', 'items_per_page', 100,
"""Number of crashes displayed per page in default report,
by default. Set to `0` to specify no limit.
""")
show_delete_crash = BoolOption(
'crashdump',
'show_delete_crash',
'false',
doc="""Show button to delete a crash from the system.""")
crashdump_fields = set(['_crash'])
crashdump_uuid_fields = set(['_crash_uuid'])
crashdump_sysinfo_fields = set(['_crash_sysinfo'])
datetime_fields = set(['crashtime', 'uploadtime', 'reporttime'])
crashdump_link_fields = set(['linked_crash'])
crashdump_ticket_fields = set(['linked_tickets'])
@property
def must_preserve_newlines(self):
return True
# INavigationContributor methods
def get_active_navigation_item(self, req):
self.log.debug('get_active_navigation_item %s' % req.path_info)
if self.nav_url:
if req.path_info == self.nav_url:
return 'crash_list'
def get_navigation_items(self, req):
if self.nav_url:
yield ('mainnav', 'crash_list',
tag.a('Crashes', href=req.href(self.nav_url)))
# ITemplateStreamFilter methods
def filter_stream(self, req, method, filename, stream, data):
if not data:
return stream
# We try all at the same time to maybe catch also changed or processed templates
if filename in [
"report_view.html", "query_results.html", "ticket.html",
"query.html"
]:
# For ticket.html
if 'fields' in data and isinstance(data['fields'], list):
for field in data['fields']:
for f in self.crashdump_fields:
if field['name'] == f and data['ticket'][f]:
field['rendered'] = self._link_crash(
req, data['ticket'][f])
for f in self.crashdump_uuid_fields:
if field['name'] == f and data['ticket'][f]:
field['rendered'] = self._link_crash(
req, data['ticket'][f], show_uuid=True)
for f in self.crashdump_sysinfo_fields:
if field['name'] == f and data['ticket'][f]:
field['rendered'] = self._link_crash(
req,
data['ticket'][f],
show_uuid=True,
sysinfo=True)
for f in self.crashdump_link_fields:
if field['name'] == f and data['ticket'][f]:
field['rendered'] = self._link_crashes_by_id(
req, data['ticket'][f])
# For query_results.html and query.html
if 'groups' in data and isinstance(data['groups'], list):
for group, tickets in data['groups']:
for ticket in tickets:
for f in self.crashdump_fields:
if f in ticket:
ticket[f] = self._link_crash(req, ticket[f])
for f in self.crashdump_uuid_fields:
if f in ticket:
ticket[f] = self._link_crash(req,
ticket[f],
show_uuid=True)
for f in self.crashdump_sysinfo_fields:
if f in ticket:
ticket[f] = self._link_crash(req,
ticket[f],
show_uuid=True,
sysinfo=True)
for f in self.crashdump_ticket_fields:
if f in ticket:
ticket[f] = self._link_tickets(req, ticket[f])
# For report_view.html
if 'row_groups' in data and isinstance(data['row_groups'], list):
#self.log.debug('got row_groups %s' % str(data['row_groups']))
for group, rows in data['row_groups']:
for row in rows:
if 'cell_groups' in row and isinstance(
row['cell_groups'], list):
for cells in row['cell_groups']:
for cell in cells:
# If the user names column in the report differently (blockedby AS "blocked by") then this will not find it
#self.log.debug('got cell header %s' % str(cell.get('header', {}).get('col')))
if cell.get('header', {}).get(
'col') in self.crashdump_fields:
cell['value'] = self._link_crash(
req, cell['value'])
cell['header']['hidden'] = False
cell['header']['title'] = _(
'Crashdump')
elif cell.get(
'header', {}
).get('col') in self.crashdump_uuid_fields:
cell['value'] = self._link_crash(
req, cell['value'], show_uuid=True)
cell['header']['hidden'] = False
cell['header']['title'] = _(
'Crashdump')
elif cell.get('header', {}).get(
'col'
) in self.crashdump_sysinfo_fields:
cell['value'] = self._link_crash(
req,
cell['value'],
show_uuid=True,
sysinfo=True)
cell['header']['hidden'] = False
cell['header']['title'] = _(
'System info')
elif cell.get('header', {}).get(
'col'
) in self.crashdump_ticket_fields:
cell['value'] = self._link_tickets(
req, cell['value'])
cell['header']['hidden'] = False
cell['header']['title'] = _(
'Linked tickets')
elif cell.get(
'header',
{}).get('col') in self.datetime_fields:
cell['value'] = self._format_datetime(
req, cell['value'])
return stream
# ITemplateProvider methods
def get_htdocs_dirs(self):
"""Return the absolute path of a directory containing additional
static resources (such as images, style sheets, etc).
"""
return [('crashdump', _htdoc_dir)]
def get_templates_dirs(self):
return [_template_dir]
# IRequestHandler methods
def match_request(self, req):
if not req.path_info.startswith('/crash'):
return False
ret = False
action = None
path_info = req.path_info[6:]
if path_info == '/list':
action = 'crash_list'
ret = True
else:
#self.log.debug('match_request %s' % path_info)
match = re.match(
r'/([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})/?(.+)?$',
path_info)
if match:
req.args['crashuuid'], action = match.groups()
ret = True
else:
match = re.match(r'/([0-9]+)/?(.+)?$', path_info)
if match:
req.args['crashid'], action = match.groups()
ret = True
if ret:
#self.log.debug('match_request raw_action:\"%s\"' % action)
if action:
e = action.split('/')
#self.log.debug('match_request raw_action->e:\"%s\"' % e)
req.args['action'] = e[0]
req.args['params'] = e[1:] if len(e) > 1 else None
#self.log.debug('match_request action->params:%s->\"%s\"' % (req.args['action'], req.args['params']))
else:
req.args['action'] = None
req.args['params'] = None
self.log.debug('match_request %s -> %s' % (path_info, str(req.args)))
return ret
# IRequestFilter methods
def pre_process_request(self, req, handler):
return handler
def post_process_request(self,
req,
template,
data,
content_type,
method=None):
if req.path_info.startswith('/ticket/'):
# In case of an invalid ticket, the data is invalid
if not data:
return template, data, content_type, method
tkt = data['ticket']
with self.env.db_query as db:
links = CrashDumpTicketLinks(self.env, tkt, db=db)
for change in data.get('changes', {}):
if not change.has_key('fields'):
continue
for field, field_data in change['fields'].iteritems():
if field in self.crashdump_link_fields:
if field_data['new'].strip():
new = set([
CrashDumpSystem.get_crash_id(n)
for n in field_data['new'].split(',')
])
else:
new = set()
if field_data['old'].strip():
old = set([
CrashDumpSystem.get_crash_id(n)
for n in field_data['old'].split(',')
])
else:
old = set()
add = new - old
sub = old - new
elms = tag()
if add:
elms.append(
tag.em(u', '.join(
[unicode(n) for n in sorted(add)])))
elms.append(u' added')
if add and sub:
elms.append(u'; ')
if sub:
elms.append(
tag.em(u', '.join(
[unicode(n) for n in sorted(sub)])))
elms.append(u' removed')
field_data['rendered'] = elms
links.crashes = new
return template, data, content_type, method
def _prepare_data(self, req, crashobj, absurls=False):
data = {
'object': crashobj,
'to_utimestamp': to_utimestamp,
'hex_format': hex_format,
'addr_format': None,
'exception_code': exception_code,
'format_bool_yesno': format_bool_yesno,
'format_source_line': format_source_line,
'format_function_plus_offset': format_function_plus_offset,
'str_or_unknown': str_or_unknown,
'format_cpu_type': format_cpu_type,
'format_cpu_vendor': format_cpu_vendor,
'format_cpu_name': format_cpu_name,
'format_platform_type': format_platform_type,
'format_os_version': format_os_version,
'format_distribution_id': format_distribution_id,
'format_distribution_codename': format_distribution_codename,
'format_milliseconds': format_milliseconds,
'format_seconds': format_seconds,
'format_size': format_size,
'format_trust_level': format_trust_level,
'format_memory_usagetype': format_memory_usagetype,
'format_gl_extension_name': format_gl_extension_name,
'format_version_number': format_version_number,
'format_thread': format_thread,
'thread_extra_info': thread_extra_info,
'format_stack_frame': format_stack_frame,
'context': web_context(req, crashobj.resource, absurls=absurls),
'preserve_newlines': self.must_preserve_newlines,
'emtpy': empty
}
xmlfile = None
xmlfile_from_db = None
minidumpfile = None
coredumpfile = None
if crashobj['minidumpreportxmlfile']:
xmlfile_from_db = crashobj['minidumpreportxmlfile']
xmlfile = self._get_dump_filename(crashobj,
'minidumpreportxmlfile')
reporttextfile = self._get_dump_filename(crashobj,
'minidumpreporttextfile')
reporthtmlfile = self._get_dump_filename(crashobj,
'minidumpreporthtmlfile')
elif crashobj['coredumpreportxmlfile']:
xmlfile_from_db = crashobj['coredumpreportxmlfile']
xmlfile = self._get_dump_filename(crashobj,
'coredumpreportxmlfile')
reporttextfile = self._get_dump_filename(crashobj,
'coredumpreporttextfile')
reporthtmlfile = self._get_dump_filename(crashobj,
'coredumpreporthtmlfile')
coredumpfile = self._get_dump_filename(crashobj, 'coredumpfile')
minidumpfile = self._get_dump_filename(crashobj, 'minidumpfile')
data['xmlfile_from_db'] = xmlfile_from_db
data['xmlfile'] = xmlfile
data['xmlfile_error'] = None
data['minidump_xml_size'] = 0
data['coredump_xml_size'] = 0
data['minidumpfile_size'] = 0
data['coredumpfile_size'] = 0
data['xmlfile_size'] = 0
data['reporttextfile_size'] = 0
data['reporthtmlfile_size'] = 0
data['show_debug_info'] = False
data['parsetime'] = 0
data['is_64_bit'] = False
if xmlfile:
start = time.time()
if minidumpfile:
try:
data['minidumpfile_size'] = os.path.getsize(minidumpfile)
data['minidumpfile'] = MiniDump(minidumpfile)
except OSError:
pass
if coredumpfile:
try:
data['coredumpfile_size'] = os.path.getsize(coredumpfile)
except OSError:
pass
if reporttextfile:
try:
data['reporttextfile_size'] = os.path.getsize(
reporttextfile)
except OSError:
pass
if reporthtmlfile:
try:
data['reporthtmlfile_size'] = os.path.getsize(
reporthtmlfile)
except OSError:
pass
if xmlfile:
try:
data['xmlfile_size'] = os.path.getsize(xmlfile)
except OSError:
pass
if os.path.isfile(xmlfile):
try:
xmlreport = XMLReport(xmlfile)
for f in xmlreport.fields:
data[f] = XMLReport.ProxyObject(xmlreport, f)
data['xmlreport'] = xmlreport
data['is_64_bit'] = xmlreport.is_64_bit
except XMLReport.XMLReportIOError as e:
data['xmlfile_error'] = str(e)
else:
wrapper = MiniDumpWrapper(data['minidumpfile'])
for f in wrapper.fields:
data[f] = MiniDumpWrapper.ProxyObject(wrapper, f)
data['xmlreport'] = None
data['xmlfile_error'] = 'XML file %s does not exist' % xmlfile
end = time.time()
data['parsetime'] = end - start
data['bits'] = 64 if data['is_64_bit'] else 32
data['addr_format'] = addr_format_64 if data[
'is_64_bit'] else addr_format_32
return data
def _get_prefs(self, req):
return {
'comments_order': req.session.get('ticket_comments_order',
'oldest'),
'comments_only': req.session.get('ticket_comments_only', 'false')
}
def process_request(self, req):
if crashdump_use_jinja2:
metadata = {'content_type': 'text/html'}
else:
metadata = None
action = req.args.get('action', 'view')
if action == 'crash_list':
page = req.args.getint('page', 1)
default_max = self.items_per_page
max = req.args.getint('max')
limit = as_int(max, default_max,
min=0) # explict max takes precedence
offset = (page - 1) * limit
sort_col = req.args.get('sort', '')
asc = req.args.getint('asc', 0, min=0, max=1)
title = ''
description = ''
data = {
'action': 'crash_list',
'max': limit,
'numrows': 0,
'title': title,
'description': description,
'message': None,
'paginator': None
}
req_status = req.args.get('status') or 'active'
#results = CrashDump.query(env=self.env, status=req_status)
results = CrashDump.query(env=self.env, status=None)
data['results'] = results
limit_offset = 0
need_paginator = limit > 0 and limit_offset
need_reorder = limit_offset is None
numrows = len(results)
paginator = None
if need_paginator:
paginator = Paginator(results, page - 1, limit, num_items)
data['paginator'] = paginator
if paginator.has_next_page:
add_link(req, 'next', report_href(page=page + 1),
_('Next Page'))
if paginator.has_previous_page:
add_link(req, 'prev', report_href(page=page - 1),
_('Previous Page'))
pagedata = []
shown_pages = paginator.get_shown_pages(21)
for p in shown_pages:
pagedata.append([
report_href(page=p), None,
str(p),
_('Page %(num)d', num=p)
])
fields = ['href', 'class', 'string', 'title']
paginator.shown_pages = [
dict(zip(fields, p)) for p in pagedata
]
paginator.current_page = {
'href': None,
'class': 'current',
'string': str(paginator.page + 1),
'title': None
}
numrows = paginator.num_items
data['paginator'] = paginator
add_script_data(req, {'comments_prefs': self._get_prefs(req)})
if not crashdump_use_jinja2:
add_script(req, 'common/js/folding.js')
add_script(req, 'crashdump/crashdump.js')
add_stylesheet(req, 'crashdump/crashdump.css')
return 'list.html', data, metadata
start = time.time()
if 'crashuuid' in req.args:
crashobj = CrashDump.find_by_uuid(self.env, req.args['crashuuid'])
if not crashobj:
raise ResourceNotFound(
_("Crash %(id)s does not exist.",
id=req.args['crashuuid']), _("Invalid crash identifier"))
elif 'crashid' in req.args:
crashobj = CrashDump.find_by_id(self.env, req.args['crashid'])
if not crashobj:
raise ResourceNotFound(
_("Crash %(id)s does not exist.", id=req.args['crashid']),
_("Invalid crash identifier"))
else:
raise ResourceNotFound(_("No crash identifier specified."))
end = time.time()
xhr = req.get_header('X-Requested-With') == 'XMLHttpRequest'
#req.perm('crash', id, version).require('TICKET_VIEW')
params = _get_list_from_args(req.args, 'params', None)
self.log.debug('process_request %s:%s-%s' %
(action, type(params), params))
if action is None or action == 'view':
data = self._prepare_data(req, crashobj)
xmlfile = data['xmlfile'] if 'xmlfile' in data else None
data['dbtime'] = end - start
field_changes = {}
data.update({
'action': action,
'params': params,
# Store a timestamp for detecting "mid air collisions"
'start_time': crashobj['changetime']
})
self._insert_crashdump_data(req, crashobj, data,
get_reporter_id(req, 'author'),
field_changes)
if params is None:
add_script_data(req, {'comments_prefs': self._get_prefs(req)})
if not crashdump_use_jinja2:
add_script(req, 'common/js/folding.js')
add_script(req, 'crashdump/crashdump.js')
add_stylesheet(req, 'crashdump/crashdump.css')
data['show_delete_crash'] = self.show_delete_crash
linked_tickets = []
for tkt_id in crashobj.linked_tickets:
a = self._link_ticket_by_id(req, tkt_id)
if a:
linked_tickets.append(a)
data['linked_tickets'] = linked_tickets
return 'report.html', data, metadata
else:
if params[0] in [
'sysinfo', 'sysinfo_ex', 'fast_protect_version_info',
'exception', 'memory_blocks', 'memory_regions',
'modules', 'threads', 'stackdumps', 'file_info'
]:
return params[0] + '.html', data, metadata
elif params[0] == 'memory_block':
block_base = safe_list_get_as_int(params, 1, 0)
memory_block = None
for b in data['memory_blocks']:
if b.base == block_base:
memory_block = b
break
data.update({
'memory_block': memory_block,
'memory_block_base': block_base
})
return 'memory_block.html', data, metadata
elif params[0] == 'stackdump':
threadid = safe_list_get_as_int(params, 1, 0)
stackdump = None
if threadid in data['stackdumps']:
stackdump = data['stackdumps'][threadid]
self.log.debug('stackdump %s' % stackdump)
data.update({'stackdump': stackdump, 'threadid': threadid})
return 'stackdump.html', data, metadata
else:
raise ResourceNotFound(
_("Invalid sub-page request %(param)s for crash %(uuid)s.",
param=str(params[0]),
uuid=str(crashobj.uuid)))
elif action == 'sysinfo_report':
data = self._prepare_data(req, crashobj)
data['dbtime'] = end - start
if 'xmlreport' in data:
xmlfile = data['xmlreport']
data['sysinfo_report'] = None
if isinstance(xmlfile,
XMLReport) or (isinstance(xmlfile, string)
and os.path.isfile(xmlfile)):
try:
data['sysinfo_report'] = SystemInfoReport(
xmlreport=xmlfile)
except SystemInfoReport.SystemInfoReportException as e:
data['xmlfile_error'] = str(e)
else:
data['xmlfile_error'] = _(
"XML file %(file)s is unavailable", file=xmlfile)
data.update({
'action': action,
'params': params,
# Store a timestamp for detecting "mid air collisions"
'start_time': crashobj['changetime']
})
if params is None:
add_script_data(req, {'comments_prefs': self._get_prefs(req)})
if not crashdump_use_jinja2:
add_script(req, 'common/js/folding.js')
add_script(req, 'crashdump/crashdump.js')
add_stylesheet(req, 'crashdump/crashdump.css')
linked_tickets = []
for tkt_id in crashobj.linked_tickets:
a = self._link_ticket_by_id(req, tkt_id)
if a:
linked_tickets.append(a)
data['linked_tickets'] = linked_tickets
return 'sysinfo_report.html', data, metadata
else:
if params[0] in [
'sysinfo', 'sysinfo_ex', 'sysinfo_opengl',
'sysinfo_env', 'sysinfo_terra4d_dirs', 'sysinfo_cpu',
'sysinfo_locale', 'sysinfo_network', 'sysinfo_rawdata'
]:
return params[0] + '.html', data, metadata
else:
raise ResourceNotFound(
_("Invalid sub-page request %(param)s for crash %(uuid)s.",
param=str(params[0]),
uuid=str(crashobj.uuid)))
elif action == 'systeminfo_raw':
data = self._prepare_data(req, crashobj)
xmlfile = data['xmlfile'] if 'xmlfile' in data else None
data['dbtime'] = end - start
fast_protect_system_info = data[
'fast_protect_system_info'] if 'fast_protect_system_info' in data else None
if fast_protect_system_info:
if crashobj['crashhostname']:
filename = "%s_%s.terra4d-system-info" % (str(
crashobj.uuid), str(crashobj['crashhostname']))
else:
filename = "%s.terra4d-system-info" % str(crashobj.uuid)
if fast_protect_system_info.rawdata:
return self._send_data(
req,
fast_protect_system_info.rawdata.raw,
filename=filename)
raise ResourceNotFound(
_("No system information available for crash %(uuid)s.",
uuid=str(crashobj.uuid)))
elif action == 'delete':
add_script_data(req, {'comments_prefs': self._get_prefs(req)})
add_script(req, 'crashdump/crashdump.js')
add_stylesheet(req, 'crashdump/crashdump.css')
data = {'id': crashobj.id, 'uuid': crashobj.uuid}
crashobj.delete(self.dumpdata_dir)
return 'deleted.html', data, metadata
elif action == 'minidump_raw':
return self._send_file(req, crashobj, 'minidumpfile')
elif action == 'minidump_text':
return self._send_file(req, crashobj, 'minidumpreporttextfile')
elif action == 'minidump_xml':
return self._send_file(req, crashobj, 'minidumpreportxmlfile')
elif action == 'minidump_html':
return self._send_file(req, crashobj, 'minidumpreporthtmlfile')
elif action == 'coredump_raw':
return self._send_file(req, crashobj, 'coredumpfile')
elif action == 'coredump_text':
return self._send_file(req, crashobj, 'coredumpreporttextfile')
elif action == 'coredump_xml':
return self._send_file(req, crashobj, 'coredumpreportxmlfile')
elif action == 'coredump_html':
return self._send_file(req, crashobj, 'coredumpreporthtmlfile')
elif action == 'raw':
if crashobj['minidumpfile']:
return self._send_file(req, crashobj, 'minidumpfile')
elif crashobj['coredumpfile']:
return self._send_file(req, crashobj, 'coredumpfile')
elif action == 'xml':
if crashobj['minidumpreportxmlfile']:
return self._send_file(req, crashobj, 'minidumpreportxmlfile')
elif crashobj['coredumpreportxmlfile']:
return self._send_file(req, crashobj, 'coredumpreportxmlfile')
elif action == 'html':
if crashobj['minidumpreporthtmlfile']:
return self._send_file(req, crashobj, 'minidumpreporthtmlfile')
elif crashobj['coredumpreporthtmlfile']:
return self._send_file(req, crashobj, 'coredumpreporthtmlfile')
elif action == 'text':
if crashobj['minidumpreporttextfile']:
return self._send_file(req, crashobj, 'minidumpreporttextfile')
elif crashobj['coredumpreporttextfile']:
return self._send_file(req, crashobj, 'coredumpreporttextfile')
raise ResourceNotFound(
_("Invalid action %(action)s for crash %(uuid)s specified.",
action=str(action),
uuid=str(crashobj.uuid)))
def _send_data(self, req, data, filename):
# Force browser to download files instead of rendering
# them, since they might contain malicious code enabling
# XSS attacks
req.send_header('Content-Disposition',
'attachment; filename=%s' % filename)
req.send_header('Content-Length', '%i' % len(data))
req.send(content=data,
content_type='application/force-download',
status=200)
def _send_file(self, req, crashobj, name):
filename = self._get_dump_filename(crashobj, name)
item_name = os.path.basename(filename)
# Force browser to download files instead of rendering
# them, since they might contain malicious code enabling
# XSS attacks
req.send_header('Content-Disposition',
'attachment; filename=%s' % item_name)
req.send_file(filename, mimetype='application/force-download')
def _query_link(self, req, name, value, text=None):
"""Return a link to /query with the appropriate name and value"""
default_query = self.crashlink_query.lstrip('?')
args = arg_list_to_args(parse_arg_list(default_query))
args[name] = value
if name == 'resolution':
args['status'] = 'closed'
return tag.a(text or value, href=req.href.query(args))
def _insert_crashdump_data(self, req, crashobj, data, author_id,
field_changes):
"""Insert crashobj data into the template `data`"""
replyto = req.args.get('replyto')
data['replyto'] = replyto
data['version'] = crashobj.resource.version
data['description_change'] = None
data['author_id'] = author_id
if crashobj.resource.version is not None:
crashobj.values.update(values)
context = web_context(req, crashobj.resource)
# Display the owner and reporter links when not obfuscated
chrome = Chrome(self.env)
for user in 'reporter', 'owner':
if chrome.format_author(req, crashobj[user]) == crashobj[user]:
data['%s_link' % user] = self._query_link(
req, user, crashobj[user])
data['context'] = context
def _format_datetime(self, req, timestamp):
try:
utimestamp = long(timestamp)
return format_datetime(from_utimestamp(utimestamp))
except ValueError:
return str(timestamp)
def _get_dump_filename(self, crashobj, name):
item_name = crashobj[name]
if not item_name:
return None
crash_file = os.path.join(self.env.path, self.dumpdata_dir, item_name)
return crash_file
def _link_ticket_by_id(self, req, ticketid):
ret = None
try:
ticket = Ticket(self.env, ticketid)
if 'TICKET_VIEW' in req.perm(ticket.resource):
ret = \
tag.a(
'#%s' % ticket.id,
class_=ticket['status'],
href=req.href.ticket(int(ticket.id)),
title=shorten_line(ticket['summary'])
)
except ResourceNotFound:
pass
return ret
def _link_tickets(self, req, tickets):
if tickets is None:
return None
if not isinstance(tickets, str) and not isinstance(tickets, unicode):
self.log.debug('_link_tickets %s invalid type (%s)' %
(tickets, type(tickets)))
return None
if not tickets:
return None
items = []
for i, word in enumerate(re.split(r'([;,\s]+)', tickets)):
if i % 2:
items.append(word)
elif word:
ticketid = word
word = '#%s' % word
try:
ticket = Ticket(self.env, ticketid)
if 'TICKET_VIEW' in req.perm(ticket.resource):
word = \
tag.a(
'#%s' % ticket.id,
class_=ticket['status'],
href=req.href.ticket(int(ticket.id)),
title=shorten_line(ticket['summary'])
)
except ResourceNotFound:
pass
items.append(word)
if items:
return tag(items)
else:
return None
def _link_crash_by_id(self, req, id):
ret = None
try:
crash = CrashDump(env=self.env, id=id)
ret = \
tag.a(
'CrashId#%i' % crash.id,
class_=crash['status'],
href=req.href('crash', crash.uuid),
title=crash.uuid
)
except ResourceNotFound:
pass
return ret
def _link_crashes_by_id(self, req, ids):
items = []
for i, word in enumerate(re.split(r'([;,\s]+)', ids)):
if i % 2:
items.append(word)
elif word:
crashid = word
word = 'CrashId#%s' % word
try:
crash = CrashDump(env=self.env, id=crashid)
word = \
tag.a(
'CrashId#%i' % crash.id,
class_=crash['status'],
href=req.href('crash', crash.uuid),
title=crash.uuid
)
except ResourceNotFound:
pass
items.append(word)
if items:
return tag(items)
else:
return None
def _link_crash(self, req, uuid, show_uuid=False, sysinfo=False):
ret = None
try:
crash = CrashDump(env=self.env, uuid=uuid)
if sysinfo:
href = req.href('crash', crash.uuid, 'sysinfo_report')
title = 'CrashId#%i (%s)' % (crash.id, crash.uuid)
else:
if show_uuid:
title = str(crash.uuid)
else:
title = 'CrashId#%i (%s)' % (crash.id, crash.uuid)
href = req.href('crash', crash.uuid)
if show_uuid:
ret = \
tag.a(
str(crash.uuid),
class_=crash['status'],
href=href,
title=title,
style="white-space: nowrap"
)
else:
ret = \
tag.a(
'CrashId#%i' % crash.id,
class_=crash['status'],
href=href,
title=crash.uuid
)
except ResourceNotFound:
pass
return ret
# IAdminPanelProvider methods
def get_admin_panels(self, req):
if req.perm.has_permission('TRAC_ADMIN'):
yield ('crashdump', 'Crash dump', 'maintenance', 'Maintenance')
def render_admin_panel(self, req, cat, page, path_info):
assert req.perm.has_permission('TRAC_ADMIN')
action = req.args.get('action', 'view')
if req.method == 'POST':
confirm = req.args.get('confirm', 0)
if 'purge_threshold' in req.args:
purge_threshold_str = req.args.get('purge_threshold', '')
purge_threshold = user_time(req, parse_date, purge_threshold_str, hint='datetime') \
if purge_threshold_str else None
else:
purge_threshold = None
if not confirm:
self.log.debug('render_admin_panel purge not yet confirmed')
if purge_threshold is not None:
crashes = CrashDump.query_old_data(self.env,
purge_threshold)
data = {
'datetime_hint': get_datetime_format_hint(req.lc_time),
'purge_threshold': purge_threshold_str,
'purge_crashes': crashes
}
return 'crashdump_admin_%s.html' % page, data
elif confirm == 'no':
self.log.debug('render_admin_panel purge canceled')
req.redirect(req.href.admin(cat, page))
elif confirm == 'yes':
self.log.debug('render_admin_panel purge confirmed')
req.redirect(req.href.admin(cat, page))
else:
now = datetime.now(req.tz)
purge_threshold = datetime(now.year, now.month, now.day, 0)
purge_threshold -= timedelta(days=365)
data = {
'datetime_hint': get_datetime_format_hint(req.lc_time),
'purge_threshold': purge_threshold,
'purge_crashes': None,
}
print('crashdump_admin_%s.html' % page, data)
return 'crashdump_admin_%s.html' % page, data
yield ('sha', self._format_sha_link)
#######################
# IRepositoryConnector
_persistent_cache = BoolOption('git', 'persistent_cache', 'false',
"enable persistent caching of commit tree")
_cached_repository = BoolOption('git', 'cached_repository', 'false',
"wrap `GitRepository` in `CachedRepository`")
_shortrev_len = IntOption('git', 'shortrev_len', 7,
"length rev sha sums should be tried to be abbreviated to"
" (must be >= 4 and <= 40)")
_git_bin = PathOption('git', 'git_bin', '/usr/bin/git', "path to git executable (relative to trac project folder!)")
def get_supported_types(self):
yield ("git", 8)
def get_repository(self, type, dir, authname):
"""GitRepository factory method"""
assert type == "git"
if not self._version:
raise TracError("GIT backend not available")
elif not self._version['v_compatible']:
raise TracError("GIT version %s installed not compatible (need >= %s)" %
(self._version['v_str'], self._version['v_min_str']))
class PatchMan(Component):
implements(ITemplateProvider)
_git_bin = PathOption('git', 'git_bin', '/usr/bin/git',
"path to git executable")
APPLIED = "APPLIED", "green"
PENDING = "PENDING", "orange"
REJECTED = "REJECTED", "red"
RECIPIENTS = [
"*****@*****.**", "*****@*****.**"
]
DIFFERENTIAL_URL_KEY = "Differential Revision"
def getPatchPath(self, patchId):
filename = "patch" + str(patchId)
filepath = self.env.path + "/attachments/PatchManager/0.2/" + filename
return filepath
def addNewPatch(self,
FromAddr,
Subject,
Branch,
Date,
FileContent,
DifferentialURL=""):
db = self.env.get_db_cnx()
cursor = db.cursor()
cursor.execute(
"INSERT INTO Patches (email, subject, branch, commit_time, submit_time, rejected, differential_url)\
VALUES (%s, %s, %s, %s, %s, %s, %s) ",
(FromAddr, Subject, Branch, Date, datetime.today(), "0",
DifferentialURL))
id = db.get_last_id(cursor, "Patches")
filepath = self.getPatchPath(id)
flags = os.O_CREAT + os.O_WRONLY + os.O_EXCL
if hasattr(os, 'O_BINARY'):
flags += os.O_BINARY
targetfile = os.fdopen(os.open(filepath, flags, 0666), 'w')
# file = Attachment(self.env, 'PatchManager', '0.2')
# tmp_file = Attachment(self.env, file.resource.parent)
# self.env.log.debug("attachment path: " + str(file.path) + ", " + str(attachment.path))
# FileContent.file.seek(0, 2) # seek to end of file
# size = FileContent.file.tell()
FileContent.file.seek(0)
# file.insert(filename, FileContent.file, size)
with targetfile:
shutil.copyfileobj(FileContent.file, targetfile)
db.commit()
jenkins_url = self.config.get('jenkins', 'url')
jenkins_user = self.config.get('jenkins', 'user')
jenkins_passwd = self.config.get('jenkins', 'passwd')
job_name = self.config.get('jenkins', 'patch_test_job_name')
jenkins = JenkinsApi(jenkins_url, jenkins_user, jenkins_passwd,
self.env.log)
if self.is_jenkins_enabled():
jenkins.test_patch(id, job_name, Branch)
for r in self.RECIPIENTS:
self.sendMail(
r, "[PATCH] <%s> %s" %
(self.cutString(FromAddr, 15), self.cutString(Subject, 40)),
"New patch submitted to rasdaman.org:\n\nSubject\t %s\nFrom\t %s\nDate\t %s"
% (Subject, FromAddr, Date), None)
return
def cutString(self, s, n):
if len(s) > n:
return s[:n] + "..."
else:
return s
# http://kutuma.blogspot.com/2007/08/sending-emails-via-gmail-with-python.html
def sendMail(self, to, subject, text, attach):
gmailUser = self.config.get('gmail', 'user')
gmailPassword = self.config.get('gmail', 'password')
msg = MIMEMultipart()
msg['From'] = gmailUser
msg['To'] = to
msg['Subject'] = subject
msg.attach(MIMEText(text))
if attach != None:
part = MIMEBase('application', 'octet-stream')
part.set_payload(open(attach, 'rb').read())
Encoders.encode_base64(part)
part.add_header(
'Content-Disposition',
'attachment; filename="%s"' % os.path.basename(attach))
msg.attach(part)
try:
mailServer = smtplib.SMTP("smtp.gmail.com", 587)
mailServer.ehlo()
mailServer.starttls()
mailServer.ehlo()
mailServer.login(gmailUser, gmailPassword)
mailServer.sendmail(gmailUser, to, msg.as_string())
# Should be mailServer.quit(), but that crashes...
mailServer.close()
except:
self.env.log.debug("failed sending email to " + to)
def sendPatchStatusMail(self, id, patchStatus):
"""Send an email to the submitter of a patch with the given id. The
subject is of the form 'rasdaman.org: PATCH patchStatus'"""
db = self.env.get_db_cnx()
cursor = db.cursor()
cursor.execute(
"SELECT email, subject, submit_time FROM Patches WHERE id='" +
str(id) + "'")
for email, subject, submit_time in cursor:
# self.env.log.debug("in sendPatchStatusMail: " + email + ' ' + subject + ' ' + submit_time)
if email and subject and submit_time:
self.sendMail(
email, "rasdaman.org: PATCH " + patchStatus,
"Your patch submitted to rasdaman.org on " +
str(submit_time) + " has been " + patchStatus +
"\n\nPatch description:\n" + str(subject), None)
def checkPatchStatus(self, repo_patches, email, subject, branch):
if email is None or subject is None:
return self.PENDING
r = re.compile(r'\[PATCH[^\]]*\]( *\[[^\]]*\])* *(.*)', re.DOTALL)
match = r.match(subject)
if not match is None:
subject = match.group(2)
key = self.getPatchUID(email, subject, branch)
if (not repo_patches.has_key(key)):
key = self.getPatchUID(email, subject, 'master')
if (repo_patches.has_key(key)):
return self.APPLIED
else:
return self.PENDING
def getPatchUID(self, author, message, branch):
author = author.replace('"', '')
return self._deleteExtraWhiteSpace(author + ":" + message + " (" +
branch + ")").strip()
def listBranches(self):
results = deque()
repDir = self.config.get('trac', 'repository_dir')
gitBin = self.config.get('git', 'git_bin')
#REPO should be repDir if everything is fine with trac.ini
REPO = repDir
output = self.getSystemOutput('cd ' + REPO + '; git branch')
res = ""
results.append({'branch': 'master'})
for branch in output:
temp = branch[2:-1]
if temp != 'master':
results.append({'branch': temp})
return results
def listPatches(self, req, applied, pending, rejected, offset=0, count=10):
result = deque()
repo_patches = dict()
subject_patches = dict()
repo = self.env.get_repository()
rev = repo.get_youngest_rev()
while (not (rev is None)):
t = repo.get_changeset(rev)
properties = t.get_properties().get("git-author")
branch = ""
self.env.log.debug("branches: %s" % t.get_branches())
for tmp in t.get_branches():
branch, head = tmp
msg = t.message
end = msg.find("\n")
if end != -1:
msg = msg[:end]
self.env.log.debug(
"branch: %s\n author: %s\n properties: %s\n msg: %s\n" %
(branch, t.author, properties, msg))
if (properties is None):
key = self.getPatchUID(t.author, msg, branch)
else:
key = self.getPatchUID(properties[0], msg, branch)
rev = repo.previous_rev(rev)
repo_patches.update([(key, True)])
subject_patches.update([(key, t.message)])
db = self.env.get_db_cnx()
cursor = db.cursor()
cursor.execute(
"SELECT id, email, subject, branch, commit_time, submit_time, rejected, test_status, test_url, differential_url FROM Patches ORDER BY submit_time DESC LIMIT %s OFFSET %s",
(count, offset))
jenkins_url = self.config.get('jenkins', 'url')
jenkins_user = self.config.get('jenkins', 'user')
jenkins_passwd = self.config.get('jenkins', 'passwd')
jenkins = JenkinsApi(jenkins_url, jenkins_user, jenkins_passwd,
self.env.log)
job_name = self.config.get('jenkins', 'patch_test_job_name')
test_status = {}
if self.is_jenkins_enabled():
test_status = jenkins.get_test_patch_map(job_name)
self.env.log.debug(test_status)
update_cursor = db.cursor()
refreshPage = False
for id, email, subject, branch, commit_time, submit_time, rejected_att, test, test_url, differential_url in cursor:
if not branch:
branch = 'master'
if rejected_att == 1:
status = self.REJECTED
else:
status = self.checkPatchStatus(repo_patches, email, subject,
branch)
if not applied is None and status != self.APPLIED:
continue
if not pending is None and status != self.PENDING:
continue
if not rejected is None and status != self.REJECTED:
continue
ind = submit_time.find(".")
if ind != -1:
submit_time = submit_time[:ind]
r = re.compile(r'\[PATCH[^\]]*\] +(.*)', re.DOTALL)
match = r.match(subject)
if not match is None:
tmp = match.group(1)
else:
tmp = subject
key = self.getPatchUID(email, tmp, branch)
if (not subject_patches.has_key(key)):
key = self.getPatchUID(email, tmp, 'master')
if (subject_patches.has_key(key)):
subj = subject_patches[key]
else:
subj = subject
if (status == self.PENDING) and (id in test_status):
refreshPage |= bool(test_status[id]['building'])
if (not bool(test_status[id]['building'])) and (test is None):
test = test_status[id]['buildStatus']
test_url = test_status[id]['url']
update_cursor.execute(
"UPDATE Patches SET test_status=%s WHERE id=%s",
(test, id))
update_cursor.execute(
"UPDATE Patches SET test_url=%s WHERE id=%s",
(test_url, id))
test_status_color = 'red'
if test == 'SUCCESS':
test_status_color = 'green'
result.append({
'id': id,
'email': self.encodeEmail(email, req),
'subject': subj,
'branch': branch,
'commit_time': commit_time,
'submit_time': submit_time,
'status': status[0],
'status_color': status[1],
'test': test,
'test_url': test_url,
'test_status_color': test_status_color,
'differential_url': differential_url
})
db.commit()
return result, refreshPage
def encodeEmail(self, email, req):
result = ""
if email is None:
return result
if 'anonymous' == req.perm.username:
ind = email.find("<")
if ind != -1:
email = email[:ind]
for c in email:
result += chr(ord(c) + 1)
return result
def patchCount(self):
db = self.env.get_db_cnx()
cursor = db.cursor()
cursor.execute("SELECT count(id) as cnt FROM Patches")
result = cursor.fetchone()[0]
return result
def _getAgreementText(self):
path = self.get_htdocs_dirs()
f = open(path[0][1] + '/agreement.txt')
result = f.read()
f.close()
return result
# ITemplateProvider methods
def get_templates_dirs(self):
from pkg_resources import resource_filename
return [resource_filename(__name__, 'templates')]
def get_htdocs_dirs(self):
from pkg_resources import resource_filename
return [('patchmanager', resource_filename(__name__, 'htdocs'))]
def _getPatchDetails(self, emailContent):
if (type(emailContent) == str):
msg = PatchModel(emailContent)
else:
msg = PatchModel(emailContent.file.read())
return {
'From': msg.getFrom(),
'Subject': msg.getSubject(),
'Date': msg.getDate(),
self.DIFFERENTIAL_URL_KEY: msg.getReviewUrl()
}
def _getBundleDetails(self, bundle, selectbranch):
_bundle_file = bundle.file
_bundle_file.seek(0)
_rep_dir = self.config.get('trac', 'repository_dir')
_git_bin = self.config.get('git', 'git_bin')
_cmd = ""
# Temporarily store the bundle file
_temp_bundle_file = tempfile.NamedTemporaryFile()
_temp_bundle_file.write(_bundle_file.read())
_temp_bundle_file.seek(0)
_bundle_path = _temp_bundle_file.name
# DEBUG
# _bundle_file.seek(0)
# self.env.log.debug("BUNDLE FILE: " + _bundle_file.read(500))
# self.env.log.debug("TEMP BUNDLE FILE: " + _temp_bundle_file.read(500))
# Clone the repo for testing
_temp_dir = tempfile.mkdtemp()
_cmd = "cd " + _temp_dir + "; " + _git_bin + " clone -b " + selectbranch + " " + _rep_dir + " " + _temp_dir
self.env.log.debug("terminal call: " + _cmd)
os.system(_cmd)
# Check there is a single commit inside the bundle
_cmd = "cd " + _temp_dir + "; test $( " + _git_bin + " bundle list-heads " + _bundle_path + " | wc -l ) = 1"
self.env.log.debug("terminal call:" + _cmd)
if os.system(_cmd) != os.EX_OK:
raise TracError(
"The uploaded bundle contains more than one commit.")
# _temp_bundle_file.seek(0)
# Need to apply the bundle and fetch the details there (From, To, Date, etc.)
# checkout to a tmp branch (otherwise git-fetch won't apply)
_cmd = "cd " + _temp_dir + "; " + _git_bin + " checkout -b tmp "
self.env.log.debug(("terminal call: " + _cmd + " 2>&1"))
os.system(_cmd)
# apply the bundle
_cmd = "cd " + _temp_dir + "; " + _git_bin + " fetch " + _bundle_path + " " + selectbranch + ":" + selectbranch
self.env.log.debug(("terminal call: " + _cmd + " 2>&1"))
messages = self.getSystemOutput(_cmd + " 2>&1")
self.env.log.debug(self.toString(messages))
# patch the bundle's commit and extract metadata
_cmd = "cd " + _temp_dir + "; " + _git_bin + " show " + selectbranch + " HEAD --pretty=email"
self.env.log.debug(("terminal call: " + _cmd + " 2>&1"))
messages = self.getSystemOutput(_cmd + " 2>&1")
_bundle_patch = self.toString(messages)
# self.env.log.debug("Bundle diffs: " + _bundle_patch)
# cleaning
_temp_bundle_file.close()
os.system("rm -r " + _temp_dir)
return self._getPatchDetails(_bundle_patch)
def _deleteExtraWhiteSpace(self, string1):
result = StringIO.StringIO()
lastSpace = False
for i in range(len(string1)):
if (not (string1[i] in string.whitespace)):
result.write(string1[i])
elif (not lastSpace):
result.write(" ")
lastSpace = string1[i] in string.whitespace
return result.getvalue()
def getAttachment(self, id):
try:
return Attachment(self.env, 'PatchManager', '0.2',
"patch" + str(id))
except:
pass
try:
return Attachment(self.env, 'PatchManager', '0.1',
"patch" + str(id))
except:
pass
return None
def processUploadPatch(self, req):
if (not 'agree' in req.args or req.args['agree'] != 'on'):
raise TracError('Please accept the license agreement')
if (not 'agreement' in req.args):
raise TracError('Please accept the license agreement')
uploadedAgreement = self._deleteExtraWhiteSpace(req.args['agreement'])
localAgreement = self._deleteExtraWhiteSpace(self._getAgreementText())
if (uploadedAgreement != localAgreement):
raise TracError(
'Uploaded agreement differs from our local agreement. Please contact admin!'
)
file_name = self.getFileName(req.args['patchfile'])
self.env.log.debug("Name of the uploaded file: " + file_name)
selectbranch = req.args["selectbranch"]
data = {}
if re.match(r'.*\.patch', file_name):
data.update(self._getPatchDetails(req.args["patchfile"]))
elif re.match(r'.*\.bundle', file_name):
data.update(
self._getBundleDetails(req.args["patchfile"], selectbranch))
else:
raise TracError(
"Please upload a file with either .patch or .bundle extension."
)
if data['From'] is None:
raise TracError("The patch doesn't have any author credentials")
if data['Subject'] is None:
raise TracError("The patch doesn't have any description")
subject = data['Subject']
r = re.compile(r'\[PATCH[^\]]*\] +ticket:[0-9]+ .+', re.DOTALL)
match = r.match(subject)
if match is None and selectbranch == 'master':
raise TracError(
"The subject of the patch is invalid; Please edit it so that the subject starts with ticket:NUMBER, where NUMBER is a valid ticket number on the rasdaman.org tracker."
)
self.env.log.debug("A new patch was submitted! Getting details...")
self.addNewPatch(data['From'], data['Subject'],
req.args["selectbranch"], data['Date'],
req.args["patchfile"],
data[self.DIFFERENTIAL_URL_KEY])
return {'page': 'addpatchdetails.html', 'data': data}
def automatic_test_results(self):
jenkins_url = self.config.get('jenkins', 'url')
jenkins_user = self.config.get('jenkins', 'user')
jenkins_passwd = self.config.get('jenkins', 'passwd')
job_name = self.config.get('jenkins', 'automatic_test_job_name')
jenkins = JenkinsApi(jenkins_url, jenkins_user, jenkins_passwd,
self.env.log)
tests = jenkins.get_automatic_build_map(job_name)
keys = sorted(tests.keys(), reverse=True)
result = list()
for k in keys:
result.append(tests[k])
return result
def processDeletePatch(self, req):
if not 'TRAC_ADMIN' in req.perm:
raise TracError(
'You do not have enough priviledges to delete a patch!')
db = self.env.get_db_cnx()
cursor = db.cursor()
for id in self.getIDs(req.args['select']):
self.sendPatchStatusMail(id, "DELETED")
cursor.execute("DELETE FROM Patches WHERE id=" + str(id))
patchpath = self.getPatchPath(id)
os.remove(patchpath)
db.commit()
def processDownloadPatch(self, req):
tempDir = tempfile.mkdtemp("patch")
path = os.path.join(self.env.path, 'attachments', 'PatchManager',
'0.2')
cmd = "cd " + tempDir + "; tar -czf archive.tgz -C " + path + " "
for id in self.getIDs(req.args['select']):
cmd += "patch" + str(id) + " "
self.env.log.debug("executing " + cmd)
os.system(cmd)
req.send_header('Content-Disposition',
'attachment; filename="patches.tgz"')
req.send_file(tempDir + "/archive.tgz", 'application/x-tar-gz')
def getIDs(self, args):
if type(args) == type("ab") or type(args) == type(u"ab"):
return [args]
else:
return args
def getFileName(self, arg):
if arg.filename:
return arg.filename
else:
raise TracError("This resource has no filename.")
def getSystemOutput(self, cmd):
Messages = []
proc = os.popen(cmd, "r")
for line in proc:
Messages.append(line)
status = proc.close()
return Messages
def processApplyPatch(self, req):
if not 'TRAC_ADMIN' in req.perm:
raise TracError(
'You do not have enough priviledges to apply a patch!')
tempDir = tempfile.mkdtemp("patch")
os.removedirs(tempDir)
repDir = self.config.get('trac', 'repository_dir')
gitBin = self.config.get('git', 'git_bin')
# git commands
gitAm = gitBin + ' am -3 --ignore-whitespace --whitespace=fix ' # see LSIS ticket #45
gitBranch = gitBin + ' branch '
gitClone = gitBin + ' clone '
gitCo = gitBin + ' checkout '
gitFetch = gitBin + ' fetch '
gitLog = gitBin + ' log '
gitPush = gitBin + ' push '
log = ""
for id in self.getIDs(req.args['select']):
try:
patch_path = self.getPatchPath(id)
except RuntimeError:
print ""
try:
db = self.env.get_db_cnx()
cursor = db.cursor()
cursor.execute(
"SELECT email, subject, branch, submit_time FROM Patches WHERE id='"
+ str(id) + "'")
for email, subject, branch, submit_time in cursor:
if not branch:
branch = 'master'
tempDir = tempfile.mkdtemp('tbranch' + str(id))
os.removedirs(tempDir)
self.env.log.debug("terminal call: " + gitClone + " -b " +
branch + " " + repDir + " " + tempDir)
os.system(gitClone + " -b " + branch + " " + repDir + " " +
tempDir)
# Need to distinguish between an email-patch and a binary bundle:
_bundle_head_pattern = "git bundle"
_cmd = "head -n 1 " + patch_path + " | grep \"" + _bundle_head_pattern + "\""
if os.system(_cmd) != os.EX_OK:
self.env.log.debug("applying patch...")
self.env.log.debug("terminal call2: " + "cd " +
tempDir + "; " + gitAm +
patch_path + " 2>&1")
messages = self.getSystemOutput("cd " + tempDir +
"; " + gitAm +
patch_path + " 2>&1")
log += self.toString(messages)
else:
self.env.log.debug("applying git bundle...")
_tmp_branch = "tmp"
# apply the bundle
messages = self.getSystemOutput("cd " + tempDir +
"; " + gitCo + " -b " +
_tmp_branch + " 2>&1")
log += self.toString(messages)
log += "\n"
messages = self.getSystemOutput("cd " + tempDir +
"; " + gitFetch +
patch_path + " " +
branch + ":" + branch +
" 2>&1")
log += self.toString(messages)
# get back to master and clean up
log += "\n"
messages = self.getSystemOutput("cd " + tempDir +
"; " + gitCo +
" master 2>&1")
log += self.toString(messages)
log += "\n"
messages = self.getSystemOutput("cd " + tempDir +
"; " + gitBranch +
" -D " + _tmp_branch +
" 2>&1")
log += self.toString(messages)
# Log (DEBUG)
log += "\n"
messages = self.getSystemOutput("cd " + tempDir +
"; " + gitLog +
" --graph -3 " +
branch + " 2>&1")
log += self.toString(messages)
# Push changes to public repo
try:
self.env.log.debug("Applying patch on %s" % branch)
messages = self.getSystemOutput("cd " + tempDir +
"; " + gitPush +
" origin " + branch +
" 2>&1")
log += "\n" + self.toString(messages)
self.env.log.debug("Patch applied on %s" % branch)
os.system("rm -Rf " + tempDir)
self.setRejected(id, 0)
if email and subject and submit_time:
subjectCut = str(subject)[8:80]
if len(str(subject)) > 80:
subjectCut = subjectCut + "..."
self.sendMail(
"*****@*****.**",
"Patch applied: " + subjectCut,
"New patch has been applied in rasdaman:\n\n" +
str(subject) + "\n\nSubmitted on: " +
str(submit_time) + "\nSubmitted by: " +
str(email), None)
self.sendPatchStatusMail(id, "APPLIED")
except RuntimeError:
self.env.log("Patch was not applied")
print ""
except RuntimeError:
self.env.log.debug("Reached here - patch check failed")
print ""
return {
'page': 'applypatchlog.html',
'data': {
'messages': log.strip()
}
}
def toString(self, msgs):
res = ""
for m in msgs:
res += str(m)
return res.strip() + "\n\n"
def setRejected(self, id, flag):
db = self.env.get_db_cnx()
cursor = db.cursor()
try:
cursor.execute("UPDATE Patches SET rejected=" + str(flag) +
" WHERE id=" + id)
except:
self.env.log.error("Error un-rejecting patch %s" % id)
db.commit()
def processRejectPatch(self, req):
if not 'TRAC_ADMIN' in req.perm:
raise TracError(
'You do not have enough priviledges to reject a patch!')
Messages = []
for id in self.getIDs(req.args['select']):
self.setRejected(id, 1)
self.sendPatchStatusMail(id, "REJECTED")
return {'data': {'messages': Messages}}
def processTryApplyPatch(self, req):
if not 'TRAC_ADMIN' in req.perm:
raise TracError(
'You do not have enough priviledges to apply a patch!')
repDir = self.config.get('trac', 'repository_dir')
gitBin = self.config.get('git', 'git_bin')
# git commands
gitAm = gitBin + ' am -3 --ignore-whitespace --whitespace=fix ' # see LSIS ticket #45
gitClone = gitBin + ' clone '
gitCo = gitBin + ' checkout '
Messages = []
for id in self.getIDs(req.args['select']):
try:
patch_path = self.getPatchPath(id)
except RuntimeError:
print ""
try:
self.setRejected(id, 0)
# self.sendPatchStatusMail(id, "APPLIED")
# send an email to rasdaman-dev
db = self.env.get_db_cnx()
cursor = db.cursor()
cursor.execute(
"SELECT email, subject, branch, submit_time FROM Patches WHERE id='"
+ str(id) + "'")
for email, subject, branch, submit_time in cursor:
if not branch:
branch = 'master'
self.env.log.debug("apply patch to branch: " + branch)
tempDir = tempfile.mkdtemp('tbranch' + str(id))
os.removedirs(tempDir)
self.env.log.debug("terminal call: " + gitClone + " -b " +
branch + " " + repDir + " " + tempDir)
messages = self.getSystemOutput(gitClone + " -b " +
branch + " " + repDir +
" " + tempDir)
Messages.extend(messages)
messages = self.getSystemOutput("cd " + tempDir + "; " +
gitAm + patch_path +
" 2>&1")
Messages.extend(messages)
self.env.log.debug("terminal call2: " + "cd " + tempDir +
"; " + gitAm + patch_path + " 2>&1")
os.system("rm -Rf " + tempDir)
except RuntimeError:
print ""
return {'data': {'messages': Messages}}
def process_command(self, patchop, req):
ind = patchop.find("-")
if ind != -1:
req.args.update([("select", patchop[ind + 1:])])
patchop = patchop[:ind]
if (patchop == "Upload patch"):
return self.processUploadPatch(req)
elif (patchop == "Delete"):
self.processDeletePatch(req)
req.redirect(req.href.patchmanager())
elif (patchop == "Download Selected"):
self.processDownloadPatch(req)
elif (patchop == "Apply"):
return self.processApplyPatch(req)
elif (patchop == "Try Apply"):
return self.processTryApplyPatch(req)
elif (patchop == "Reject"):
ret = self.processRejectPatch(req)
req.redirect(req.href.patchmanager())
return ret
else:
raise TracError('Don\'t know how to handle operation: "' +
patchop + '"')
def is_jenkins_enabled(self):
isEnabled = self.config.get('jenkins', 'enable', 'false')
return isEnabled.lower() == 'true'
class Environment(Component, ComponentManager):
"""Trac environment manager.
Trac stores project information in a Trac environment. It consists
of a directory structure containing among other things:
* a configuration file,
* project-specific templates and plugins,
* the wiki and ticket attachments files,
* the SQLite database file (stores tickets, wiki pages...)
in case the database backend is sqlite
"""
implements(ISystemInfoProvider)
required = True
system_info_providers = ExtensionPoint(ISystemInfoProvider)
setup_participants = ExtensionPoint(IEnvironmentSetupParticipant)
components_section = ConfigSection(
'components', """This section is used to enable or disable components
provided by plugins, as well as by Trac itself. The component
to enable/disable is specified via the name of the
option. Whether its enabled is determined by the option value;
setting the value to `enabled` or `on` will enable the
component, any other value (typically `disabled` or `off`)
will disable the component.
The option name is either the fully qualified name of the
components or the module/package prefix of the component. The
former enables/disables a specific component, while the latter
enables/disables any component in the specified
package/module.
Consider the following configuration snippet:
{{{
[components]
trac.ticket.report.ReportModule = disabled
acct_mgr.* = enabled
}}}
The first option tells Trac to disable the
[wiki:TracReports report module].
The second option instructs Trac to enable all components in
the `acct_mgr` package. Note that the trailing wildcard is
required for module/package matching.
To view the list of active components, go to the ''Plugins''
page on ''About Trac'' (requires `CONFIG_VIEW`
[wiki:TracPermissions permissions]).
See also: TracPlugins
""")
shared_plugins_dir = PathOption(
'inherit', 'plugins_dir', '',
"""Path to the //shared plugins directory//.
Plugins in that directory are loaded in addition to those in
the directory of the environment `plugins`, with this one
taking precedence.
Non-absolute paths are relative to the Environment `conf`
directory.
""")
base_url = Option(
'trac', 'base_url', '', """Reference URL for the Trac deployment.
This is the base URL that will be used when producing
documents that will be used outside of the web browsing
context, like for example when inserting URLs pointing to Trac
resources in notification e-mails.""")
base_url_for_redirect = BoolOption(
'trac', 'use_base_url_for_redirect', False,
"""Optionally use `[trac] base_url` for redirects.
In some configurations, usually involving running Trac behind
a HTTP proxy, Trac can't automatically reconstruct the URL
that is used to access it. You may need to use this option to
force Trac to use the `base_url` setting also for
redirects. This introduces the obvious limitation that this
environment will only be usable when accessible from that URL,
as redirects are frequently used.
""")
secure_cookies = BoolOption(
'trac', 'secure_cookies', False,
"""Restrict cookies to HTTPS connections.
When true, set the `secure` flag on all cookies so that they
are only sent to the server on HTTPS connections. Use this if
your Trac instance is only accessible through HTTPS.
""")
anonymous_session_lifetime = IntOption(
'trac', 'anonymous_session_lifetime', '90',
"""Lifetime of the anonymous session, in days.
Set the option to 0 to disable purging old anonymous sessions.
(''since 1.0.17'')""")
project_name = Option('project', 'name', 'My Project',
"""Name of the project.""")
project_description = Option('project', 'descr', 'My example project',
"""Short description of the project.""")
project_url = Option(
'project', 'url', '',
"""URL of the main project web site, usually the website in
which the `base_url` resides. This is used in notification
e-mails.""")
project_admin = Option(
'project', 'admin', '',
"""E-Mail address of the project's administrator.""")
project_admin_trac_url = Option(
'project', 'admin_trac_url', '.',
"""Base URL of a Trac instance where errors in this Trac
should be reported.
This can be an absolute or relative URL, or '.' to reference
this Trac instance. An empty value will disable the reporting
buttons.
""")
project_footer = Option(
'project', 'footer',
N_('Visit the Trac open source project at<br />'
'<a href="http://trac.edgewall.org/">'
'http://trac.edgewall.org/</a>'),
"""Page footer text (right-aligned).""")
project_icon = Option('project', 'icon', 'common/trac.ico',
"""URL of the icon of the project.""")
log_type = ChoiceOption('logging',
'log_type',
log.LOG_TYPES + log.LOG_TYPE_ALIASES,
"""Logging facility to use.
Should be one of (`none`, `file`, `stderr`, `syslog`, `winlog`).""",
case_sensitive=False)
log_file = Option(
'logging', 'log_file', 'trac.log',
"""If `log_type` is `file`, this should be a path to the
log-file. Relative paths are resolved relative to the `log`
directory of the environment.""")
log_level = ChoiceOption('logging',
'log_level',
log.LOG_LEVELS + log.LOG_LEVEL_ALIASES,
"""Level of verbosity in log.
Should be one of (`CRITICAL`, `ERROR`, `WARNING`, `INFO`, `DEBUG`).
""",
case_sensitive=False)
log_format = Option(
'logging', 'log_format', None, """Custom logging format.
If nothing is set, the following will be used:
`Trac[$(module)s] $(levelname)s: $(message)s`
In addition to regular key names supported by the
[http://docs.python.org/library/logging.html Python logger library]
one could use:
- `$(path)s` the path for the current environment
- `$(basename)s` the last path component of the current environment
- `$(project)s` the project name
Note the usage of `$(...)s` instead of `%(...)s` as the latter form
would be interpreted by the !ConfigParser itself.
Example:
`($(thread)d) Trac[$(basename)s:$(module)s] $(levelname)s: $(message)s`
""")
def __init__(self, path, create=False, options=[]):
"""Initialize the Trac environment.
:param path: the absolute path to the Trac environment
:param create: if `True`, the environment is created and
populated with default data; otherwise, the
environment is expected to already exist.
:param options: A list of `(section, name, value)` tuples that
define configuration options
"""
ComponentManager.__init__(self)
self.path = os.path.normpath(os.path.normcase(path))
self.log = None
self.config = None
if create:
self.create(options)
for setup_participant in self.setup_participants:
setup_participant.environment_created()
else:
self.verify()
self.setup_config()
def __repr__(self):
return '<%s %r>' % (self.__class__.__name__, self.path)
@lazy
def name(self):
"""The environment name.
:since: 1.2
"""
return os.path.basename(self.path)
@property
def env(self):
"""Property returning the `Environment` object, which is often
required for functions and methods that take a `Component` instance.
"""
# The cached decorator requires the object have an `env` attribute.
return self
@property
def system_info(self):
"""List of `(name, version)` tuples describing the name and
version information of external packages used by Trac and plugins.
"""
info = []
for provider in self.system_info_providers:
info.extend(provider.get_system_info() or [])
return sorted(set(info),
key=lambda args: (args[0] != 'Trac', args[0].lower()))
def get_systeminfo(self):
"""Return a list of `(name, version)` tuples describing the name
and version information of external packages used by Trac and plugins.
:since 1.3.1: deprecated and will be removed in 1.5.1. Use
system_info property instead.
"""
return self.system_info
# ISystemInfoProvider methods
def get_system_info(self):
yield 'Trac', self.trac_version
yield 'Python', sys.version
yield 'setuptools', setuptools.__version__
if pytz is not None:
yield 'pytz', pytz.__version__
if hasattr(self, 'webfrontend_version'):
yield self.webfrontend, self.webfrontend_version
def component_activated(self, component):
"""Initialize additional member variables for components.
Every component activated through the `Environment` object
gets three member variables: `env` (the environment object),
`config` (the environment configuration) and `log` (a logger
object)."""
component.env = self
component.config = self.config
component.log = self.log
def _component_name(self, name_or_class):
name = name_or_class
if not isinstance(name_or_class, basestring):
name = name_or_class.__module__ + '.' + name_or_class.__name__
return name.lower()
@lazy
def _component_rules(self):
_rules = {}
for name, value in self.components_section.options():
name = name.rstrip('.*').lower()
_rules[name] = as_bool(value)
return _rules
def is_component_enabled(self, cls):
"""Implemented to only allow activation of components that are
not disabled in the configuration.
This is called by the `ComponentManager` base class when a
component is about to be activated. If this method returns
`False`, the component does not get activated. If it returns
`None`, the component only gets activated if it is located in
the `plugins` directory of the environment.
"""
component_name = self._component_name(cls)
rules = self._component_rules
cname = component_name
while cname:
enabled = rules.get(cname)
if enabled is not None:
return enabled
idx = cname.rfind('.')
if idx < 0:
break
cname = cname[:idx]
# By default, all components in the trac package except
# in trac.test or trac.tests are enabled
return component_name.startswith('trac.') and \
not component_name.startswith('trac.test.') and \
not component_name.startswith('trac.tests.') or None
def enable_component(self, cls):
"""Enable a component or module."""
self._component_rules[self._component_name(cls)] = True
super(Environment, self).enable_component(cls)
@contextmanager
def component_guard(self, component, reraise=False):
"""Traps any runtime exception raised when working with a component
and logs the error.
:param component: the component responsible for any error that
could happen inside the context
:param reraise: if `True`, an error is logged but not suppressed.
By default, errors are suppressed.
"""
try:
yield
except TracError as e:
self.log.warning("Component %s failed with %s", component,
exception_to_unicode(e))
if reraise:
raise
except Exception as e:
self.log.error("Component %s failed with %s", component,
exception_to_unicode(e, traceback=True))
if reraise:
raise
def verify(self):
"""Verify that the provided path points to a valid Trac environment
directory."""
try:
tag = read_file(os.path.join(self.path, 'VERSION')).splitlines()[0]
if tag != _VERSION:
raise Exception(
_("Unknown Trac environment type '%(type)s'", type=tag))
except Exception as e:
raise TracError(
_("No Trac environment found at %(path)s\n"
"%(e)s",
path=self.path,
e=e))
@lazy
def db_exc(self):
"""Return an object (typically a module) containing all the
backend-specific exception types as attributes, named
according to the Python Database API
(http://www.python.org/dev/peps/pep-0249/).
To catch a database exception, use the following pattern::
try:
with env.db_transaction as db:
...
except env.db_exc.IntegrityError as e:
...
"""
return DatabaseManager(self).get_exceptions()
@property
def db_query(self):
"""Return a context manager
(`~trac.db.api.QueryContextManager`) which can be used to
obtain a read-only database connection.
Example::
with env.db_query as db:
cursor = db.cursor()
cursor.execute("SELECT ...")
for row in cursor.fetchall():
...
Note that a connection retrieved this way can be "called"
directly in order to execute a query::
with env.db_query as db:
for row in db("SELECT ..."):
...
:warning: after a `with env.db_query as db` block, though the
`db` variable is still defined, you shouldn't use it as it
might have been closed when exiting the context, if this
context was the outermost context (`db_query` or
`db_transaction`).
If you don't need to manipulate the connection itself, this
can even be simplified to::
for row in env.db_query("SELECT ..."):
...
"""
return QueryContextManager(self)
@property
def db_transaction(self):
"""Return a context manager
(`~trac.db.api.TransactionContextManager`) which can be used
to obtain a writable database connection.
Example::
with env.db_transaction as db:
cursor = db.cursor()
cursor.execute("UPDATE ...")
Upon successful exit of the context, the context manager will
commit the transaction. In case of nested contexts, only the
outermost context performs a commit. However, should an
exception happen, any context manager will perform a rollback.
You should *not* call `commit()` yourself within such block,
as this will force a commit even if that transaction is part
of a larger transaction.
Like for its read-only counterpart, you can directly execute a
DML query on the `db`::
with env.db_transaction as db:
db("UPDATE ...")
:warning: after a `with env.db_transaction` as db` block,
though the `db` variable is still available, you shouldn't
use it as it might have been closed when exiting the
context, if this context was the outermost context
(`db_query` or `db_transaction`).
If you don't need to manipulate the connection itself, this
can also be simplified to::
env.db_transaction("UPDATE ...")
"""
return TransactionContextManager(self)
def shutdown(self, tid=None):
"""Close the environment."""
from trac.versioncontrol.api import RepositoryManager
RepositoryManager(self).shutdown(tid)
DatabaseManager(self).shutdown(tid)
if tid is None:
log.shutdown(self.log)
def create(self, options=[]):
"""Create the basic directory structure of the environment,
initialize the database and populate the configuration file
with default values.
If options contains ('inherit', 'file'), default values will
not be loaded; they are expected to be provided by that file
or other options.
:raises TracError: if the base directory of `path` does not exist.
:raises TracError: if `path` exists and is not empty.
"""
base_dir = os.path.dirname(self.path)
if not os.path.exists(base_dir):
raise TracError(
_(
"Base directory '%(env)s' does not exist. Please create it "
"and retry.",
env=base_dir))
if os.path.exists(self.path) and os.listdir(self.path):
raise TracError(_("Directory exists and is not empty."))
# Create the directory structure
if not os.path.exists(self.path):
os.mkdir(self.path)
os.mkdir(self.htdocs_dir)
os.mkdir(self.log_dir)
os.mkdir(self.plugins_dir)
os.mkdir(self.templates_dir)
# Create a few files
create_file(os.path.join(self.path, 'VERSION'), _VERSION + '\n')
create_file(
os.path.join(self.path, 'README'),
'This directory contains a Trac environment.\n'
'Visit http://trac.edgewall.org/ for more information.\n')
# Setup the default configuration
os.mkdir(self.conf_dir)
config = Configuration(self.config_file_path)
for section, name, value in options:
config.set(section, name, value)
config.save()
self.setup_config()
if not any((section, option) == ('inherit', 'file')
for section, option, value in options):
self.config.set_defaults(self)
self.config.save()
# Create the sample configuration
create_file(self.config_file_path + '.sample')
self._update_sample_config()
# Create the database
DatabaseManager(self).init_db()
@lazy
def database_version(self):
"""Returns the current version of the database.
:since 1.0.2:
"""
return DatabaseManager(self) \
.get_database_version('database_version')
@lazy
def database_initial_version(self):
"""Returns the version of the database at the time of creation.
In practice, for database created before 0.11, this will
return `False` which is "older" than any db version number.
:since 1.0.2:
"""
return DatabaseManager(self) \
.get_database_version('initial_database_version')
@lazy
def trac_version(self):
"""Returns the version of Trac.
:since: 1.2
"""
from trac import core, __version__
return get_pkginfo(core).get('version', __version__)
def setup_config(self):
"""Load the configuration file."""
self.config = Configuration(self.config_file_path,
{'envname': self.name})
if not self.config.exists:
raise TracError(
_("The configuration file is not found at "
"%(path)s",
path=self.config_file_path))
self.setup_log()
plugins_dir = self.shared_plugins_dir
load_components(self, plugins_dir and (plugins_dir, ))
@lazy
def config_file_path(self):
"""Path of the trac.ini file."""
return os.path.join(self.conf_dir, 'trac.ini')
@lazy
def log_file_path(self):
"""Path to the log file."""
if not os.path.isabs(self.log_file):
return os.path.join(self.log_dir, self.log_file)
return self.log_file
def _get_path_to_dir(self, *dirs):
path = self.path
for dir in dirs:
path = os.path.join(path, dir)
return os.path.realpath(path)
@lazy
def attachments_dir(self):
"""Absolute path to the attachments directory.
:since: 1.3.1
"""
return self._get_path_to_dir('files', 'attachments')
@lazy
def conf_dir(self):
"""Absolute path to the conf directory.
:since: 1.0.11
"""
return self._get_path_to_dir('conf')
@lazy
def files_dir(self):
"""Absolute path to the files directory.
:since: 1.3.2
"""
return self._get_path_to_dir('files')
@lazy
def htdocs_dir(self):
"""Absolute path to the htdocs directory.
:since: 1.0.11
"""
return self._get_path_to_dir('htdocs')
@lazy
def log_dir(self):
"""Absolute path to the log directory.
:since: 1.0.11
"""
return self._get_path_to_dir('log')
@lazy
def plugins_dir(self):
"""Absolute path to the plugins directory.
:since: 1.0.11
"""
return self._get_path_to_dir('plugins')
@lazy
def templates_dir(self):
"""Absolute path to the templates directory.
:since: 1.0.11
"""
return self._get_path_to_dir('templates')
def setup_log(self):
"""Initialize the logging sub-system."""
self.log, log_handler = \
self.create_logger(self.log_type, self.log_file_path,
self.log_level, self.log_format)
self.log.addHandler(log_handler)
self.log.info('-' * 32 + ' environment startup [Trac %s] ' + '-' * 32,
self.trac_version)
def create_logger(self, log_type, log_file, log_level, log_format):
log_id = 'Trac.%s' % hashlib.sha1(self.path).hexdigest()
if log_format:
log_format = log_format.replace('$(', '%(') \
.replace('%(path)s', self.path) \
.replace('%(basename)s', self.name) \
.replace('%(project)s', self.project_name)
return log.logger_handler_factory(log_type,
log_file,
log_level,
log_id,
format=log_format)
def get_known_users(self, as_dict=False):
"""Returns information about all known users, i.e. users that
have logged in to this Trac environment and possibly set their
name and email.
By default this function returns a iterator that yields one
tuple for every user, of the form (username, name, email),
ordered alpha-numerically by username. When `as_dict` is `True`
the function returns a dictionary mapping username to a
(name, email) tuple.
:since 1.2: the `as_dict` parameter is available.
"""
return self._known_users_dict if as_dict else iter(self._known_users)
@cached
def _known_users(self):
return self.db_query("""
SELECT DISTINCT s.sid, n.value, e.value
FROM session AS s
LEFT JOIN session_attribute AS n ON (n.sid=s.sid
AND n.authenticated=1 AND n.name = 'name')
LEFT JOIN session_attribute AS e ON (e.sid=s.sid
AND e.authenticated=1 AND e.name = 'email')
WHERE s.authenticated=1 ORDER BY s.sid
""")
@cached
def _known_users_dict(self):
return {u[0]: (u[1], u[2]) for u in self._known_users}
def invalidate_known_users_cache(self):
"""Clear the known_users cache."""
del self._known_users
del self._known_users_dict
def backup(self, dest=None):
"""Create a backup of the database.
:param dest: Destination file; if not specified, the backup is
stored in a file called db_name.trac_version.bak
"""
return DatabaseManager(self).backup(dest)
def needs_upgrade(self):
"""Return whether the environment needs to be upgraded."""
for participant in self.setup_participants:
with self.component_guard(participant, reraise=True):
if participant.environment_needs_upgrade():
self.log.warning(
"Component %s requires environment upgrade",
participant)
return True
return False
def upgrade(self, backup=False, backup_dest=None):
"""Upgrade database.
:param backup: whether or not to backup before upgrading
:param backup_dest: name of the backup file
:return: whether the upgrade was performed
"""
upgraders = []
for participant in self.setup_participants:
with self.component_guard(participant, reraise=True):
if participant.environment_needs_upgrade():
upgraders.append(participant)
if not upgraders:
return
if backup:
try:
self.backup(backup_dest)
except Exception as e:
raise BackupError(e)
for participant in upgraders:
self.log.info("upgrading %s...", participant)
with self.component_guard(participant, reraise=True):
participant.upgrade_environment()
# Database schema may have changed, so close all connections
dbm = DatabaseManager(self)
if dbm.connection_uri != 'sqlite::memory:':
dbm.shutdown()
self._update_sample_config()
del self.database_version
return True
@lazy
def href(self):
"""The application root path"""
return Href(urlsplit(self.abs_href.base).path)
@lazy
def abs_href(self):
"""The application URL"""
if not self.base_url:
self.log.warning("base_url option not set in configuration, "
"generated links may be incorrect")
return Href(self.base_url)
def _update_sample_config(self):
filename = os.path.join(self.config_file_path + '.sample')
if not os.path.isfile(filename):
return
config = Configuration(filename)
config.set_defaults()
try:
config.save()
except EnvironmentError as e:
self.log.warning("Couldn't write sample configuration file (%s)%s",
e, exception_to_unicode(e, traceback=True))
else:
self.log.info(
"Wrote sample configuration file with the new "
"settings and their default values: %s", filename)
class OidcPlugin(Component):
""" Authenticate via OpenID Connect
"""
implements(INavigationContributor, IRequestHandler)
RETURN_URL_SKEY = 'trac_oidc.return_url'
client_secret_file = PathOption(
'trac_oidc', 'client_secret_file', 'client_secret.json',
"""Path to client_secret file. Relative paths are interpreted
relative to the ``conf`` subdirectory of the trac environment.""")
# deprecated
absolute_trust_root = BoolOption(
'openid', 'absolute_trust_root', 'true',
"""Whether we should use absolute trust root or by project.""")
login_managers = ExtensionPoint(ILoginManager)
def __init__(self):
# We should show our own "Logout" link only if the stock
# LoginModule is disabled.
self.show_logout_link = not is_component_enabled(self.env, LoginModule)
self.userdb = UserDatabase(self.env)
# INavigationContributor methods
def get_active_navigation_item(self, req):
return 'trac_oidc.login'
def get_navigation_items(self, req):
oidc_href = req.href.trac_oidc
path_qs = req.path_info
if req.query_string:
path_qs += '?' + req.query_string
if not req.authname or req.authname == 'anonymous':
# Not logged in, show login link
login_link = tag.a(_('Login using Google'),
href=oidc_href('login', return_to=path_qs))
yield 'metanav', 'trac_oidc.login', login_link
elif self.show_logout_link:
# Logged in and LoginModule is disabled, show logout link
yield ('metanav', 'trac_oidc.login',
_('logged in as %(user)s', user=req.authname))
yield ('metanav', 'trac_oidc.logout',
logout_link(oidc_href, return_to=path_qs))
# IRequestHandler methods
def match_request(self, req):
path_info = req.path_info
if path_info == '/login' and self.show_logout_link:
# Stock LoginModule is disabled, so handle default /login too
return True
return path_info in ('/trac_oidc/login', '/trac_oidc/logout',
'/trac_oidc/redirect')
def process_request(self, req):
if req.path_info.endswith('/logout'):
return_url = self._get_return_url(req)
self._forget_user(req)
return req.redirect(return_url)
elif req.path_info.endswith('/login'):
# Start the login process by redirectory to OP
req.session[self.RETURN_URL_SKEY] = self._get_return_url(req)
authenticator = self._get_authenticator(req)
return req.redirect(authenticator.get_auth_url(req))
elif req.path_info.endswith('/redirect'):
# Finish the login process after redirect from OP
return_url = req.session.pop(self.RETURN_URL_SKEY, req.abs_href())
id_token = self._retrieve_id(req)
if id_token:
authname = self._find_or_create_session(req, id_token)
assert authname
self.log.debug("Logging in as %r", authname)
self._remember_user(req, authname)
return req.redirect(return_url)
# private methods
def _retrieve_id(self, req):
""" Retrieve oidc id_token from provider.
Returns ``None`` if authentication was unsuccessful for any reason.
"""
authenticator = self._get_authenticator(req)
try:
return authenticator.get_identity(req)
except AuthenticationFailed as ex:
self.log.info("Authentication failed: %s", ex)
add_warning(req, "Authentication failed: %s", ex)
except AuthenticationError as ex:
self.log.error("Authentication error: %s", ex)
add_warning(req, "Authentication error: %s", ex)
def _find_or_create_session(self, req, id_token):
""" Find or create authenticated session for subject.
"""
userdb = self.userdb
authname = userdb.find_session(id_token)
if not authname:
# There is no authenticated session for the user,
# create a new one
# XXX: should it be configurable whether this happens?
authname = userdb.create_session(id_token)
add_notice(
req,
_(
"Hello! You appear to be new here. "
"A new authenticated session with "
"username '%(authname)s' has been created for you.",
authname=authname))
return authname
def _remember_user(self, req, authname):
for lm in self.login_managers:
lm.remember_user(req, authname)
def _forget_user(self, req):
for lm in self.login_managers:
lm.forget_user(req)
def _get_authenticator(self, req):
conf_dir = os.path.join(self.env.path, 'conf')
client_secret_file = os.path.join(conf_dir, self.client_secret_file)
redirect_url = req.abs_href.trac_oidc('redirect')
openid_realm = self._get_openid_realm(req)
self.log.debug('openid_realm = %r', openid_realm)
return Authenticator(client_secret_file, redirect_url, openid_realm,
self.log)
def _get_openid_realm(self, req):
""" Get the OpenID realm.
This computes the OpenID realm in exactly the same manner
that the ``TracAuthOpenID`` plugin does.
Note that I'm not sure this is really the “right” way to do it,
but, since we want to get back the same identity URLs from google
as we did using ``TracAuthOpenID``, here we are.
"""
href = req.href()
abs_href = self.env.abs_href()
if href and abs_href.endswith(href):
base_url = abs_href[:-len(href)]
else: # pragma: NO COVER
base_url = abs_href
if self.absolute_trust_root:
path = '/'
else:
path = href
return base_url + path
@staticmethod
def _get_return_url(req):
return_to = req.args.getfirst('return_to', '/')
# We expect return_to to be a URL relative to the trac's base_path.
# Be paranoid about this.
scheme, netloc, path, query, anchor = urlsplit(return_to)
if scheme or netloc or '..' in path.split('/') or anchor:
# return url looks suspicious, ignore it.
return req.abs_href()
return_url = req.abs_href(path)
if query:
return_url += '?' + query
return return_url
