class SVNExternalViewer(Component):
implements(IRepositoryViewerProvider)
implements(IProjectFeaturelistItemProvider)
def __init__(self):
self.title = 'SVN'
if 'title' in self.component_config:
self.title = self.component_config['title']
def get_repository_viewers(self, repository):
"""Announce announce an external viewer"""
if 'url_base' not in self.component_config:
logger.warning("url_base is not configured!")
elif repository.type == 'svn':
return (self.title,
self.component_config['url_base'] + '/' + repository.name)
def get_project_featurelist_items(self, project):
if project.get_repository_type('svn').get_repositories(project):
if 'url_base' not in self.component_config:
logger.warning("url_base is not configured!")
return (self.title, [])
return (self.title, [{
'href':
self.component_config['url_base'] + '/' + project.name,
'name':
'view'
}])
class GitIntegration(Component):
"""Cydra component for integration between Git and Cydra"""
implements(IRepositoryViewerProvider)
implements(IProjectFeaturelistItemProvider)
def __init__(self):
pass
def get_repository_viewers(self, repository):
"""Announce git as a web viewer"""
if 'url_base' not in self.component_config:
logger.warning("url_base is not configured!")
elif repository.type == 'git':
return ('Git HTTP', self.component_config['url_base'] + '/' +
repository.project.name + '/' + repository.name + '.git')
def get_project_featurelist_items(self, project):
if project.get_repository_type('git').get_repositories(project):
if 'url_base' not in self.component_config:
logger.warning("url_base is not configured!")
return ('Git HTTP', [])
return ('Git HTTP', [{
'href':
self.component_config['url_base'] + '/' + project.name,
'name':
'view'
}])
class HtpasswdUsers(Component):
implements(IUserAuthenticator)
implements(IUserTranslator)
implements(IUserStore)
def __init__(self):
config = self.get_component_config()
if 'file' not in config:
raise InsufficientConfiguration(missing='file', component=self.get_component_name())
self.htpasswd = HtpasswdFile(config['file'])
def username_to_user(self, username):
self.htpasswd.load_if_changed()
if username in self.htpasswd.users():
return HtpasswdUser(self, username, username=username, full_name=username)
def userid_to_user(self, userid):
if userid is None or userid == '*':
warnings.warn("You should not call this directly. Use cydra.get_user()", DeprecationWarning, stacklevel=2)
return self.compmgr.get_user(userid='*')
self.htpasswd.load_if_changed()
if userid in self.htpasswd.users():
return HtpasswdUser(self, userid, username=userid, full_name=userid)
else:
# since the client was looking for a specific ID,
# we return a dummy user object with empty data
return User(self, userid, full_name='N/A')
def groupid_to_group(self, groupid):
pass
def user_password(self, user, password):
self.htpasswd.load_if_changed()
return self.htpasswd.check_password(user.userid, password)
def create_user(self, **kwargs):
self.htpasswd.load_if_changed()
userid = None
if 'id' in kwargs:
userid = kwargs['id']
elif 'username' in kwargs:
userid = kwargs['username']
else:
raise ValueError("No username/id specified")
if userid in self.htpasswd.users():
raise ValueError("User with this id already exists")
else:
self.htpasswd.set_password(userid, hashlib.sha1(os.urandom(8)).hexdigest())
self.htpasswd.save()
return userid
class RepositoryProviderComponent(Component):
"""Base class for components providing repositories"""
implements(IProjectObserver)
implements(IRepositoryProvider)
abstract = True
def pre_delete_project(self, project, archiver=None):
"""Handle project delete by deleting all repositories"""
for repo in self.get_repositories(project):
repo.delete(archiver, project_deletion=True)
class StaticDefaultConfigurator(Component):
"""Applies a static default configuration to every created project"""
implements(IProjectObserver)
def post_create_project(self, project):
config = self.component_config.get("config", {})
logger.debug("Extending project %s with config %r", project.name, config)
merge(project.data, config)
project.save()
class GitServerGlue(Component):
"""Cydra component for integration between GitServerGlue and Cydra"""
implements(IRepositoryViewerProvider)
implements(IProjectFeaturelistItemProvider)
def __init__(self):
pass
def get_repository_viewers(self, repository):
"""Add clone URL to the viewers"""
res = []
if 'ssh_url_base' not in self.component_config:
logger.warning("ssh_url_base is not configured!")
elif repository.type == 'git':
res.append(
('ssh://', self.component_config['ssh_url_base'] + '/git/' +
repository.project.name + '/' + repository.name + '.git'))
if 'http_url_base' not in self.component_config:
logger.warning("http_url_base is not configured!")
elif repository.type == 'git':
res.append(
('http://', self.component_config['http_url_base'] + '/' +
repository.project.name + '/' + repository.name + '.git'))
return res
def get_project_featurelist_items(self, project):
if project.get_repository_type('git').get_repositories(project):
if 'http_url_base' not in self.component_config:
logger.warning("http_url_base is not configured!")
return ('Git HTTP', [])
return ('Git HTTP', [{
'href':
self.component_config['http_url_base'] + '/' + project.name,
'name':
'view'
}])
class MemorySubjectCache(Component):
"""Caches subjects in memory
"""
implements(ISubjectCache)
def __init__(self):
config = self.get_component_config()
groupttl = config.get('group_ttl', 60 * 60 * 24 * 14)
groupsize = config.get('group_size', 50)
userttl = config.get('user_ttl', 5 * 60)
usersize = config.get('user_size', 500)
self.groupcache = SimpleCache(lifetime=groupttl,
killtime=groupttl,
maxsize=groupsize)
self.usercache = SimpleCache(lifetime=userttl,
killtime=userttl,
maxsize=usersize)
self.usernamemap = {}
def get_user(self, userid):
return self.usercache.get(userid)
def get_user_by_name(self, username):
if username in self.usernamemap:
return self.get_user(self.usernamemap[username])
def get_users(self, userids):
res = {}
for userid in userids:
res[userid] = self.get_user(userid)
return res
def add_users(self, users):
for user in users:
self.usercache.set(user.userid, user)
self.usernamemap[user.username] = user.userid
def get_group(self, groupid):
return self.groupcache.get(groupid)
def get_groups(self, groupids):
res = {}
for groupid in groupids:
res[groupid] = self.get_group(groupid)
return res
def add_groups(self, groups):
for group in groups:
self.groupcache.set(group.groupid, group)
class HtpasswdUsers(Component):
implements(IUserAuthenticator)
implements(IUserTranslator)
def __init__(self):
config = self.get_component_config()
if 'file' not in config:
raise InsufficientConfiguration(missing='file', component=self.get_component_name())
self.htpasswd = HtpasswdFile(config['file'])
def username_to_user(self, username):
self.htpasswd.load_if_changed()
if username in self.htpasswd.users():
return User(self.compmgr, username, username=username, full_name=username)
def userid_to_user(self, userid):
if userid is None or userid == '*':
warnings.warn("You should not call this directly. Use cydra.get_user()", DeprecationWarning, stacklevel=2)
return User(self.compmgr, '*', username='******', full_name='Guest')
self.htpasswd.load_if_changed()
if userid in self.htpasswd.users():
return User(self.compmgr, userid, username=userid, full_name=userid)
else:
# since the client was looking for a specific ID,
# we return a dummy user object with empty data
return User(self.compmgr, userid, full_name='N/A')
def groupid_to_group(self, groupid):
pass
def user_password(self, user, password):
self.htpasswd.load_if_changed()
return self.htpasswd.check_password(user.userid, password)
class TwistedGit(Component):
"""Cydra component for integration between TwistedGit and Cydra"""
implements(IRepositoryViewerProvider)
def __init__(self):
pass
def get_repository_viewers(self, repository):
"""Add clone URL to the viewers"""
if 'url_base' not in self.component_config:
logger.warning("url_base is not configured!")
elif repository.type == 'git':
return ('Git over ssh', self.component_config['url_base'] + '/' +
repository.project.name + '/' + repository.name + '.git')
class ConfigurationFile(Component):
implements(IConfigurationProvider)
def get_config(self):
cconfig = self.get_component_config()
config = {}
cfiles = []
if 'file' in cconfig:
if isinstance(cconfig['file'], list):
cfiles.extend(cconfig['file'])
else:
cfiles.append(cconfig['file'])
else:
cfiles.extend(self.find_default_locations())
for cfile in cfiles:
merge(config, self.load_file(cfile))
return config
def find_default_locations(self):
locations = [
'/etc/cydra.conf',
os.path.join(os.path.expanduser('~'), '.cydra'), 'cydra.conf'
]
locations = [os.path.abspath(location) for location in locations]
return [location for location in locations if os.path.exists(location)]
def load_file(self, filename):
cfile = codecs.open(filename, "r", "utf-8")
try:
return json.load(cfile)
except ValueError:
# it is not in JSON format, try YAML if available
if load_yaml:
try:
cfile.seek(0)
return load_yaml(cfile)
except:
logger.exception("Unable to parse YAML")
finally:
cfile.close()
logger.error("Unable to parse configfile: " + filename)
return {}
class StaticGlobalPermissionProvider(Component):
"""Global permissions defined in config"""
implements(IPermissionProvider)
def get_permissions(self, project, user, object):
if project is not None:
return {}
if user is not None and not user.is_guest and object == 'projects':
return {'create': True}
def get_group_permissions(self, project, group, object):
return {}
def get_permission(self, project, user, object, permission):
if project is not None:
return None
if user is not None and not user.is_guest and object == 'projects' and permission == 'create':
return True
def get_group_permission(self, project, group, object, permission):
return None
def set_permission(self, project, user, object, permission, value=None):
return None
def set_group_permission(self,
project,
group,
object,
permission,
value=None):
return None
def get_projects_user_has_permissions_on(self, userid):
return []
class ConfigurationFile(Component):
implements(IConfigurationProvider)
def get_config(self):
cconfig = self.compmgr.config.get_component_config(
self.get_component_name(), {})
config = {}
cfiles = []
if 'file' in cconfig:
if isinstance(cconfig['file'], list):
cfiles.extend(cconfig['file'])
else:
cfiles.append(cconfig['file'])
else:
cfiles.extend(self.find_default_locations())
for cfile in cfiles:
self.compmgr.config.merge(config, self.load_file(cfile))
return config
def find_default_locations(self):
locations = [
'/etc/cydra.conf',
os.path.join(os.path.expanduser('~'), '.cydra'), 'cydra.conf'
]
locations = [os.path.abspath(location) for location in locations]
#print "Config locations:", locations
return [location for location in locations if os.path.exists(location)]
def load_file(self, filename):
cfile = open(filename, "r")
try:
return json.load(cfile)
finally:
cfile.close()
class HgRepositories(Component):
implements(IRepository)
repository_type = 'hg'
repository_type_title = 'Mercurial'
def __init__(self):
config = self.get_component_config()
if 'base' not in config:
InsufficientConfiguration(missing='base',
component=self.get_component_name())
self._base = config['base']
self.hgcommand = config.get('hgcommand', 'hg')
def get_repositories(self, project):
if not os.path.exists(os.path.join(self._base, project.name)):
return []
return [
HgRepository(self.compmgr, self._base, project, x)
for x in os.listdir(os.path.join(self._base, project.name))
]
def get_repository(self, project, repository_name):
return HgRepository(self.compmgr, self._base, project, repository_name)
def can_create(self, project, user=None):
if user:
return project.get_permission(user, 'repository.hg', 'create')
else:
return True
def create_repository(self, project, repository_name, **params):
if not is_valid_repository_name(repository_name):
raise CydraError("Invalid Repository Name", name=repository_name)
path = os.path.join(self._base, project.name, repository_name)
if os.path.exists(path):
raise CydraError('Path already exists', path=path)
if not os.path.exists(os.path.join(self._base, project.name)):
os.mkdir(os.path.join(self._base, project.name))
hg_cmd = subprocess.Popen([self.hgcommand, 'init', path],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output, errors = hg_cmd.communicate()
if hg_cmd.returncode != 0:
if hg_cmd.returncode == 127:
# assume this is command not found
raise CydraError(
'Command not found encountered while calling hg',
stderr=errors)
else:
raise CydraError('Error encountered while calling hg',
stderr=errors,
code=hg_cmd.returncode)
repository = HgRepository(self.compmgr, self._base, project,
repository_name)
repository.set_params(**params)
repository.sync() # synchronize repository
return repository
def get_params(self):
return [param_description, param_contact]
class ADUsers(Component):
implements(IUserAuthenticator)
implements(IUserTranslator)
def __init__(self):
config = self.get_component_config()
self.ldap = LdapLookup(**config)
if not self.ldap.connect():
raise Exception('Connection failed')
def username_to_user(self, username):
user = self._ldap_to_user(self.ldap.get_user(username))
if user is None:
logger.error("Translation failed for: %s" % username)
return user
def userid_to_user(self, userid):
if userid is None or userid == '*':
return User(self.compmgr, '*', username='******', full_name='Guest')
user = self._ldap_to_user(self.ldap.get_user(userid))
if user is None:
logger.error("Translation failed for: %s" % userid)
# since the client was looking for a specific ID,
# we return a dummy user object with empty data
return User(self.compmgr, userid, full_name='N/A')
else:
return user
def _ldap_to_user(self, data):
if data is None:
return None
dn, userobj = data
if 'memberOf' in userobj:
groups = [
self._ldap_to_group(self.ldap.get_dn(x))
for x in userobj['memberOf']
]
else:
groups = []
return User(self.compmgr,
userobj['userPrincipalName'][0],
username=userobj['sAMAccountName'][0],
full_name=force_unicode(userobj['displayName'][0]),
groups=groups)
def groupid_to_group(self, groupid):
group = self._ldap_to_group(self.ldap.get_group(groupid))
if group is None:
logger.error("Group lookup error for %s", groupid)
return group
def _ldap_to_group(self, data):
if data is None:
return None
dn, groupobj = data
return Group(self.compmgr,
groupobj['name'][0],
name=groupobj['name'][0])
def user_password(self, user, password):
if not user or not password:
return False
try:
conn = ldap.initialize(self.get_component_config()['uri'])
conn.simple_bind_s(user.userid, password)
except ldap.INVALID_CREDENTIALS:
logger.exception("Authentication failed")
return False
return True
class InternalPermissionProvider(Component):
"""Stores permissions in the project's dict
Example::
'permissions': {
'*': {'object': ['read']},
'user': {'*': ['admin']}
}
"""
implements(IPermissionProvider)
MODE_GROUP, MODE_USER = range(2)
PERMISSION_ROOT = {
MODE_GROUP: 'group_permissions',
MODE_USER: '******'
}
def get_permissions(self, project, user, obj):
return self._get_permissions(self.MODE_USER, project, user, obj)
def get_group_permissions(self, project, group, obj):
return self._get_permissions(self.MODE_GROUP, project, group, obj)
def _get_permissions(self, mode, project, subject, obj):
if project is None:
return {} # no project, no permissions
# Resolve root of permissions and translator
# depending on what we try to find
permroot = self.PERMISSION_ROOT[mode]
if mode == self.MODE_USER:
translator = self.compmgr.get_user
elif mode == self.MODE_GROUP:
translator = self.compmgr.get_group
else:
raise ValueError('Unknown mode')
res = {}
perms = project.data.get(permroot, {})
# if both subject and obj are None, return all (subject, obj, perm)
# copy whole structure to prevent side effects
if subject is None and obj is None:
for s, objs in perms.items():
s = translator(s)
res[s] = {}
for o, perm in objs:
res[s][o] = perm.copy()
# Inject global owner permissions if necessary
if mode == self.MODE_USER:
res.setdefault(project.owner, {}).setdefault(
'*', {}).update(virtual_owner_permissions)
return res
# construct a list of objects in the hierarchy
if obj is not None:
objparts = list(object_walker(obj))
objparts.reverse()
# if subject is none, find all subjects and return all (subject, perm)
# we know here that obj is not none as we handled subject none and obj none
# case above
if subject is None:
for s, p in perms.items():
s = translator(s)
res[s] = {}
for o in objparts:
if o in p:
res[s].update(p[o].copy())
# delete empty entries
if res[s] == {}:
del res[s]
# Inject global owner permissions if necessary
if mode == self.MODE_USER:
res.setdefault(project.owner,
{}).update(virtual_owner_permissions)
return res
# subject is given.
# in case of user mode, we also check the guest account
subjects = [subject.id]
if mode == self.MODE_USER:
subjects.append('*')
for p in [perms[x] for x in subjects if x in perms]:
if obj is not None:
for o in objparts:
if o in p:
res.update(p[o].copy())
else:
for o in p:
res[o] = p[o].copy()
# this is the owner, Inject global owner perms
if mode == self.MODE_USER and project.owner == subject:
if obj is None:
res.setdefault('*', {}).update(virtual_owner_permissions)
else:
res.update(virtual_owner_permissions)
# also inject all group permissions
if mode == self.MODE_USER:
for group in [x for x in subject.groups if x is not None
]: # safeguard against failing translators
res.update(self.get_group_permissions(project, group, obj))
return res
def get_permission(self, project, user, obj, permission):
return self._get_permission(self.MODE_USER, project, user, obj,
permission)
def get_group_permission(self, project, group, obj, permission):
return self._get_permission(self.MODE_GROUP, project, group, obj,
permission)
def _get_permission(self, mode, project, subject, obj, permission):
if project is None:
return None
if subject is None:
return None
if obj is None:
return None
# Resolve root of permissions and translator
# depending on what we try to find
permroot = self.PERMISSION_ROOT[mode]
if mode == self.MODE_USER:
translator = self.compmgr.get_user
elif mode == self.MODE_GROUP:
translator = self.compmgr.get_group
else:
raise ValueError('Unknown mode')
# the owner can do everything
if mode == self.MODE_USER and project.owner == subject:
return True
perms = project.data.get(permroot, {})
# What we want to find here is a specific permission on a specific
# object. First get the most precise. If we have a conflict, return the most positive one
ret = None
# If we are in user mode, check groups first
if mode == self.MODE_USER:
for group in subject.groups:
ret = self._merge_perm_values(
ret,
self.get_group_permission(project, group, obj, permission))
# root level -> find subject in perms
if subject.id in perms:
perms = perms[subject.id]
elif mode == self.MODE_USER and '*' in perms: # if we are in user mode, fall back to guest
perms = perms['*']
else:
return ret
# subject level. Now walk the tree. deeper value overwrites lower
subjret = None
for o in object_walker(obj):
if o in perms:
# object level
perm = perms[o].get(permission, None)
if perm is None:
perm = perms[o].get('admin', None)
if perm is not None:
subjret = perm
# now merge subjret with the previous value
return self._merge_perm_values(ret, subjret)
def _merge_perm_values(self, a, b):
if a == False or b == False:
return False
elif a == True or b == True:
return True
else:
return None
def set_permission(self, project, user, obj, permission, value=None):
return self._set_permission(self.MODE_USER, project, user, obj,
permission, value)
def set_group_permission(self,
project,
group,
obj,
permission,
value=None):
return self._set_permission(self.MODE_GROUP, project, group, obj,
permission, value)
def _set_permission(self,
mode,
project,
subject,
obj,
permission,
value=None):
if project is None:
return None
if subject is None:
return None
if obj is None:
return None
# Resolve root of permissions depending on what we try to find
permroot = self.PERMISSION_ROOT[mode]
if value is None:
# check if the permission is set, otherwise do nothing
if permission in project.data.get(permroot,
{}).get(subject.id,
{}).get(obj, {}):
# remove permission
del project.data[permroot][subject.id][obj][permission]
if project.data[permroot][subject.id][obj] == {}:
del project.data[permroot][subject.id][obj]
if project.data[permroot][subject.id] == {}:
del project.data[permroot][subject.id]
else:
project.data.setdefault(permroot,
{}).setdefault(subject.id, {}).setdefault(
obj, {})[permission] = value
project.save()
return True
def get_projects_user_has_permissions_on(self, user):
res = set([
project for project in self.compmgr.get_projects_where_key_exists(
['permissions', user.userid]) if any(
project.data.get('permissions', {}).get(user.userid,
{}).values())
])
for group in user.groups:
res.update(
set([
project
for project in self.compmgr.get_projects_where_key_exists(
['group_permissions', group.id]) if any(
project.data.get('group_permissions', {}).get(
group.id, {}).values())
]))
res.update(self.compmgr.get_projects_owned_by(user))
return res
class SVNRepositories(Component):
implements(IRepository)
repository_type = 'svn'
repository_type_title = 'SVN'
def __init__(self):
config = self.get_component_config()
if 'base' not in config:
InsufficientConfiguration(missing='base',
component=self.get_component_name())
self._base = config['base']
self.svncommand = config.get('svncommand', 'svnadmin')
def get_repositories(self, project):
if not os.path.exists(os.path.join(self._base, project.name)):
return []
else:
return [SVNRepository(self.compmgr, self._base, project)]
def get_repository(self, project, repository_name):
return SVNRepository(self.compmgr, self._base, project)
def can_create(self, project, user=None):
if user:
return len(self.get_repositories(
project)) == 0 and project.get_permission(
user, 'repository.svn', 'create')
else:
return len(self.get_repositories(project)) == 0
def create_repository(self, project, repository_name):
if repository_name != project.name:
raise CydraError(
"SVN Repository name has to be the same as the project's name")
path = os.path.join(self._base, project.name)
if os.path.exists(path):
raise CydraError('Path already exists', path=path)
svn_cmd = subprocess.Popen([self.svncommand, 'create', path],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output, errors = svn_cmd.communicate()
if svn_cmd.returncode != 0:
if svn_cmd.returncode == 127:
# assume this is command not found
raise CydraError(
'Command not found encountered while calling svn',
stderr=errors)
else:
raise CydraError('Error encountered while calling svn',
stderr=errors,
code=hg_cmd.returncode)
# Customize config
repository = SVNRepository(self.compmgr, self._base, project)
repository.sync()
return repository
def get_params(self):
return []
class MongoDataSource(Component):
"""Datasource that saves projects into a MongoDB database
This datasource encodes keys to allow for '.' in key names.
"""
implements(IDataSource)
implements(IPubkeyStore)
def __init__(self):
config = self.get_component_config()
if 'host' not in config:
raise Exception('Host not configured')
if 'database' not in config:
raise Exception('Database not configured')
self.connection = Connection(config['host'])
self.database = self.connection[config['database']]
if 'user' in config and 'password' in config:
self.database.authenticate(config['user'], config['password'])
@staticmethod
def _encode_key(val, magic='%'):
"""Helper function to encode . in keys as mongoDB does not allow them"""
ret = val
ret = ret.replace(magic, magic + '1')
ret = ret.replace('.', magic + '2')
return ret
@staticmethod
def _decode_key(val, magic='%'):
"""Helper function to decode . in keys as mongoDB does not allow them"""
ret = val
ret = ret.replace(magic + '2', '.')
ret = ret.replace(magic + '1', magic)
return ret
@staticmethod
def _process_dict_keys(data, f):
if type(data) not in [list, set, dict]:
return data
ret = type(data)()
if isinstance(data, dict):
for key, val in data.items():
ret[f(key)] = MongoDataSource._process_dict_keys(val, f)
elif isinstance(data, list):
for val in data:
ret.append(MongoDataSource._process_dict_keys(val, f))
elif isinstance(data, set):
for val in data:
ret.add(MongoDataSource._process_dict_keys(val, f))
else:
raise Exception("The universe exploded")
return ret
@staticmethod
def _encode_dict_keys(data):
return MongoDataSource._process_dict_keys(data,
MongoDataSource._encode_key)
@staticmethod
def _decode_dict_keys(data):
return MongoDataSource._process_dict_keys(data,
MongoDataSource._decode_key)
def get_project(self, projectname):
# Check name
if not is_valid_project_name(projectname):
return None
project = self.database.projects.find_one({'name': projectname})
if project is not None:
return Project(self.compmgr, self._decode_dict_keys(project))
def save_project(self, project):
self.database.projects.save(self._encode_dict_keys(project.data))
def create_project(self, projectname, owner):
# Check name
if not is_valid_project_name(projectname):
return None
if self.get_project(projectname) is None:
self.database.projects.insert({
'name': projectname,
'owner': owner.userid
})
return self.get_project(projectname)
def list_projects(self):
ret = []
for p in self.database.projects.find(sort=[('name', ASCENDING)]):
ret.append(Project(self.compmgr, self._decode_dict_keys(p)))
return ret
def get_project_names(self):
ret = []
for p in self.database.projects.find(fields=['name'],
sort=[('name', ASCENDING)]):
ret.append(self._decode_dict_keys(p)['name'])
return ret
def get_projects_owned_by(self, user):
if user is None:
return []
ret = []
for p in self.database.projects.find({'owner': user.userid},
sort=[('name', ASCENDING)]):
ret.append(Project(self.compmgr, self._decode_dict_keys(p)))
return ret
def get_projects_where_key_exists(self, key):
search = {self._encode_key(str(key)): {'$exists': True}}
if isinstance(key, list):
search = {'.'.join(map(self._encode_key, key)): {'$exists': True}}
ret = []
for p in self.database.projects.find(search,
sort=[('name', ASCENDING)]):
ret.append(Project(self.compmgr, self._decode_dict_keys(p)))
return ret
def get_pubkeys(self, user):
"""Does user have a pubkey with blob"""
return self.database.pubkeys.find({'userid': user.userid},
sort=[('name', ASCENDING)])
def user_has_pubkey(self, user, blob):
return bool(
self.database.pubkeys.find_one({
'blob': binary.Binary(blob),
'userid': user.userid
}))
def add_pubkey(self, user, blob, name="unnamed", fingerprint=""):
"""Add a new public key for a user"""
if self.user_has_pubkey(user, blob):
return False
self.database.pubkeys.insert({
'userid': user.userid,
'blob': binary.Binary(blob),
'name': name,
'fingerprint': fingerprint
})
return True
def remove_pubkey(self, user, **kwargs):
spec = {'userid': user.userid}
for k, v in kwargs.items():
if k in ['name', 'blob', 'fingerprint']:
spec[k] = v
if len(spec) > 1:
self.database.pubkeys.remove(spec)
return True
class FileDataSource(Component):
"""Datasource that saves projects into files
"""
implements(IDataSource)
def __init__(self):
config = self.get_component_config()
if 'base' not in config:
raise InsufficientConfiguration(
missing='base', component=self.get_component_name())
self._base = config['base']
def _get_project_path(self, name):
return os.path.join(self._base, name + '.yaml')
def get_project(self, projectname):
# Check name
if not is_valid_project_name(projectname):
return None
path = self._get_project_path(projectname)
if os.path.exists(path):
with open(path, 'r') as f:
return Project(self.compmgr, yaml.safe_load(f))
def save_project(self, project):
# Check name
if not is_valid_project_name(project.name):
return None
path = self._get_project_path(project.name)
with open(path, 'w') as f:
yaml.safe_dump(project.data, f)
def create_project(self, projectname, owner):
# Check name
if not is_valid_project_name(projectname):
return None
if self.get_project(projectname) is None:
path = self._get_project_path(projectname)
with open(path, 'w') as f:
yaml.safe_dump({'name': projectname, 'owner': owner.userid}, f)
return self.get_project(projectname)
def delete_project(self, project):
# Check name
if not is_valid_project_name(project.name):
return None
path = self._get_project_path(project.name)
os.remove(path)
def list_projects(self):
ret = []
for filename in os.listdir(self._base):
if filename.endswith(".yaml"):
ret.append(self.get_project(filename[:-len(".yaml")]))
return ret
def get_project_names(self):
ret = []
for filename in os.listdir(self._base):
if filename.endswith(".yaml"):
ret.append(filename[:-len(".yaml")])
return ret
def get_projects_owned_by(self, user):
if user is None:
return []
ret = []
for project in self.list_projects():
if project.owner == user:
ret.append(project)
return ret
def get_projects_where_key_exists(self, key):
ret = []
for project in self.list_projects():
if isinstance(key, list):
look_in = project.data
found = True
for component in key:
if component not in look_in:
found = False
break
look_in = look_in[component]
if found:
ret.append(project)
else:
if str(key) in project.data:
ret.append(project)
return ret
class TracEnvironments(Component):
"""Cydra component for trac support"""
implements(ICliProjectCommandProvider)
implements(IRepositoryObserver)
implements(IBlueprintProvider)
implements(IProjectActionProvider)
implements(IRepositoryActionProvider)
implements(IProjectFeaturelistItemProvider)
implements(IProjectSyncParticipant)
# this map might look silly right now, but it is not guaranteed that cydra and trac name
# repository types the same
typemap = {'svn': 'svn', 'hg': 'hg', 'git': 'git'}
def __init__(self):
"""Initialize Trac component
This will raise an exception if the base path for the environments
has not been configured"""
if 'base' not in self.component_config:
raise Exception("trac environments base path not configured")
def get_env_path(self, project):
return os.path.join(self.component_config['base'], project.name)
def get_default_options(self, project):
"""Get the default set of options Cydra enforces
These options override everything"""
return [
('project', 'name', project.name),
('trac', 'database', 'sqlite:db/trac.db'
), #TODO: implement the possibility to override this
('trac', 'repository_sync_per_request',
''), # We use hooks, don't sync per request
('trac', 'permission_policies', 'CydraPermissionPolicy'),
('components', 'tracext.hg.*', 'enabled'),
('components', 'tracext.git.*', 'enabled'),
('components', 'cydraplugins.*', 'enabled'),
('git', 'cached_repository', 'true'),
('header_logo', 'src',
'common/trac_banner.png'), # Perhaps let user set a custom one
]
def has_env(self, project):
"""Does the project contain a Trac environment"""
return os.path.exists(self.get_env_path(project))
def create(self, project):
"""Create a Trac environment for the project
This will NOT call trac-admin but interfaces with Trac's API
directly. It performs the same steps as trac-admin"""
if self.has_env(project):
return True
# When creating environments, you can supply a list
# of (section, option, value) tuples to trac which serve as a default
options = self.get_default_options(project)
# If an (inherit, file, xy) option is present, trac will omit the default values
if 'inherit_config' in self.component_config:
options.append(
('inherit', 'file', self.component_config['inherit_config']))
try:
# create environment
env = Environment(self.get_env_path(project),
create=True,
options=options)
# preload wiki pages
wiki_pages = None
if 'preload_wiki' in self.component_config:
wiki_pages = self.component_config['preload_wiki']
else:
try:
wiki_pages = pkg_resources.resource_filename(
'trac.wiki', 'default-pages')
except Exception as e:
logger.exception(
"Exception while trying to find wiki pages")
wiki_pages = None
if wiki_pages:
try:
WikiAdmin(env).load_pages(wiki_pages)
except Exception as e:
logger.exception("Unable to load wiki pages from %s",
wiki_pages)
else:
logger.warning("Not wiki pages found for preloading")
# all done
return True
except Exception as e:
logger.exception(
"Caught exception while creating Trac environment in " +
self.get_env_path(project))
return False
def sync_project(self, project):
"""For project.ISyncParticipant"""
self.sync(project)
def sync(self, project):
"""Sync the trac environment with cydra
This sets the options returned by ``get_default_options`` and adds Trac's own defaults if necessary"""
if not self.has_env(project):
logger.warning('Project %s has no Trac Environment to sync',
project.name)
return
tracini = os.path.join(self.get_env_path(project), 'conf', 'trac.ini')
options = self.get_default_options(project)
# if inherit is enabled, the default values are supposed to be in
# the inherited file. Thus, we can truncate the config file to get a bare minimum
if 'inherit_config' in self.component_config:
options.append(
('inherit', 'file', self.component_config['inherit_config']))
with open(tracini, 'w') as f:
f.truncate()
# re-create the configuration file
config = Configuration(tracini)
for section, name, value in options:
config.set(section, name, value)
config.save()
# load defaults
if not any((section, option) == ('inherit', 'file')
for section, option, value in options):
config.set_defaults()
config.save()
# check if repositories in cydra match repositories in trac
env = Environment(self.get_env_path(project))
rm = RepositoryManager(env)
trac_repos = rm.get_real_repositories()
trac_repo_names = [r.reponame for r in trac_repos]
for repotype, repos in project.data.get('plugins', {}).get('trac',
{}).items():
for repo, tracname in (repos or {}).items():
if tracname not in trac_repo_names:
logger.warning(
"Removing trac mapping from cydra for %s repo %s",
repo, tracname)
del repos[repo]
if not repos:
del project.data.get('plugins', {}).get('trac',
{})[repotype]
# Now do the reverse
revmap = dict([(y, x) for (x, y) in self.typemap.items()])
for repo in trac_repos:
logger.debug('Looking at trac repo %s', repo.reponame)
try:
baseparts = repo.get_base().split(
':'
) # This is extremely naiive and possibly breaks some time
repotype, path = baseparts[0], baseparts[-1]
except:
logger.error("Unable to parse: " + repo.get_base())
reponame = os.path.basename(path)
if repotype == 'git':
reponame = reponame[:-4]
try:
repository = project.get_repository(revmap[repotype], reponame)
except:
logger.error("Unable to locate %s %s (%s)", repotype, reponame,
path)
repository = None
logger.debug('Cydra repo %r', repository)
if repository:
# set this mapping if not there already
project.data.setdefault('plugins', {}).setdefault(
'trac', {}).setdefault(repository.type,
{})[repository.name] = repo.reponame
logger.info('Setting trac mapping for %s %s -> %s',
repository.type, repository.name, repo.reponame)
else:
logger.error("Unable to load %s %s (%s)", revmap[repotype],
reponame, path)
project.save()
def register_repository(self, repository, name=None):
"""Register a repository with trac"""
project = repository.project
tracname = name if name is not None else repository.name
if repository.name in project.data.get('plugins',
{}).get('trac', {}).get(
repository.type, {}):
logger.error(
"Repository %s:%s is already registered in project %s",
repository.type, repository.name, project.name)
return False
if repository.type not in self.typemap:
logger.error("Repository type %s is not supported in Trac",
repository.type)
return False
if not self.has_env(project):
logger.warning(
"Tried to add repository %s:%s to Trac of project %s, but there is no environment",
repository.type, repository.name, project.name)
return False
try:
env = Environment(self.get_env_path(project))
DbRepositoryProvider(env).add_repository(
tracname, repository.path, self.typemap[repository.type])
# save mapping in project
project.data.setdefault('plugins',
{}).setdefault('trac', {}).setdefault(
repository.type,
{})[repository.name] = tracname
project.save()
# Synchronise repository
rm = RepositoryManager(env)
repos = rm.get_repository(tracname)
repos.sync(lambda rev: logger.debug("Synced revision: %s", rev),
clean=True)
return True
except Exception as e:
logger.exception(
"Exception occured while addingrepository %s:%s to Trac of project %s",
repository.type, repository.name, project.name)
return False
def get_cli_project_commands(self):
return [('trac', self.cli_command)]
def cli_command(self, project, args):
"""Manipulate the trac environment for a project
Supported arguments:
- create: creates an environment
- sync: Synchronizes the configuration with Cydra's requirements
- addrepo <type> <name> [tracname]: adds the repository to trac, identified by tracname
- updatedefaults <file>: Adds trac's default options to config"""
if len(args) < 1 or args[0] not in [
'create', 'addrepo', 'sync', 'updatedefaults'
]:
print self.cli_command.__doc__
return
if args[0] == 'create':
if self.has_env(project):
print "Project already has a Trac environment!"
return
if self.create(project):
print "Environment created"
else:
print "Creation failed!"
elif args[0] == 'sync':
self.sync(project)
print project.name, "synced"
elif args[0] == 'addrepo':
if len(args) < 3:
print self.cli_command.__doc__
return
repository = project.get_repository(args[1], args[2])
if not repository:
print "Unknown repository"
return
ret = False
if len(args) == 4:
ret = self.register_repository(repository, args[3])
else:
ret = self.register_repository(repository)
if ret:
print "Successfully added repository"
else:
print "Adding repository failed!"
elif args[0] == 'updatedefaults':
if len(args) < 2:
print self.cli_command.__doc__
return
config = Configuration(args[1])
# load defaults
config.set_defaults()
config.save()
def get_blueprint(self):
"""Get the blueprint for trac actions in the web interface"""
from flask import Blueprint, render_template, abort, redirect, url_for, flash, request, jsonify, current_app
from cydra.web.wsgihelper import InsufficientPermissions
from werkzeug.exceptions import NotFound
from werkzeug.local import LocalProxy
blueprint = Blueprint('trac',
__name__,
static_folder='static',
template_folder='templates')
cydra_instance = LocalProxy(lambda: current_app.config['cydra'])
cydra_user = LocalProxy(lambda: request.environ['cydra_user'])
@blueprint.route('/project/<projectname>/trac/create',
methods=['POST'])
def create(projectname):
project = cydra_instance.get_project(projectname)
if project is None:
raise NotFound('Unknown project')
if not project.get_permission(cydra_user, '*', 'admin'):
raise InsufficientPermissions()
self.create(project)
return redirect(
url_for('frontend.project', projectname=projectname))
@blueprint.route(
'/project/<projectname>/trac/register_repository/<repositorytype>/<repositoryname>',
methods=['POST'])
def register_repository(projectname, repositorytype, repositoryname):
project = cydra_instance.get_project(projectname)
if project is None:
raise NotFound('Unknown project')
if not project.get_permission(cydra_user, '*', 'admin'):
raise InsufficientPermissions()
repository = project.get_repository(repositorytype, repositoryname)
if repository is None:
raise NotFound('Unknown repository')
self.register_repository(repository)
return redirect(
url_for('frontend.project', projectname=projectname))
return blueprint
def get_project_featurelist_items(self, project):
if not self.has_env(project):
return
if 'url_base' not in self.component_config:
logger.warning("url_base is not configured!")
return ('Trac', [])
else:
return ('Trac', [{
'href':
self.component_config['url_base'] + '/' + project.name,
'name':
'view'
}])
def get_project_actions(self, project):
if not self.has_env(project):
return ('Create Trac', 'trac.create', 'post')
def get_repository_actions(self, repository):
if self.has_env(repository.project):
if repository.name not in repository.project.data.get(
'plugins', {}).get('trac', {}).get(repository.type, {}):
return ('Register in Trac', 'trac.register_repository', 'post')
else:
pass # TODO: deregister
def repository_change_commit(self, repository, revisions):
self._changeset_event(repository, 'changeset_added', revisions)
# IRepositoryObserver
def repository_post_commit(self, repository, revisions):
self._changeset_event(repository, 'changeset_modified', revisions)
# IRepositoryObserver
def pre_delete_repository(self, repository):
tracrepo = repository.project.data.get('plugins', {}).get(
'trac', {}).get(repository.type, {}).get(repository.name)
if tracrepo and self.has_env(repository.project):
# remove it
logger.info("Removing repository %s from Trac environment %s",
tracrepo, repository.project.name)
env = Environment(self.get_env_path(repository.project))
DbRepositoryProvider(env).remove_repository(tracrepo)
# clean up
del repository.project.data['plugins']['trac'][repository.type][
repository.name]
if not repository.project.data['plugins']['trac'][repository.type]:
del repository.project.data['plugins']['trac'][repository.type]
def _changeset_event(self, repository, event, revisions):
project = repository.project
if not self.has_env(project):
return
tracrepos = project.data.get('plugins',
{}).get('trac',
{}).get(repository.type,
{}).get(repository.name)
if tracrepos:
try:
env = Environment(self.get_env_path(project))
RepositoryManager(env).notify(event, tracrepos, revisions)
except Exception as e:
logger.exception(
"Exception occured while calling post-commit hook for revs %s on repository %s (%s) in project %s",
revisions, repository.name, repository.type, project.name)
class StaticPermissionProvider(DictBasedPermissionProvider):
"""Handle permissions defined in config
Global (ie. non project specific) permissions can be defined::
'global_user_permissions': {'*': {'projects': {'create': True}},
'user1': {'projects': {'create': True}}}
'global_group_permissions': {'*': {'projects': {'create': True}}}
As well as project specific/for all projects::
'user_permissions': {'*': {'user1': {'*': {'admin': True}}}}
'group_permissions': {'proj1': {'group1': {'repository.svn.*': {'admin': True}}}}
Note that the user '*' is restricted to any logged-in users
to prevent unintentional access rights for non-logged-in guests."""
implements(IPermissionProvider)
def _get_user_base(self, project, user):
if user is not None and user.is_guest:
return {}
if project is None:
return self.component_config.get('global_user_permissions', {})
if project.name in self.component_config.get('user_permissions', {}):
return self.component_config.get('user_permissions')[project.name]
return self.component_config.get('user_permissions', {}).get('*', {})
def _get_group_base(self, project):
if project is None:
return self.component_config.get('global_group_permissions', {})
if project.name in self.component_config.get('group_permissions', {}):
return self.component_config.get('group_permissions')[project.name]
return self.component_config.get('group_permissions', {}).get('*', {})
def get_permissions(self, project, user, obj):
base = self._get_user_base(project, user)
return self._get_permissions(base, self.compmgr.get_user, project,
user, obj)
def get_group_permissions(self, project, group, obj):
base = self._get_group_base(project)
return self._get_permissions(base, self.compmgr.get_group, project,
group, obj)
def get_permission(self, project, user, obj, permission):
base = self._get_user_base(project, user)
return self._get_permission(base, project, user, obj, permission)
def get_group_permission(self, project, group, obj, permission):
base = self._get_group_base(project)
return self._get_permission(base, project, group, obj, permission)
def get_projects_user_has_permissions_on(self, user):
projects = set()
for projectid, perms in self.component_config.get(
'user_permissions', {}):
if user.id in perms or '*' in perms:
if projectid == '*':
pass # TODO
else:
projects.add(projectid)
for projectid, perms in self.component_config.get(
'group_permissions', {}):
if any(group.id in perms for group in user.groups):
if projectid == '*':
pass # TODO
else:
projects.add(projectid)
return [self.compmgr.get_project(x) for x in projects]
# Set operations are not supported...
def set_permission(self, project, user, obj, permission, value=None):
return None
def set_group_permission(self,
project,
group,
obj,
permission,
value=None):
return None
class DictBasedPermissionProvider(Component):
"""Base for permissions providers using a dict for storage"""
implements(IPermissionProvider)
abstract = True
def get_permissions(self, project, user, obj):
pass
def get_group_permissions(self, project, group, obj):
pass
def _get_permissions(self, permission_dict, subject_translator, project,
subject, obj):
"""Return permissions based on"""
res = {}
# if both subject and obj are None, return all (subject, obj, perm)
# copy whole structure to prevent side effects
if subject is None and obj is None:
for s, objs in permission_dict.items():
s = subject_translator(s)
res[s] = {}
for o, perm in objs:
res[s][o] = perm.copy()
return res
# construct a list of objects in the hierarchy
if obj is not None:
objparts = list(object_walker(obj))
objparts.reverse()
# if subject is none, find all subjects and return all (subject, perm)
# we know here that obj is not none as we handled subject none and obj none
# case above
if subject is None:
for s, p in permission_dict.items():
s = subject_translator(s)
res[s] = {}
for o in objparts:
if o in p:
res[s].update(p[o].copy())
# delete empty entries
if res[s] == {}:
del res[s]
return res
# subject is given.
# in case of user mode, we also check the guest account
subjects = [subject.id]
if isinstance(subject, User):
subjects.append('*')
for p in [
permission_dict[x] for x in subjects if x in permission_dict
]:
if obj is not None:
for o in objparts:
if o in p:
res.update(p[o].copy())
else:
for o in p:
res[o] = p[o].copy()
# also inject all group permissions
if isinstance(subject, User):
for group in [x for x in subject.groups if x is not None
]: # safeguard against failing translators
res.update(self.get_group_permissions(project, group, obj))
return res
def _get_permission(self, permission_dict, project, subject, obj,
permission):
if subject is None or obj is None or permission is None:
return None
subject_translator = self.compmgr.get_group if isinstance(
subject, Group) else self.compmgr.get_user
permissions = self._get_permissions(permission_dict,
subject_translator, project,
subject, None)
for o in object_walker(obj):
if o in permissions:
return permissions[o].get(permission)
return None
class HtpasswdUsers(Component):
implements(IUserAuthenticator)
implements(IUserTranslator)
def __init__(self):
config = self.get_component_config()
if 'file' not in config:
raise InsufficientConfiguration(
missing='file', component=self.get_component_name())
self.users = {}
with open(config['file'], 'r') as f:
for line in f.readlines():
user, hash = line.strip().split(':', 1)
self.users[user] = hash
def username_to_user(self, username):
if username in self.users:
return User(self.compmgr,
username,
username=username,
full_name=username)
def userid_to_user(self, userid):
if userid is None or userid == '*':
warnings.warn(
"You should not call this directly. Use cydra.get_user()",
DeprecationWarning,
stacklevel=2)
return User(self.compmgr, '*', username='******', full_name='Guest')
if userid in self.users:
return User(self.compmgr,
userid,
username=userid,
full_name=userid)
else:
# since the client was looking for a specific ID,
# we return a dummy user object with empty data
return User(self.compmgr, userid, full_name='N/A')
def groupid_to_group(self, groupid):
pass
def user_password(self, user, password):
if not user.userid in self.users:
return
hash = self.users[user.userid]
if hash.startswith('{SHA}'):
pass
elif hash.startswith('$apr1$'):
pass
else:
import crypt
if hash == crypt.crypt(password, hash[:2]):
return True
return False
class GitRepositories(Component):
"""Component for git based repositories
Configuration:
- base: Path to the directory where repositories are stored
- gitcommand: Path to git command. Defaults to git"""
implements(IRepository)
repository_type = 'git'
repository_type_title = 'Git'
def __init__(self):
config = self.get_component_config()
if 'base' not in config:
raise InsufficientConfiguration(
missing='base', component=self.get_component_name())
self._base = config['base']
self.gitcommand = config.get('gitcommand', 'git')
def get_repositories(self, project):
"""Returns a list of repositories for the project
This list is based on the filesystem"""
if not os.path.exists(os.path.join(self._base, project.name)):
return []
return [
GitRepository(self.compmgr, self._base, project, x[:-4])
for x in os.listdir(os.path.join(self._base, project.name))
if x[-4:] == '.git'
]
def get_repository(self, project, repository_name):
"""Return a GitRepository instance for a repository"""
if not self._repo_exists(project, repository_name):
return
else:
return GitRepository(self.compmgr, self._base, project,
repository_name)
def can_create(self, project, user=None):
"""Returns whether the user create a new repository"""
if user:
return project.get_permission(user, 'repository.git', 'create')
else:
return True
def create_repository(self, project, repository_name, **params):
"""Create a new git repository
A repository's name can only contain letters, numbers and dashes/underscores."""
if not is_valid_repository_name(repository_name):
raise CydraError("Invalid Repository Name", name=repository_name)
path = os.path.join(self._base, project.name, repository_name + '.git')
if os.path.exists(path):
raise CydraError('Path already exists', path=path)
if not os.path.exists(os.path.join(self._base, project.name)):
os.mkdir(os.path.join(self._base, project.name))
git_cmd = subprocess.Popen([self.gitcommand, 'init', '--bare', path],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output, errors = git_cmd.communicate()
if git_cmd.returncode != 0:
if git_cmd.returncode == 127:
# assume this is command not found
raise CydraError(
'Command not found encountered while calling git',
stderr=errors)
else:
raise CydraError('Error encountered while calling git',
stderr=errors,
code=git_cmd.returncode)
# Customize config
repository = GitRepository(self.compmgr, self._base, project,
repository_name)
repository.set_params(**params)
repository.sync() # synchronize repository
return repository
def get_params(self):
return [param_description]
def _repo_exists(self, project, name):
if not is_valid_repository_name(name):
raise CydraError("Invalid Repository Name", name=name)
path = os.path.join(self._base, project.name, name + '.git')
return os.path.exists(path)
