class SVNExternalViewer(Component): implements(IRepositoryViewerProvider) implements(IProjectFeaturelistItemProvider) def __init__(self): self.title = 'SVN' if 'title' in self.component_config: self.title = self.component_config['title'] def get_repository_viewers(self, repository): """Announce announce an external viewer""" if 'url_base' not in self.component_config: logger.warning("url_base is not configured!") elif repository.type == 'svn': return (self.title, self.component_config['url_base'] + '/' + repository.name) def get_project_featurelist_items(self, project): if project.get_repository_type('svn').get_repositories(project): if 'url_base' not in self.component_config: logger.warning("url_base is not configured!") return (self.title, []) return (self.title, [{ 'href': self.component_config['url_base'] + '/' + project.name, 'name': 'view' }])
class GitIntegration(Component): """Cydra component for integration between Git and Cydra""" implements(IRepositoryViewerProvider) implements(IProjectFeaturelistItemProvider) def __init__(self): pass def get_repository_viewers(self, repository): """Announce git as a web viewer""" if 'url_base' not in self.component_config: logger.warning("url_base is not configured!") elif repository.type == 'git': return ('Git HTTP', self.component_config['url_base'] + '/' + repository.project.name + '/' + repository.name + '.git') def get_project_featurelist_items(self, project): if project.get_repository_type('git').get_repositories(project): if 'url_base' not in self.component_config: logger.warning("url_base is not configured!") return ('Git HTTP', []) return ('Git HTTP', [{ 'href': self.component_config['url_base'] + '/' + project.name, 'name': 'view' }])
class HtpasswdUsers(Component): implements(IUserAuthenticator) implements(IUserTranslator) implements(IUserStore) def __init__(self): config = self.get_component_config() if 'file' not in config: raise InsufficientConfiguration(missing='file', component=self.get_component_name()) self.htpasswd = HtpasswdFile(config['file']) def username_to_user(self, username): self.htpasswd.load_if_changed() if username in self.htpasswd.users(): return HtpasswdUser(self, username, username=username, full_name=username) def userid_to_user(self, userid): if userid is None or userid == '*': warnings.warn("You should not call this directly. Use cydra.get_user()", DeprecationWarning, stacklevel=2) return self.compmgr.get_user(userid='*') self.htpasswd.load_if_changed() if userid in self.htpasswd.users(): return HtpasswdUser(self, userid, username=userid, full_name=userid) else: # since the client was looking for a specific ID, # we return a dummy user object with empty data return User(self, userid, full_name='N/A') def groupid_to_group(self, groupid): pass def user_password(self, user, password): self.htpasswd.load_if_changed() return self.htpasswd.check_password(user.userid, password) def create_user(self, **kwargs): self.htpasswd.load_if_changed() userid = None if 'id' in kwargs: userid = kwargs['id'] elif 'username' in kwargs: userid = kwargs['username'] else: raise ValueError("No username/id specified") if userid in self.htpasswd.users(): raise ValueError("User with this id already exists") else: self.htpasswd.set_password(userid, hashlib.sha1(os.urandom(8)).hexdigest()) self.htpasswd.save() return userid
class RepositoryProviderComponent(Component): """Base class for components providing repositories""" implements(IProjectObserver) implements(IRepositoryProvider) abstract = True def pre_delete_project(self, project, archiver=None): """Handle project delete by deleting all repositories""" for repo in self.get_repositories(project): repo.delete(archiver, project_deletion=True)
class StaticDefaultConfigurator(Component): """Applies a static default configuration to every created project""" implements(IProjectObserver) def post_create_project(self, project): config = self.component_config.get("config", {}) logger.debug("Extending project %s with config %r", project.name, config) merge(project.data, config) project.save()
class GitServerGlue(Component): """Cydra component for integration between GitServerGlue and Cydra""" implements(IRepositoryViewerProvider) implements(IProjectFeaturelistItemProvider) def __init__(self): pass def get_repository_viewers(self, repository): """Add clone URL to the viewers""" res = [] if 'ssh_url_base' not in self.component_config: logger.warning("ssh_url_base is not configured!") elif repository.type == 'git': res.append( ('ssh://', self.component_config['ssh_url_base'] + '/git/' + repository.project.name + '/' + repository.name + '.git')) if 'http_url_base' not in self.component_config: logger.warning("http_url_base is not configured!") elif repository.type == 'git': res.append( ('http://', self.component_config['http_url_base'] + '/' + repository.project.name + '/' + repository.name + '.git')) return res def get_project_featurelist_items(self, project): if project.get_repository_type('git').get_repositories(project): if 'http_url_base' not in self.component_config: logger.warning("http_url_base is not configured!") return ('Git HTTP', []) return ('Git HTTP', [{ 'href': self.component_config['http_url_base'] + '/' + project.name, 'name': 'view' }])
class MemorySubjectCache(Component): """Caches subjects in memory """ implements(ISubjectCache) def __init__(self): config = self.get_component_config() groupttl = config.get('group_ttl', 60 * 60 * 24 * 14) groupsize = config.get('group_size', 50) userttl = config.get('user_ttl', 5 * 60) usersize = config.get('user_size', 500) self.groupcache = SimpleCache(lifetime=groupttl, killtime=groupttl, maxsize=groupsize) self.usercache = SimpleCache(lifetime=userttl, killtime=userttl, maxsize=usersize) self.usernamemap = {} def get_user(self, userid): return self.usercache.get(userid) def get_user_by_name(self, username): if username in self.usernamemap: return self.get_user(self.usernamemap[username]) def get_users(self, userids): res = {} for userid in userids: res[userid] = self.get_user(userid) return res def add_users(self, users): for user in users: self.usercache.set(user.userid, user) self.usernamemap[user.username] = user.userid def get_group(self, groupid): return self.groupcache.get(groupid) def get_groups(self, groupids): res = {} for groupid in groupids: res[groupid] = self.get_group(groupid) return res def add_groups(self, groups): for group in groups: self.groupcache.set(group.groupid, group)
class HtpasswdUsers(Component): implements(IUserAuthenticator) implements(IUserTranslator) def __init__(self): config = self.get_component_config() if 'file' not in config: raise InsufficientConfiguration(missing='file', component=self.get_component_name()) self.htpasswd = HtpasswdFile(config['file']) def username_to_user(self, username): self.htpasswd.load_if_changed() if username in self.htpasswd.users(): return User(self.compmgr, username, username=username, full_name=username) def userid_to_user(self, userid): if userid is None or userid == '*': warnings.warn("You should not call this directly. Use cydra.get_user()", DeprecationWarning, stacklevel=2) return User(self.compmgr, '*', username='******', full_name='Guest') self.htpasswd.load_if_changed() if userid in self.htpasswd.users(): return User(self.compmgr, userid, username=userid, full_name=userid) else: # since the client was looking for a specific ID, # we return a dummy user object with empty data return User(self.compmgr, userid, full_name='N/A') def groupid_to_group(self, groupid): pass def user_password(self, user, password): self.htpasswd.load_if_changed() return self.htpasswd.check_password(user.userid, password)
class TwistedGit(Component): """Cydra component for integration between TwistedGit and Cydra""" implements(IRepositoryViewerProvider) def __init__(self): pass def get_repository_viewers(self, repository): """Add clone URL to the viewers""" if 'url_base' not in self.component_config: logger.warning("url_base is not configured!") elif repository.type == 'git': return ('Git over ssh', self.component_config['url_base'] + '/' + repository.project.name + '/' + repository.name + '.git')
class ConfigurationFile(Component): implements(IConfigurationProvider) def get_config(self): cconfig = self.get_component_config() config = {} cfiles = [] if 'file' in cconfig: if isinstance(cconfig['file'], list): cfiles.extend(cconfig['file']) else: cfiles.append(cconfig['file']) else: cfiles.extend(self.find_default_locations()) for cfile in cfiles: merge(config, self.load_file(cfile)) return config def find_default_locations(self): locations = [ '/etc/cydra.conf', os.path.join(os.path.expanduser('~'), '.cydra'), 'cydra.conf' ] locations = [os.path.abspath(location) for location in locations] return [location for location in locations if os.path.exists(location)] def load_file(self, filename): cfile = codecs.open(filename, "r", "utf-8") try: return json.load(cfile) except ValueError: # it is not in JSON format, try YAML if available if load_yaml: try: cfile.seek(0) return load_yaml(cfile) except: logger.exception("Unable to parse YAML") finally: cfile.close() logger.error("Unable to parse configfile: " + filename) return {}
class StaticGlobalPermissionProvider(Component): """Global permissions defined in config""" implements(IPermissionProvider) def get_permissions(self, project, user, object): if project is not None: return {} if user is not None and not user.is_guest and object == 'projects': return {'create': True} def get_group_permissions(self, project, group, object): return {} def get_permission(self, project, user, object, permission): if project is not None: return None if user is not None and not user.is_guest and object == 'projects' and permission == 'create': return True def get_group_permission(self, project, group, object, permission): return None def set_permission(self, project, user, object, permission, value=None): return None def set_group_permission(self, project, group, object, permission, value=None): return None def get_projects_user_has_permissions_on(self, userid): return []
class ConfigurationFile(Component): implements(IConfigurationProvider) def get_config(self): cconfig = self.compmgr.config.get_component_config( self.get_component_name(), {}) config = {} cfiles = [] if 'file' in cconfig: if isinstance(cconfig['file'], list): cfiles.extend(cconfig['file']) else: cfiles.append(cconfig['file']) else: cfiles.extend(self.find_default_locations()) for cfile in cfiles: self.compmgr.config.merge(config, self.load_file(cfile)) return config def find_default_locations(self): locations = [ '/etc/cydra.conf', os.path.join(os.path.expanduser('~'), '.cydra'), 'cydra.conf' ] locations = [os.path.abspath(location) for location in locations] #print "Config locations:", locations return [location for location in locations if os.path.exists(location)] def load_file(self, filename): cfile = open(filename, "r") try: return json.load(cfile) finally: cfile.close()
class HgRepositories(Component): implements(IRepository) repository_type = 'hg' repository_type_title = 'Mercurial' def __init__(self): config = self.get_component_config() if 'base' not in config: InsufficientConfiguration(missing='base', component=self.get_component_name()) self._base = config['base'] self.hgcommand = config.get('hgcommand', 'hg') def get_repositories(self, project): if not os.path.exists(os.path.join(self._base, project.name)): return [] return [ HgRepository(self.compmgr, self._base, project, x) for x in os.listdir(os.path.join(self._base, project.name)) ] def get_repository(self, project, repository_name): return HgRepository(self.compmgr, self._base, project, repository_name) def can_create(self, project, user=None): if user: return project.get_permission(user, 'repository.hg', 'create') else: return True def create_repository(self, project, repository_name, **params): if not is_valid_repository_name(repository_name): raise CydraError("Invalid Repository Name", name=repository_name) path = os.path.join(self._base, project.name, repository_name) if os.path.exists(path): raise CydraError('Path already exists', path=path) if not os.path.exists(os.path.join(self._base, project.name)): os.mkdir(os.path.join(self._base, project.name)) hg_cmd = subprocess.Popen([self.hgcommand, 'init', path], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, errors = hg_cmd.communicate() if hg_cmd.returncode != 0: if hg_cmd.returncode == 127: # assume this is command not found raise CydraError( 'Command not found encountered while calling hg', stderr=errors) else: raise CydraError('Error encountered while calling hg', stderr=errors, code=hg_cmd.returncode) repository = HgRepository(self.compmgr, self._base, project, repository_name) repository.set_params(**params) repository.sync() # synchronize repository return repository def get_params(self): return [param_description, param_contact]
class ADUsers(Component): implements(IUserAuthenticator) implements(IUserTranslator) def __init__(self): config = self.get_component_config() self.ldap = LdapLookup(**config) if not self.ldap.connect(): raise Exception('Connection failed') def username_to_user(self, username): user = self._ldap_to_user(self.ldap.get_user(username)) if user is None: logger.error("Translation failed for: %s" % username) return user def userid_to_user(self, userid): if userid is None or userid == '*': return User(self.compmgr, '*', username='******', full_name='Guest') user = self._ldap_to_user(self.ldap.get_user(userid)) if user is None: logger.error("Translation failed for: %s" % userid) # since the client was looking for a specific ID, # we return a dummy user object with empty data return User(self.compmgr, userid, full_name='N/A') else: return user def _ldap_to_user(self, data): if data is None: return None dn, userobj = data if 'memberOf' in userobj: groups = [ self._ldap_to_group(self.ldap.get_dn(x)) for x in userobj['memberOf'] ] else: groups = [] return User(self.compmgr, userobj['userPrincipalName'][0], username=userobj['sAMAccountName'][0], full_name=force_unicode(userobj['displayName'][0]), groups=groups) def groupid_to_group(self, groupid): group = self._ldap_to_group(self.ldap.get_group(groupid)) if group is None: logger.error("Group lookup error for %s", groupid) return group def _ldap_to_group(self, data): if data is None: return None dn, groupobj = data return Group(self.compmgr, groupobj['name'][0], name=groupobj['name'][0]) def user_password(self, user, password): if not user or not password: return False try: conn = ldap.initialize(self.get_component_config()['uri']) conn.simple_bind_s(user.userid, password) except ldap.INVALID_CREDENTIALS: logger.exception("Authentication failed") return False return True
class InternalPermissionProvider(Component): """Stores permissions in the project's dict Example:: 'permissions': { '*': {'object': ['read']}, 'user': {'*': ['admin']} } """ implements(IPermissionProvider) MODE_GROUP, MODE_USER = range(2) PERMISSION_ROOT = { MODE_GROUP: 'group_permissions', MODE_USER: '******' } def get_permissions(self, project, user, obj): return self._get_permissions(self.MODE_USER, project, user, obj) def get_group_permissions(self, project, group, obj): return self._get_permissions(self.MODE_GROUP, project, group, obj) def _get_permissions(self, mode, project, subject, obj): if project is None: return {} # no project, no permissions # Resolve root of permissions and translator # depending on what we try to find permroot = self.PERMISSION_ROOT[mode] if mode == self.MODE_USER: translator = self.compmgr.get_user elif mode == self.MODE_GROUP: translator = self.compmgr.get_group else: raise ValueError('Unknown mode') res = {} perms = project.data.get(permroot, {}) # if both subject and obj are None, return all (subject, obj, perm) # copy whole structure to prevent side effects if subject is None and obj is None: for s, objs in perms.items(): s = translator(s) res[s] = {} for o, perm in objs: res[s][o] = perm.copy() # Inject global owner permissions if necessary if mode == self.MODE_USER: res.setdefault(project.owner, {}).setdefault( '*', {}).update(virtual_owner_permissions) return res # construct a list of objects in the hierarchy if obj is not None: objparts = list(object_walker(obj)) objparts.reverse() # if subject is none, find all subjects and return all (subject, perm) # we know here that obj is not none as we handled subject none and obj none # case above if subject is None: for s, p in perms.items(): s = translator(s) res[s] = {} for o in objparts: if o in p: res[s].update(p[o].copy()) # delete empty entries if res[s] == {}: del res[s] # Inject global owner permissions if necessary if mode == self.MODE_USER: res.setdefault(project.owner, {}).update(virtual_owner_permissions) return res # subject is given. # in case of user mode, we also check the guest account subjects = [subject.id] if mode == self.MODE_USER: subjects.append('*') for p in [perms[x] for x in subjects if x in perms]: if obj is not None: for o in objparts: if o in p: res.update(p[o].copy()) else: for o in p: res[o] = p[o].copy() # this is the owner, Inject global owner perms if mode == self.MODE_USER and project.owner == subject: if obj is None: res.setdefault('*', {}).update(virtual_owner_permissions) else: res.update(virtual_owner_permissions) # also inject all group permissions if mode == self.MODE_USER: for group in [x for x in subject.groups if x is not None ]: # safeguard against failing translators res.update(self.get_group_permissions(project, group, obj)) return res def get_permission(self, project, user, obj, permission): return self._get_permission(self.MODE_USER, project, user, obj, permission) def get_group_permission(self, project, group, obj, permission): return self._get_permission(self.MODE_GROUP, project, group, obj, permission) def _get_permission(self, mode, project, subject, obj, permission): if project is None: return None if subject is None: return None if obj is None: return None # Resolve root of permissions and translator # depending on what we try to find permroot = self.PERMISSION_ROOT[mode] if mode == self.MODE_USER: translator = self.compmgr.get_user elif mode == self.MODE_GROUP: translator = self.compmgr.get_group else: raise ValueError('Unknown mode') # the owner can do everything if mode == self.MODE_USER and project.owner == subject: return True perms = project.data.get(permroot, {}) # What we want to find here is a specific permission on a specific # object. First get the most precise. If we have a conflict, return the most positive one ret = None # If we are in user mode, check groups first if mode == self.MODE_USER: for group in subject.groups: ret = self._merge_perm_values( ret, self.get_group_permission(project, group, obj, permission)) # root level -> find subject in perms if subject.id in perms: perms = perms[subject.id] elif mode == self.MODE_USER and '*' in perms: # if we are in user mode, fall back to guest perms = perms['*'] else: return ret # subject level. Now walk the tree. deeper value overwrites lower subjret = None for o in object_walker(obj): if o in perms: # object level perm = perms[o].get(permission, None) if perm is None: perm = perms[o].get('admin', None) if perm is not None: subjret = perm # now merge subjret with the previous value return self._merge_perm_values(ret, subjret) def _merge_perm_values(self, a, b): if a == False or b == False: return False elif a == True or b == True: return True else: return None def set_permission(self, project, user, obj, permission, value=None): return self._set_permission(self.MODE_USER, project, user, obj, permission, value) def set_group_permission(self, project, group, obj, permission, value=None): return self._set_permission(self.MODE_GROUP, project, group, obj, permission, value) def _set_permission(self, mode, project, subject, obj, permission, value=None): if project is None: return None if subject is None: return None if obj is None: return None # Resolve root of permissions depending on what we try to find permroot = self.PERMISSION_ROOT[mode] if value is None: # check if the permission is set, otherwise do nothing if permission in project.data.get(permroot, {}).get(subject.id, {}).get(obj, {}): # remove permission del project.data[permroot][subject.id][obj][permission] if project.data[permroot][subject.id][obj] == {}: del project.data[permroot][subject.id][obj] if project.data[permroot][subject.id] == {}: del project.data[permroot][subject.id] else: project.data.setdefault(permroot, {}).setdefault(subject.id, {}).setdefault( obj, {})[permission] = value project.save() return True def get_projects_user_has_permissions_on(self, user): res = set([ project for project in self.compmgr.get_projects_where_key_exists( ['permissions', user.userid]) if any( project.data.get('permissions', {}).get(user.userid, {}).values()) ]) for group in user.groups: res.update( set([ project for project in self.compmgr.get_projects_where_key_exists( ['group_permissions', group.id]) if any( project.data.get('group_permissions', {}).get( group.id, {}).values()) ])) res.update(self.compmgr.get_projects_owned_by(user)) return res
class SVNRepositories(Component): implements(IRepository) repository_type = 'svn' repository_type_title = 'SVN' def __init__(self): config = self.get_component_config() if 'base' not in config: InsufficientConfiguration(missing='base', component=self.get_component_name()) self._base = config['base'] self.svncommand = config.get('svncommand', 'svnadmin') def get_repositories(self, project): if not os.path.exists(os.path.join(self._base, project.name)): return [] else: return [SVNRepository(self.compmgr, self._base, project)] def get_repository(self, project, repository_name): return SVNRepository(self.compmgr, self._base, project) def can_create(self, project, user=None): if user: return len(self.get_repositories( project)) == 0 and project.get_permission( user, 'repository.svn', 'create') else: return len(self.get_repositories(project)) == 0 def create_repository(self, project, repository_name): if repository_name != project.name: raise CydraError( "SVN Repository name has to be the same as the project's name") path = os.path.join(self._base, project.name) if os.path.exists(path): raise CydraError('Path already exists', path=path) svn_cmd = subprocess.Popen([self.svncommand, 'create', path], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, errors = svn_cmd.communicate() if svn_cmd.returncode != 0: if svn_cmd.returncode == 127: # assume this is command not found raise CydraError( 'Command not found encountered while calling svn', stderr=errors) else: raise CydraError('Error encountered while calling svn', stderr=errors, code=hg_cmd.returncode) # Customize config repository = SVNRepository(self.compmgr, self._base, project) repository.sync() return repository def get_params(self): return []
class MongoDataSource(Component): """Datasource that saves projects into a MongoDB database This datasource encodes keys to allow for '.' in key names. """ implements(IDataSource) implements(IPubkeyStore) def __init__(self): config = self.get_component_config() if 'host' not in config: raise Exception('Host not configured') if 'database' not in config: raise Exception('Database not configured') self.connection = Connection(config['host']) self.database = self.connection[config['database']] if 'user' in config and 'password' in config: self.database.authenticate(config['user'], config['password']) @staticmethod def _encode_key(val, magic='%'): """Helper function to encode . in keys as mongoDB does not allow them""" ret = val ret = ret.replace(magic, magic + '1') ret = ret.replace('.', magic + '2') return ret @staticmethod def _decode_key(val, magic='%'): """Helper function to decode . in keys as mongoDB does not allow them""" ret = val ret = ret.replace(magic + '2', '.') ret = ret.replace(magic + '1', magic) return ret @staticmethod def _process_dict_keys(data, f): if type(data) not in [list, set, dict]: return data ret = type(data)() if isinstance(data, dict): for key, val in data.items(): ret[f(key)] = MongoDataSource._process_dict_keys(val, f) elif isinstance(data, list): for val in data: ret.append(MongoDataSource._process_dict_keys(val, f)) elif isinstance(data, set): for val in data: ret.add(MongoDataSource._process_dict_keys(val, f)) else: raise Exception("The universe exploded") return ret @staticmethod def _encode_dict_keys(data): return MongoDataSource._process_dict_keys(data, MongoDataSource._encode_key) @staticmethod def _decode_dict_keys(data): return MongoDataSource._process_dict_keys(data, MongoDataSource._decode_key) def get_project(self, projectname): # Check name if not is_valid_project_name(projectname): return None project = self.database.projects.find_one({'name': projectname}) if project is not None: return Project(self.compmgr, self._decode_dict_keys(project)) def save_project(self, project): self.database.projects.save(self._encode_dict_keys(project.data)) def create_project(self, projectname, owner): # Check name if not is_valid_project_name(projectname): return None if self.get_project(projectname) is None: self.database.projects.insert({ 'name': projectname, 'owner': owner.userid }) return self.get_project(projectname) def list_projects(self): ret = [] for p in self.database.projects.find(sort=[('name', ASCENDING)]): ret.append(Project(self.compmgr, self._decode_dict_keys(p))) return ret def get_project_names(self): ret = [] for p in self.database.projects.find(fields=['name'], sort=[('name', ASCENDING)]): ret.append(self._decode_dict_keys(p)['name']) return ret def get_projects_owned_by(self, user): if user is None: return [] ret = [] for p in self.database.projects.find({'owner': user.userid}, sort=[('name', ASCENDING)]): ret.append(Project(self.compmgr, self._decode_dict_keys(p))) return ret def get_projects_where_key_exists(self, key): search = {self._encode_key(str(key)): {'$exists': True}} if isinstance(key, list): search = {'.'.join(map(self._encode_key, key)): {'$exists': True}} ret = [] for p in self.database.projects.find(search, sort=[('name', ASCENDING)]): ret.append(Project(self.compmgr, self._decode_dict_keys(p))) return ret def get_pubkeys(self, user): """Does user have a pubkey with blob""" return self.database.pubkeys.find({'userid': user.userid}, sort=[('name', ASCENDING)]) def user_has_pubkey(self, user, blob): return bool( self.database.pubkeys.find_one({ 'blob': binary.Binary(blob), 'userid': user.userid })) def add_pubkey(self, user, blob, name="unnamed", fingerprint=""): """Add a new public key for a user""" if self.user_has_pubkey(user, blob): return False self.database.pubkeys.insert({ 'userid': user.userid, 'blob': binary.Binary(blob), 'name': name, 'fingerprint': fingerprint }) return True def remove_pubkey(self, user, **kwargs): spec = {'userid': user.userid} for k, v in kwargs.items(): if k in ['name', 'blob', 'fingerprint']: spec[k] = v if len(spec) > 1: self.database.pubkeys.remove(spec) return True
class FileDataSource(Component): """Datasource that saves projects into files """ implements(IDataSource) def __init__(self): config = self.get_component_config() if 'base' not in config: raise InsufficientConfiguration( missing='base', component=self.get_component_name()) self._base = config['base'] def _get_project_path(self, name): return os.path.join(self._base, name + '.yaml') def get_project(self, projectname): # Check name if not is_valid_project_name(projectname): return None path = self._get_project_path(projectname) if os.path.exists(path): with open(path, 'r') as f: return Project(self.compmgr, yaml.safe_load(f)) def save_project(self, project): # Check name if not is_valid_project_name(project.name): return None path = self._get_project_path(project.name) with open(path, 'w') as f: yaml.safe_dump(project.data, f) def create_project(self, projectname, owner): # Check name if not is_valid_project_name(projectname): return None if self.get_project(projectname) is None: path = self._get_project_path(projectname) with open(path, 'w') as f: yaml.safe_dump({'name': projectname, 'owner': owner.userid}, f) return self.get_project(projectname) def delete_project(self, project): # Check name if not is_valid_project_name(project.name): return None path = self._get_project_path(project.name) os.remove(path) def list_projects(self): ret = [] for filename in os.listdir(self._base): if filename.endswith(".yaml"): ret.append(self.get_project(filename[:-len(".yaml")])) return ret def get_project_names(self): ret = [] for filename in os.listdir(self._base): if filename.endswith(".yaml"): ret.append(filename[:-len(".yaml")]) return ret def get_projects_owned_by(self, user): if user is None: return [] ret = [] for project in self.list_projects(): if project.owner == user: ret.append(project) return ret def get_projects_where_key_exists(self, key): ret = [] for project in self.list_projects(): if isinstance(key, list): look_in = project.data found = True for component in key: if component not in look_in: found = False break look_in = look_in[component] if found: ret.append(project) else: if str(key) in project.data: ret.append(project) return ret
class TracEnvironments(Component): """Cydra component for trac support""" implements(ICliProjectCommandProvider) implements(IRepositoryObserver) implements(IBlueprintProvider) implements(IProjectActionProvider) implements(IRepositoryActionProvider) implements(IProjectFeaturelistItemProvider) implements(IProjectSyncParticipant) # this map might look silly right now, but it is not guaranteed that cydra and trac name # repository types the same typemap = {'svn': 'svn', 'hg': 'hg', 'git': 'git'} def __init__(self): """Initialize Trac component This will raise an exception if the base path for the environments has not been configured""" if 'base' not in self.component_config: raise Exception("trac environments base path not configured") def get_env_path(self, project): return os.path.join(self.component_config['base'], project.name) def get_default_options(self, project): """Get the default set of options Cydra enforces These options override everything""" return [ ('project', 'name', project.name), ('trac', 'database', 'sqlite:db/trac.db' ), #TODO: implement the possibility to override this ('trac', 'repository_sync_per_request', ''), # We use hooks, don't sync per request ('trac', 'permission_policies', 'CydraPermissionPolicy'), ('components', 'tracext.hg.*', 'enabled'), ('components', 'tracext.git.*', 'enabled'), ('components', 'cydraplugins.*', 'enabled'), ('git', 'cached_repository', 'true'), ('header_logo', 'src', 'common/trac_banner.png'), # Perhaps let user set a custom one ] def has_env(self, project): """Does the project contain a Trac environment""" return os.path.exists(self.get_env_path(project)) def create(self, project): """Create a Trac environment for the project This will NOT call trac-admin but interfaces with Trac's API directly. It performs the same steps as trac-admin""" if self.has_env(project): return True # When creating environments, you can supply a list # of (section, option, value) tuples to trac which serve as a default options = self.get_default_options(project) # If an (inherit, file, xy) option is present, trac will omit the default values if 'inherit_config' in self.component_config: options.append( ('inherit', 'file', self.component_config['inherit_config'])) try: # create environment env = Environment(self.get_env_path(project), create=True, options=options) # preload wiki pages wiki_pages = None if 'preload_wiki' in self.component_config: wiki_pages = self.component_config['preload_wiki'] else: try: wiki_pages = pkg_resources.resource_filename( 'trac.wiki', 'default-pages') except Exception as e: logger.exception( "Exception while trying to find wiki pages") wiki_pages = None if wiki_pages: try: WikiAdmin(env).load_pages(wiki_pages) except Exception as e: logger.exception("Unable to load wiki pages from %s", wiki_pages) else: logger.warning("Not wiki pages found for preloading") # all done return True except Exception as e: logger.exception( "Caught exception while creating Trac environment in " + self.get_env_path(project)) return False def sync_project(self, project): """For project.ISyncParticipant""" self.sync(project) def sync(self, project): """Sync the trac environment with cydra This sets the options returned by ``get_default_options`` and adds Trac's own defaults if necessary""" if not self.has_env(project): logger.warning('Project %s has no Trac Environment to sync', project.name) return tracini = os.path.join(self.get_env_path(project), 'conf', 'trac.ini') options = self.get_default_options(project) # if inherit is enabled, the default values are supposed to be in # the inherited file. Thus, we can truncate the config file to get a bare minimum if 'inherit_config' in self.component_config: options.append( ('inherit', 'file', self.component_config['inherit_config'])) with open(tracini, 'w') as f: f.truncate() # re-create the configuration file config = Configuration(tracini) for section, name, value in options: config.set(section, name, value) config.save() # load defaults if not any((section, option) == ('inherit', 'file') for section, option, value in options): config.set_defaults() config.save() # check if repositories in cydra match repositories in trac env = Environment(self.get_env_path(project)) rm = RepositoryManager(env) trac_repos = rm.get_real_repositories() trac_repo_names = [r.reponame for r in trac_repos] for repotype, repos in project.data.get('plugins', {}).get('trac', {}).items(): for repo, tracname in (repos or {}).items(): if tracname not in trac_repo_names: logger.warning( "Removing trac mapping from cydra for %s repo %s", repo, tracname) del repos[repo] if not repos: del project.data.get('plugins', {}).get('trac', {})[repotype] # Now do the reverse revmap = dict([(y, x) for (x, y) in self.typemap.items()]) for repo in trac_repos: logger.debug('Looking at trac repo %s', repo.reponame) try: baseparts = repo.get_base().split( ':' ) # This is extremely naiive and possibly breaks some time repotype, path = baseparts[0], baseparts[-1] except: logger.error("Unable to parse: " + repo.get_base()) reponame = os.path.basename(path) if repotype == 'git': reponame = reponame[:-4] try: repository = project.get_repository(revmap[repotype], reponame) except: logger.error("Unable to locate %s %s (%s)", repotype, reponame, path) repository = None logger.debug('Cydra repo %r', repository) if repository: # set this mapping if not there already project.data.setdefault('plugins', {}).setdefault( 'trac', {}).setdefault(repository.type, {})[repository.name] = repo.reponame logger.info('Setting trac mapping for %s %s -> %s', repository.type, repository.name, repo.reponame) else: logger.error("Unable to load %s %s (%s)", revmap[repotype], reponame, path) project.save() def register_repository(self, repository, name=None): """Register a repository with trac""" project = repository.project tracname = name if name is not None else repository.name if repository.name in project.data.get('plugins', {}).get('trac', {}).get( repository.type, {}): logger.error( "Repository %s:%s is already registered in project %s", repository.type, repository.name, project.name) return False if repository.type not in self.typemap: logger.error("Repository type %s is not supported in Trac", repository.type) return False if not self.has_env(project): logger.warning( "Tried to add repository %s:%s to Trac of project %s, but there is no environment", repository.type, repository.name, project.name) return False try: env = Environment(self.get_env_path(project)) DbRepositoryProvider(env).add_repository( tracname, repository.path, self.typemap[repository.type]) # save mapping in project project.data.setdefault('plugins', {}).setdefault('trac', {}).setdefault( repository.type, {})[repository.name] = tracname project.save() # Synchronise repository rm = RepositoryManager(env) repos = rm.get_repository(tracname) repos.sync(lambda rev: logger.debug("Synced revision: %s", rev), clean=True) return True except Exception as e: logger.exception( "Exception occured while addingrepository %s:%s to Trac of project %s", repository.type, repository.name, project.name) return False def get_cli_project_commands(self): return [('trac', self.cli_command)] def cli_command(self, project, args): """Manipulate the trac environment for a project Supported arguments: - create: creates an environment - sync: Synchronizes the configuration with Cydra's requirements - addrepo <type> <name> [tracname]: adds the repository to trac, identified by tracname - updatedefaults <file>: Adds trac's default options to config""" if len(args) < 1 or args[0] not in [ 'create', 'addrepo', 'sync', 'updatedefaults' ]: print self.cli_command.__doc__ return if args[0] == 'create': if self.has_env(project): print "Project already has a Trac environment!" return if self.create(project): print "Environment created" else: print "Creation failed!" elif args[0] == 'sync': self.sync(project) print project.name, "synced" elif args[0] == 'addrepo': if len(args) < 3: print self.cli_command.__doc__ return repository = project.get_repository(args[1], args[2]) if not repository: print "Unknown repository" return ret = False if len(args) == 4: ret = self.register_repository(repository, args[3]) else: ret = self.register_repository(repository) if ret: print "Successfully added repository" else: print "Adding repository failed!" elif args[0] == 'updatedefaults': if len(args) < 2: print self.cli_command.__doc__ return config = Configuration(args[1]) # load defaults config.set_defaults() config.save() def get_blueprint(self): """Get the blueprint for trac actions in the web interface""" from flask import Blueprint, render_template, abort, redirect, url_for, flash, request, jsonify, current_app from cydra.web.wsgihelper import InsufficientPermissions from werkzeug.exceptions import NotFound from werkzeug.local import LocalProxy blueprint = Blueprint('trac', __name__, static_folder='static', template_folder='templates') cydra_instance = LocalProxy(lambda: current_app.config['cydra']) cydra_user = LocalProxy(lambda: request.environ['cydra_user']) @blueprint.route('/project/<projectname>/trac/create', methods=['POST']) def create(projectname): project = cydra_instance.get_project(projectname) if project is None: raise NotFound('Unknown project') if not project.get_permission(cydra_user, '*', 'admin'): raise InsufficientPermissions() self.create(project) return redirect( url_for('frontend.project', projectname=projectname)) @blueprint.route( '/project/<projectname>/trac/register_repository/<repositorytype>/<repositoryname>', methods=['POST']) def register_repository(projectname, repositorytype, repositoryname): project = cydra_instance.get_project(projectname) if project is None: raise NotFound('Unknown project') if not project.get_permission(cydra_user, '*', 'admin'): raise InsufficientPermissions() repository = project.get_repository(repositorytype, repositoryname) if repository is None: raise NotFound('Unknown repository') self.register_repository(repository) return redirect( url_for('frontend.project', projectname=projectname)) return blueprint def get_project_featurelist_items(self, project): if not self.has_env(project): return if 'url_base' not in self.component_config: logger.warning("url_base is not configured!") return ('Trac', []) else: return ('Trac', [{ 'href': self.component_config['url_base'] + '/' + project.name, 'name': 'view' }]) def get_project_actions(self, project): if not self.has_env(project): return ('Create Trac', 'trac.create', 'post') def get_repository_actions(self, repository): if self.has_env(repository.project): if repository.name not in repository.project.data.get( 'plugins', {}).get('trac', {}).get(repository.type, {}): return ('Register in Trac', 'trac.register_repository', 'post') else: pass # TODO: deregister def repository_change_commit(self, repository, revisions): self._changeset_event(repository, 'changeset_added', revisions) # IRepositoryObserver def repository_post_commit(self, repository, revisions): self._changeset_event(repository, 'changeset_modified', revisions) # IRepositoryObserver def pre_delete_repository(self, repository): tracrepo = repository.project.data.get('plugins', {}).get( 'trac', {}).get(repository.type, {}).get(repository.name) if tracrepo and self.has_env(repository.project): # remove it logger.info("Removing repository %s from Trac environment %s", tracrepo, repository.project.name) env = Environment(self.get_env_path(repository.project)) DbRepositoryProvider(env).remove_repository(tracrepo) # clean up del repository.project.data['plugins']['trac'][repository.type][ repository.name] if not repository.project.data['plugins']['trac'][repository.type]: del repository.project.data['plugins']['trac'][repository.type] def _changeset_event(self, repository, event, revisions): project = repository.project if not self.has_env(project): return tracrepos = project.data.get('plugins', {}).get('trac', {}).get(repository.type, {}).get(repository.name) if tracrepos: try: env = Environment(self.get_env_path(project)) RepositoryManager(env).notify(event, tracrepos, revisions) except Exception as e: logger.exception( "Exception occured while calling post-commit hook for revs %s on repository %s (%s) in project %s", revisions, repository.name, repository.type, project.name)
class StaticPermissionProvider(DictBasedPermissionProvider): """Handle permissions defined in config Global (ie. non project specific) permissions can be defined:: 'global_user_permissions': {'*': {'projects': {'create': True}}, 'user1': {'projects': {'create': True}}} 'global_group_permissions': {'*': {'projects': {'create': True}}} As well as project specific/for all projects:: 'user_permissions': {'*': {'user1': {'*': {'admin': True}}}} 'group_permissions': {'proj1': {'group1': {'repository.svn.*': {'admin': True}}}} Note that the user '*' is restricted to any logged-in users to prevent unintentional access rights for non-logged-in guests.""" implements(IPermissionProvider) def _get_user_base(self, project, user): if user is not None and user.is_guest: return {} if project is None: return self.component_config.get('global_user_permissions', {}) if project.name in self.component_config.get('user_permissions', {}): return self.component_config.get('user_permissions')[project.name] return self.component_config.get('user_permissions', {}).get('*', {}) def _get_group_base(self, project): if project is None: return self.component_config.get('global_group_permissions', {}) if project.name in self.component_config.get('group_permissions', {}): return self.component_config.get('group_permissions')[project.name] return self.component_config.get('group_permissions', {}).get('*', {}) def get_permissions(self, project, user, obj): base = self._get_user_base(project, user) return self._get_permissions(base, self.compmgr.get_user, project, user, obj) def get_group_permissions(self, project, group, obj): base = self._get_group_base(project) return self._get_permissions(base, self.compmgr.get_group, project, group, obj) def get_permission(self, project, user, obj, permission): base = self._get_user_base(project, user) return self._get_permission(base, project, user, obj, permission) def get_group_permission(self, project, group, obj, permission): base = self._get_group_base(project) return self._get_permission(base, project, group, obj, permission) def get_projects_user_has_permissions_on(self, user): projects = set() for projectid, perms in self.component_config.get( 'user_permissions', {}): if user.id in perms or '*' in perms: if projectid == '*': pass # TODO else: projects.add(projectid) for projectid, perms in self.component_config.get( 'group_permissions', {}): if any(group.id in perms for group in user.groups): if projectid == '*': pass # TODO else: projects.add(projectid) return [self.compmgr.get_project(x) for x in projects] # Set operations are not supported... def set_permission(self, project, user, obj, permission, value=None): return None def set_group_permission(self, project, group, obj, permission, value=None): return None
class DictBasedPermissionProvider(Component): """Base for permissions providers using a dict for storage""" implements(IPermissionProvider) abstract = True def get_permissions(self, project, user, obj): pass def get_group_permissions(self, project, group, obj): pass def _get_permissions(self, permission_dict, subject_translator, project, subject, obj): """Return permissions based on""" res = {} # if both subject and obj are None, return all (subject, obj, perm) # copy whole structure to prevent side effects if subject is None and obj is None: for s, objs in permission_dict.items(): s = subject_translator(s) res[s] = {} for o, perm in objs: res[s][o] = perm.copy() return res # construct a list of objects in the hierarchy if obj is not None: objparts = list(object_walker(obj)) objparts.reverse() # if subject is none, find all subjects and return all (subject, perm) # we know here that obj is not none as we handled subject none and obj none # case above if subject is None: for s, p in permission_dict.items(): s = subject_translator(s) res[s] = {} for o in objparts: if o in p: res[s].update(p[o].copy()) # delete empty entries if res[s] == {}: del res[s] return res # subject is given. # in case of user mode, we also check the guest account subjects = [subject.id] if isinstance(subject, User): subjects.append('*') for p in [ permission_dict[x] for x in subjects if x in permission_dict ]: if obj is not None: for o in objparts: if o in p: res.update(p[o].copy()) else: for o in p: res[o] = p[o].copy() # also inject all group permissions if isinstance(subject, User): for group in [x for x in subject.groups if x is not None ]: # safeguard against failing translators res.update(self.get_group_permissions(project, group, obj)) return res def _get_permission(self, permission_dict, project, subject, obj, permission): if subject is None or obj is None or permission is None: return None subject_translator = self.compmgr.get_group if isinstance( subject, Group) else self.compmgr.get_user permissions = self._get_permissions(permission_dict, subject_translator, project, subject, None) for o in object_walker(obj): if o in permissions: return permissions[o].get(permission) return None
class HtpasswdUsers(Component): implements(IUserAuthenticator) implements(IUserTranslator) def __init__(self): config = self.get_component_config() if 'file' not in config: raise InsufficientConfiguration( missing='file', component=self.get_component_name()) self.users = {} with open(config['file'], 'r') as f: for line in f.readlines(): user, hash = line.strip().split(':', 1) self.users[user] = hash def username_to_user(self, username): if username in self.users: return User(self.compmgr, username, username=username, full_name=username) def userid_to_user(self, userid): if userid is None or userid == '*': warnings.warn( "You should not call this directly. Use cydra.get_user()", DeprecationWarning, stacklevel=2) return User(self.compmgr, '*', username='******', full_name='Guest') if userid in self.users: return User(self.compmgr, userid, username=userid, full_name=userid) else: # since the client was looking for a specific ID, # we return a dummy user object with empty data return User(self.compmgr, userid, full_name='N/A') def groupid_to_group(self, groupid): pass def user_password(self, user, password): if not user.userid in self.users: return hash = self.users[user.userid] if hash.startswith('{SHA}'): pass elif hash.startswith('$apr1$'): pass else: import crypt if hash == crypt.crypt(password, hash[:2]): return True return False
class GitRepositories(Component): """Component for git based repositories Configuration: - base: Path to the directory where repositories are stored - gitcommand: Path to git command. Defaults to git""" implements(IRepository) repository_type = 'git' repository_type_title = 'Git' def __init__(self): config = self.get_component_config() if 'base' not in config: raise InsufficientConfiguration( missing='base', component=self.get_component_name()) self._base = config['base'] self.gitcommand = config.get('gitcommand', 'git') def get_repositories(self, project): """Returns a list of repositories for the project This list is based on the filesystem""" if not os.path.exists(os.path.join(self._base, project.name)): return [] return [ GitRepository(self.compmgr, self._base, project, x[:-4]) for x in os.listdir(os.path.join(self._base, project.name)) if x[-4:] == '.git' ] def get_repository(self, project, repository_name): """Return a GitRepository instance for a repository""" if not self._repo_exists(project, repository_name): return else: return GitRepository(self.compmgr, self._base, project, repository_name) def can_create(self, project, user=None): """Returns whether the user create a new repository""" if user: return project.get_permission(user, 'repository.git', 'create') else: return True def create_repository(self, project, repository_name, **params): """Create a new git repository A repository's name can only contain letters, numbers and dashes/underscores.""" if not is_valid_repository_name(repository_name): raise CydraError("Invalid Repository Name", name=repository_name) path = os.path.join(self._base, project.name, repository_name + '.git') if os.path.exists(path): raise CydraError('Path already exists', path=path) if not os.path.exists(os.path.join(self._base, project.name)): os.mkdir(os.path.join(self._base, project.name)) git_cmd = subprocess.Popen([self.gitcommand, 'init', '--bare', path], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, errors = git_cmd.communicate() if git_cmd.returncode != 0: if git_cmd.returncode == 127: # assume this is command not found raise CydraError( 'Command not found encountered while calling git', stderr=errors) else: raise CydraError('Error encountered while calling git', stderr=errors, code=git_cmd.returncode) # Customize config repository = GitRepository(self.compmgr, self._base, project, repository_name) repository.set_params(**params) repository.sync() # synchronize repository return repository def get_params(self): return [param_description] def _repo_exists(self, project, name): if not is_valid_repository_name(name): raise CydraError("Invalid Repository Name", name=name) path = os.path.join(self._base, project.name, name + '.git') return os.path.exists(path)