def _dispatch_call(self): """ Implement dispatch interface specified by WSGIController """ raw_response = '' try: raw_response = self._inspect_call(self._func) if isinstance(raw_response, HTTPError): self._error = str(raw_response) except JSONRPCError as e: self._error = safe_str(e) except Exception as e: log.error('Encountered unhandled exception: %s', traceback.format_exc(),) json_exc = JSONRPCError('Internal server error') self._error = safe_str(json_exc) if self._error is not None: raw_response = None response = dict(id=self._req_id, result=raw_response, error=self._error) try: return json.dumps(response) except TypeError as e: log.error('API FAILED. Error encoding response: %s', e) return json.dumps( dict( id=self._req_id, result=None, error="Error encoding response" ) )
def _rpc_call(self, action, environ, **rpc_args): """ Call the specified RPC Method """ raw_response = '' try: raw_response = getattr(self, action)(**rpc_args) if isinstance(raw_response, HTTPError): self._error = str(raw_response) except JSONRPCError as e: self._error = safe_str(e) except Exception as e: log.error('Encountered unhandled exception: %s', traceback.format_exc(),) json_exc = JSONRPCError('Internal server error') self._error = safe_str(json_exc) if self._error is not None: raw_response = None response = dict(id=self._req_id, result=raw_response, error=self._error) try: return json.dumps(response) except TypeError as e: log.error('API FAILED. Error encoding response: %s', e) return json.dumps( dict( id=self._req_id, result=None, error="Error encoding response" ) )
def is_valid_repo_group(repo_group_name, base_path, skip_path_check=False): """ Returns True if given path is a repository group False otherwise :param repo_name: :param base_path: """ full_path = os.path.join(safe_str(base_path), safe_str(repo_group_name)) # check if it's not a repo if is_valid_repo(repo_group_name, base_path): return False try: # we need to check bare git repos at higher level # since we might match branches/hooks/info/objects or possible # other things inside bare git repo get_scm(os.path.dirname(full_path)) return False except VCSError: pass # check if it's a valid path if skip_path_check or os.path.isdir(full_path): return True return False
def commit_change(self, repo, repo_name, cs, user, author, message, content, f_path): """ Commit a change to a single file :param repo: a db_repo.scm_instance """ user = self._get_user(user) IMC = self._get_IMC_module(repo.alias) # decoding here will force that we have proper encoded values # in any other case this will throw exceptions and deny commit content = safe_str(content) path = safe_str(f_path) # message and author needs to be unicode # proper backend should then translate that into required type message = safe_unicode(message) author = safe_unicode(author) imc = IMC(repo) imc.change(FileNode(path, content, mode=cs.get_file_mode(f_path))) try: tip = imc.commit(message=message, author=author, parents=[cs], branch=cs.branch) except Exception as e: log.error(traceback.format_exc()) raise IMCCommitError(str(e)) finally: # always clear caches, if commit fails we want fresh object also self.mark_for_invalidation(repo_name) self._handle_push(repo, username=user.username, action='push_local', repo_name=repo_name, revisions=[tip.raw_id]) return tip
def __init__(self, server, base_dn, port=None, bind_dn='', bind_pass='', tls_kind='LDAPS', tls_reqcert='DEMAND', cacertdir=None, ldap_version=3, ldap_filter='(&(objectClass=user)(!(objectClass=computer)))', search_scope='SUBTREE', attr_login='******'): if ldap is None: raise LdapImportError self.ldap_version = ldap_version self.TLS_KIND = tls_kind OPT_X_TLS_DEMAND = 2 self.TLS_REQCERT = getattr(ldap, 'OPT_X_TLS_%s' % tls_reqcert, OPT_X_TLS_DEMAND) self.cacertdir = cacertdir protocol = 'ldaps' if self.TLS_KIND == 'LDAPS' else 'ldap' if not port: port = 636 if self.TLS_KIND == 'LDAPS' else 389 self.LDAP_SERVER = str(', '.join( "%s://%s:%s" % (protocol, host.strip(), port) for host in server.split(','))) self.LDAP_BIND_DN = safe_str(bind_dn) self.LDAP_BIND_PASS = safe_str(bind_pass) self.BASE_DN = safe_str(base_dn) self.LDAP_FILTER = safe_str(ldap_filter) self.SEARCH_SCOPE = getattr(ldap, 'SCOPE_%s' % search_scope) self.attr_login = attr_login
def commit_change(self, repo, repo_name, cs, user, author, message, content, f_path): """ Commits changes :param repo: SCM instance """ user = self._get_user(user) IMC = self._get_IMC_module(repo.alias) # decoding here will force that we have proper encoded values # in any other case this will throw exceptions and deny commit content = safe_str(content) path = safe_str(f_path) # message and author needs to be unicode # proper backend should then translate that into required type message = safe_unicode(message) author = safe_unicode(author) imc = IMC(repo) imc.change(FileNode(path, content, mode=cs.get_file_mode(f_path))) try: tip = imc.commit(message=message, author=author, parents=[cs], branch=cs.branch) except Exception, e: log.error(traceback.format_exc()) raise IMCCommitError(str(e))
def test_get_repo(self): alias = 'hg' path = TEST_HG_REPO backend = get_backend(alias) repo = backend(safe_str(path)) self.assertEqual(repo.__class__, get_repo(safe_str(path), alias).__class__) self.assertEqual(repo.path, get_repo(safe_str(path), alias).path)
def test_get_repo_autoalias_git(self): alias = 'git' path = TEST_GIT_REPO backend = get_backend(alias) repo = backend(safe_str(path)) self.assertEqual(repo.__class__, get_repo(safe_str(path)).__class__) self.assertEqual(repo.path, get_repo(safe_str(path)).path)
def log_push_action(ui, repo, **kwargs): """ Register that changes have been pushed. Mercurial invokes this directly as a hook, git uses handle_git_receive. """ ex = _extract_extras() action_tmpl = ex.action + ':%s' revs = [] if ex.scm == 'hg': node = kwargs['node'] def get_revs(repo, rev_opt): if rev_opt: revs = revrange(repo, rev_opt) if len(revs) == 0: return (nullrev, nullrev) return max(revs), min(revs) else: return len(repo) - 1, 0 stop, start = get_revs(repo, [node + ':']) _h = binascii.hexlify revs = [_h(repo[r].node()) for r in xrange(start, stop + 1)] elif ex.scm == 'git': revs = kwargs.get('_git_revs', []) if '_git_revs' in kwargs: kwargs.pop('_git_revs') action = action_tmpl % ','.join(revs) action_logger(ex.username, action, ex.repository, ex.ip, commit=True) # extension hook call from kallithea import EXTENSIONS callback = getattr(EXTENSIONS, 'PUSH_HOOK', None) if callable(callback): kw = {'pushed_revs': revs} kw.update(ex) callback(**kw) if ex.make_lock is not None and not ex.make_lock: Repository.unlock(Repository.get_by_repo_name(ex.repository)) ui.status(safe_str('Released lock on repo `%s`\n' % ex.repository)) if ex.locked_by[0]: locked_by = User.get(ex.locked_by[0]).username _http_ret = HTTPLockedRC(ex.repository, locked_by) if str(_http_ret.code).startswith('2'): #2xx Codes don't raise exceptions ui.status(safe_str(_http_ret.title)) return 0
def checkSessionFlash(self, response, msg, skip=0): if 'flash' not in response.session: self.fail(safe_str(u'msg `%s` not found - session has no flash ' % msg)) try: level, m = response.session['flash'][-1 - skip] if msg in m: return except IndexError: pass self.fail(safe_str(u'msg `%s` not found in session flash (skipping %s): %s' % (msg, skip, ', '.join('`%s`' % m for level, m in response.session['flash']))))
def take_action(self, args): _caches = CacheInvalidation.query().order_by(CacheInvalidation.cache_key).all() if args.show: for c_obj in _caches: print 'key:%s active:%s' % (safe_str(c_obj.cache_key), c_obj.cache_active) elif args.cleanup: for c_obj in _caches: Session().delete(c_obj) print 'Removing key: %s' % (safe_str(c_obj.cache_key)) Session().commit() else: print 'Nothing done, exiting...'
def checkSessionFlash(self, response, msg=None, skip=0, _matcher=lambda msg, m: msg in m): if 'flash' not in response.session: pytest.fail(safe_str(u'msg `%s` not found - session has no flash:\n%s' % (msg, response))) try: level, m = response.session['flash'][-1 - skip] if _matcher(msg, m): return except IndexError: pass pytest.fail(safe_str(u'msg `%s` not found in session flash (skipping %s): %s' % (msg, skip, ', '.join('`%s`' % m for level, m in response.session['flash']))))
def command(self): #get SqlAlchemy session self._init_session() repos_location = Ui.get_repos_location() to_remove = [] for dn, dirs, f in os.walk(safe_str(repos_location)): alldirs = list(dirs) del dirs[:] if ('.hg' in alldirs or 'objects' in alldirs and ('refs' in alldirs or 'packed-refs' in f)): continue for loc in alldirs: if REMOVED_REPO_PAT.match(loc): to_remove.append([os.path.join(dn, loc), self._extract_date(loc)]) else: dirs.append(loc) #filter older than (if present)! now = datetime.datetime.now() older_than = self.options.older_than if older_than: to_remove_filtered = [] older_than_date = self._parse_older_than(older_than) for name, date_ in to_remove: repo_age = now - date_ if repo_age > older_than_date: to_remove_filtered.append([name, date_]) to_remove = to_remove_filtered print >> sys.stdout, 'removing %s deleted repos older than %s (%s)' \ % (len(to_remove), older_than, older_than_date) else: print >> sys.stdout, 'removing all [%s] deleted repos' \ % len(to_remove) if self.options.dont_ask or not to_remove: # don't ask just remove ! remove = True else: remove = ask_ok('the following repositories will be deleted completely:\n%s\n' 'are you sure you want to remove them [y/n]?' % ', \n'.join(['%s removed on %s' % (safe_str(x[0]), safe_str(x[1])) for x in to_remove])) if remove: for path, date_ in to_remove: print >> sys.stdout, 'removing repository %s' % path shutil.rmtree(path) else: print 'nothing done exiting...' sys.exit(0)
def take_action(self, args): repos_location = Ui.get_repos_location() to_remove = [] for dn_, dirs, f in os.walk(safe_str(repos_location)): alldirs = list(dirs) del dirs[:] if ('.hg' in alldirs or '.git' in alldirs or '.svn' in alldirs or 'objects' in alldirs and ('refs' in alldirs or 'packed-refs' in f)): continue for loc in alldirs: if REMOVED_REPO_PAT.match(loc): to_remove.append([os.path.join(dn_, loc), self._extract_date(loc)]) else: dirs.append(loc) if dirs: print 'Scanning: %s' % dn_ #filter older than (if present)! now = datetime.datetime.now() older_than = args.older_than if older_than: to_remove_filtered = [] older_than_date = self._parse_older_than(older_than) for name, date_ in to_remove: repo_age = now - date_ if repo_age > older_than_date: to_remove_filtered.append([name, date_]) to_remove = to_remove_filtered print 'Removing %s deleted repos older than %s (%s)' \ % (len(to_remove), older_than, older_than_date) else: print 'Removing all %s deleted repos' % len(to_remove) if args.dont_ask or not to_remove: # don't ask just remove ! remove = True else: remove = ask_ok('the following repositories will be deleted completely:\n%s\n' 'are you sure you want to remove them [y/n]?' % '\n'.join(['%s removed on %s' % (safe_str(x[0]), safe_str(x[1])) for x in to_remove])) if remove: for path, date_ in to_remove: print 'Removing repository %s' % path shutil.rmtree(path) else: print 'Nothing done, exiting...'
def test_fork_unicode(self): self.log_user() # create a fork repo_name = self.REPO org_repo = Repository.get_by_repo_name(repo_name) fork_name = safe_str(self.REPO_FORK + u'-rødgrød') creation_args = { 'repo_name': fork_name, 'repo_group': u'-1', 'fork_parent_id': org_repo.repo_id, 'repo_type': self.REPO_TYPE, 'description': 'unicode repo 1', 'private': 'False', 'landing_rev': 'rev:tip', '_authentication_token': self.authentication_token()} self.app.post(url(controller='forks', action='fork_create', repo_name=repo_name), creation_args) response = self.app.get(url(controller='forks', action='forks', repo_name=repo_name)) response.mustcontain( """<a href="/%s">%s</a>""" % (urllib.quote(fork_name), fork_name) ) fork_repo = Repository.get_by_repo_name(safe_unicode(fork_name)) assert fork_repo # fork the fork fork_name_2 = safe_str(self.REPO_FORK + u'-blåbærgrød') creation_args = { 'repo_name': fork_name_2, 'repo_group': u'-1', 'fork_parent_id': fork_repo.repo_id, 'repo_type': self.REPO_TYPE, 'description': 'unicode repo 2', 'private': 'False', 'landing_rev': 'rev:tip', '_authentication_token': self.authentication_token()} self.app.post(url(controller='forks', action='fork_create', repo_name=fork_name), creation_args) response = self.app.get(url(controller='forks', action='forks', repo_name=fork_name)) response.mustcontain( """<a href="/%s">%s</a>""" % (urllib.quote(fork_name_2), fork_name_2) ) # remove these forks response = self.app.post(url('delete_repo', repo_name=fork_name_2), params={'_authentication_token': self.authentication_token()}) response = self.app.post(url('delete_repo', repo_name=fork_name), params={'_authentication_token': self.authentication_token()})
def test_delete_non_ascii(self): self.log_user() non_ascii = "ąęł" repo_name = "%s%s" % (safe_str(self.NEW_REPO), non_ascii) repo_name_unicode = safe_unicode(repo_name) description = 'description for newly created repo' + non_ascii description_unicode = safe_unicode(description) response = self.app.post(url('repos'), fixture._get_repo_create_params(repo_private=False, repo_name=repo_name, repo_type=self.REPO_TYPE, repo_description=description, _authentication_token=self.authentication_token())) ## run the check page that triggers the flash message response = self.app.get(url('repo_check_home', repo_name=repo_name)) self.assertEqual(response.json, {u'result': True}) self.checkSessionFlash(response, u'Created repository <a href="/%s">%s</a>' % (urllib.quote(repo_name), repo_name_unicode)) # test if the repo was created in the database new_repo = Session().query(Repository) \ .filter(Repository.repo_name == repo_name_unicode).one() self.assertEqual(new_repo.repo_name, repo_name_unicode) self.assertEqual(new_repo.description, description_unicode) # test if the repository is visible in the list ? response = self.app.get(url('summary_home', repo_name=repo_name)) response.mustcontain(repo_name) response.mustcontain(self.REPO_TYPE) # test if the repository was created on filesystem try: vcs.get_repo(safe_str(os.path.join(Ui.get_by_key('paths', '/').ui_value, repo_name_unicode))) except vcs.exceptions.VCSError: pytest.fail('no repo %s in filesystem' % repo_name) response = self.app.post(url('delete_repo', repo_name=repo_name), params={'_method': 'delete', '_authentication_token': self.authentication_token()}) self.checkSessionFlash(response, 'Deleted repository %s' % (repo_name_unicode)) response.follow() #check if repo was deleted from db deleted_repo = Session().query(Repository) \ .filter(Repository.repo_name == repo_name_unicode).scalar() self.assertEqual(deleted_repo, None) self.assertEqual(os.path.isdir(os.path.join(Ui.get_by_key('paths', '/').ui_value, repo_name_unicode)), False)
def get_paths(self, repo): """ recursive walk in root dir and return a set of all path in that dir based on repository walk function """ index_paths_ = set() try: cs = self._get_index_changeset(repo) for _topnode, _dirs, files in cs.walk('/'): for f in files: index_paths_.add(jn(safe_str(repo.path), safe_str(f.path))) except RepositoryError: log.debug(traceback.format_exc()) pass return index_paths_
def log_pull_action(ui, repo, **kwargs): """ Logs user last pull action :param ui: :param repo: """ ex = _extract_extras() user = User.get_by_username(ex.username) action = 'pull' action_logger(user, action, ex.repository, ex.ip, commit=True) # extension hook call from kallithea import EXTENSIONS callback = getattr(EXTENSIONS, 'PULL_HOOK', None) if callable(callback): kw = {} kw.update(ex) callback(**kw) if ex.make_lock is not None and ex.make_lock: Repository.lock(Repository.get_by_repo_name(ex.repository), user.user_id) #msg = 'Made lock on repo `%s`' % repository #ui.status(msg) if ex.locked_by[0]: locked_by = User.get(ex.locked_by[0]).username _http_ret = HTTPLockedRC(ex.repository, locked_by) if str(_http_ret.code).startswith('2'): #2xx Codes don't raise exceptions ui.status(safe_str(_http_ret.title)) return 0
def _rename_filesystem_repo(self, old, new): """ renames repository on filesystem :param old: old name :param new: new name """ log.info('renaming repo from %s to %s', old, new) old_path = safe_str(os.path.join(self.repos_path, old)) new_path = safe_str(os.path.join(self.repos_path, new)) if os.path.isdir(new_path): raise Exception( 'Was trying to rename to already existing dir %s' % new_path ) shutil.move(old_path, new_path)
def _get_cache_parameters(query): """For a query with cache_region and cache_namespace configured, return the corresponding Cache instance and cache key, based on this query's current criterion and parameter values. """ if not hasattr(query, '_cache_parameters'): raise ValueError("This Query does not have caching " "parameters configured.") region, namespace, cache_key = query._cache_parameters namespace = _namespace_from_query(namespace, query) if cache_key is None: # cache key - the value arguments from this query's parameters. args = [safe_str(x) for x in _params_from_query(query)] args.extend(filter(lambda k: k not in ['None', None, u'None'], [str(query._limit), str(query._offset)])) cache_key = " ".join(args) if cache_key is None: raise Exception('Cache key cannot be None') # get cache #cache = query.cache_manager.get_cache_region(namespace, region) cache = get_cache_region(namespace, region) # optional - hash the cache_key too for consistent length # import uuid # cache_key= str(uuid.uuid5(uuid.NAMESPACE_DNS, cache_key)) return cache, cache_key
def _get_scm_size(alias, root_path): if not alias.startswith('.'): alias += '.' size_scm, size_root = 0, 0 for path, dirs, files in os.walk(safe_str(root_path)): if path.find(alias) != -1: for f in files: try: size_scm += os.path.getsize(os.path.join(path, f)) except OSError: pass else: for f in files: try: size_root += os.path.getsize(os.path.join(path, f)) except OSError: pass size_scm_f = h.format_byte_size(size_scm) size_root_f = h.format_byte_size(size_root) size_total_f = h.format_byte_size(size_root + size_scm) return size_scm_f, size_root_f, size_total_f
def __get_cs(self, rev, silent_empty=False): """ Safe way to get changeset if error occur it redirects to tip with proper message :param rev: revision to fetch :silent_empty: return None if repository is empty """ try: return c.db_repo_scm_instance.get_changeset(rev) except EmptyRepositoryError as e: if silent_empty: return None url_ = url('files_add_home', repo_name=c.repo_name, revision=0, f_path='', anchor='edit') add_new = h.link_to(_('Click here to add new file'), url_, class_="alert-link") h.flash(h.literal(_('There are no files yet. %s') % add_new), category='warning') raise HTTPNotFound() except (ChangesetDoesNotExistError, LookupError): msg = _('Such revision does not exist for this repository') h.flash(msg, category='error') raise HTTPNotFound() except RepositoryError as e: h.flash(safe_str(e), category='error') raise HTTPNotFound()
def index(self, repo_name, revision, f_path, annotate=False): # redirect to given revision from form if given post_revision = request.POST.get('at_rev', None) if post_revision: cs = self.__get_cs(post_revision) # FIXME - unused! c.revision = revision c.changeset = self.__get_cs(revision) c.branch = request.GET.get('branch', None) c.f_path = f_path c.annotate = annotate cur_rev = c.changeset.revision # prev link try: prev_rev = c.db_repo_scm_instance.get_changeset(cur_rev).prev(c.branch) c.url_prev = url('files_home', repo_name=c.repo_name, revision=prev_rev.raw_id, f_path=f_path) if c.branch: c.url_prev += '?branch=%s' % c.branch except (ChangesetDoesNotExistError, VCSError): c.url_prev = '#' # next link try: next_rev = c.db_repo_scm_instance.get_changeset(cur_rev).next(c.branch) c.url_next = url('files_home', repo_name=c.repo_name, revision=next_rev.raw_id, f_path=f_path) if c.branch: c.url_next += '?branch=%s' % c.branch except (ChangesetDoesNotExistError, VCSError): c.url_next = '#' # files or dirs try: c.file = c.changeset.get_node(f_path) if c.file.is_file(): c.load_full_history = False file_last_cs = c.file.last_changeset c.file_changeset = (c.changeset if c.changeset.revision < file_last_cs.revision else file_last_cs) #determine if we're on branch head _branches = c.db_repo_scm_instance.branches c.on_branch_head = revision in _branches.keys() + _branches.values() _hist = [] c.file_history = [] if c.load_full_history: c.file_history, _hist = self._get_node_history(c.changeset, f_path) c.authors = [] for a in set([x.author for x in _hist]): c.authors.append((h.email(a), h.person(a))) else: c.authors = c.file_history = [] except RepositoryError, e: h.flash(safe_str(e), category='error') raise HTTPNotFound()
def __init__(self, reponame, username, *args, **kwargs): from kallithea import CONFIG from kallithea.lib.utils2 import safe_int _code = CONFIG.get('lock_ret_code') self.code = safe_int(_code, self.code) self.title = self.explanation = safe_str( 'Repository `%s` locked by user `%s`' % (reponame, username)) super(HTTPLockedRC, self).__init__(*args, **kwargs)
def test_repo_clone_without_update(self): repo = MercurialRepository(safe_str(TEST_HG_REPO)) repo_clone = MercurialRepository(TEST_HG_REPO_CLONE + '_wo_update', src_url=TEST_HG_REPO, update_after_clone=False) self.assertEqual(len(repo.revisions), len(repo_clone.revisions)) self.assertEqual(os.path.isfile(os.path.join(TEST_HG_REPO_CLONE \ + '_wo_update', 'MANIFEST.in')), False,)
def make_ui(read_from='file', path=None, checkpaths=True, clear_session=True): """ A function that will read python rc files or database and make an mercurial ui object from read options :param path: path to mercurial config file :param checkpaths: check the path :param read_from: read from 'file' or 'db' """ baseui = ui.ui() # clean the baseui object baseui._ocfg = config.config() baseui._ucfg = config.config() baseui._tcfg = config.config() if read_from == 'file': if not os.path.isfile(path): log.debug('hgrc file is not present at %s, skipping...', path) return False log.debug('reading hgrc from %s', path) cfg = config.config() cfg.read(path) for section in ui_sections: for k, v in cfg.items(section): log.debug('settings ui from file: [%s] %s=%s', section, k, v) baseui.setconfig(safe_str(section), safe_str(k), safe_str(v)) elif read_from == 'db': sa = meta.Session() ret = sa.query(Ui).all() hg_ui = ret for ui_ in hg_ui: if ui_.ui_active: ui_val = safe_str(ui_.ui_value) if ui_.ui_section == 'hooks' and BRAND != 'kallithea' and ui_val.startswith('python:' + BRAND + '.lib.hooks.'): ui_val = ui_val.replace('python:' + BRAND + '.lib.hooks.', 'python:kallithea.lib.hooks.') log.debug('settings ui from db: [%s] %s=%s', ui_.ui_section, ui_.ui_key, ui_val) baseui.setconfig(safe_str(ui_.ui_section), safe_str(ui_.ui_key), ui_val) if ui_.ui_key == 'push_ssl': # force set push_ssl requirement to False, kallithea # handles that baseui.setconfig(safe_str(ui_.ui_section), safe_str(ui_.ui_key), False) if clear_session: meta.Session.remove() # prevent interactive questions for ssh password / passphrase ssh = baseui.config('ui', 'ssh', default='ssh') baseui.setconfig('ui', 'ssh', '%s -oBatchMode=yes -oIdentitiesOnly=yes' % ssh) return baseui
def get_filesystem_repos(path): """ Scans given path for repos and return (name,(type,path)) tuple :param path: path to scan for repositories :param recursive: recursive search and return names with subdirs in front """ # remove ending slash for better results path = safe_str(path.rstrip(os.sep)) log.debug('now scanning in %s', path) def isdir(*n): return os.path.isdir(os.path.join(*n)) for root, dirs, _files in os.walk(path): recurse_dirs = [] for subdir in dirs: # skip removed repos if REMOVED_REPO_PAT.match(subdir): continue #skip .<something> dirs TODO: rly? then we should prevent creating them ... if subdir.startswith('.'): continue cur_path = os.path.join(root, subdir) if isdir(cur_path, '.git'): log.warning('ignoring non-bare Git repo: %s', cur_path) continue if (isdir(cur_path, '.hg') or isdir(cur_path, '.svn') or isdir(cur_path, 'objects') and (isdir(cur_path, 'refs') or os.path.isfile(os.path.join(cur_path, 'packed-refs')))): if not os.access(cur_path, os.R_OK) or not os.access(cur_path, os.X_OK): log.warning('ignoring repo path without access: %s', cur_path) continue if not os.access(cur_path, os.W_OK): log.warning('repo path without write access: %s', cur_path) try: scm_info = get_scm(cur_path) assert cur_path.startswith(path) repo_path = cur_path[len(path) + 1:] yield repo_path, scm_info continue # no recursion except VCSError: # We should perhaps ignore such broken repos, but especially # the bare git detection is unreliable so we dive into it pass recurse_dirs.append(subdir) dirs[:] = recurse_dirs
def _set_cache_parameters(query, region, namespace, cache_key): if hasattr(query, '_cache_parameters'): region, namespace, cache_key = query._cache_parameters raise ValueError("This query is already configured " "for region %r namespace %r" % (region, namespace) ) query._cache_parameters = region, safe_str(namespace), cache_key
def get_node(self, repo, path, index_rev=None): """ gets a filenode based on given full path. It operates on string for hg git compatibility. :param repo: scm repo instance :param path: full path including root location :return: FileNode """ # FIXME: paths should be normalized ... or even better: don't include repo.path path = safe_str(path) repo_path = safe_str(repo.path) assert path.startswith(repo_path) assert path[len(repo_path)] in (os.path.sep, os.path.altsep) node_path = path[len(repo_path) + 1:] cs = self._get_index_changeset(repo, index_rev=index_rev) node = cs.get_node(node_path) return node
def rawfile(self, repo_name, revision, f_path): cs = self.__get_cs(revision) file_node = self.__get_filenode(cs, f_path) response.content_disposition = 'attachment; filename=%s' % \ safe_str(f_path.split(Repository.url_sep())[-1]) response.content_type = file_node.mimetype return file_node.content
def get_crypt_password(password): """ Cryptographic function used for password hashing based on pybcrypt or Python's own OpenSSL wrapper on windows :param password: password to hash """ if is_windows: return hashlib.sha256(password).hexdigest() elif is_unix: import bcrypt return bcrypt.hashpw(safe_str(password), bcrypt.gensalt(10)) else: raise Exception('Unknown or unsupported platform %s' \ % __platform__)
def pre_push(ui, repo, **kwargs): # pre push function, currently used to ban pushing when # repository is locked ex = _extract_extras() usr = User.get_by_username(ex.username) if ex.locked_by[0] and usr.user_id != int(ex.locked_by[0]): locked_by = User.get(ex.locked_by[0]).username # this exception is interpreted in git/hg middlewares and based # on that proper return code is server to client _http_ret = HTTPLockedRC(ex.repository, locked_by) if str(_http_ret.code).startswith('2'): #2xx Codes don't raise exceptions ui.status(safe_str(_http_ret.title)) else: raise _http_ret
def _get_by_id(self, repo_name): """ Gets a special pattern _<ID> from clone url and tries to replace it with a repository_name for support of _<ID> permanent URLs :param repo_name: """ data = repo_name.split('/') if len(data) >= 2: from kallithea.lib.utils import get_repo_by_id by_id_match = get_repo_by_id(repo_name) if by_id_match: data[1] = safe_str(by_id_match) return '/'.join(data)
def plain(cls, source, universal_newline=True): """ >>> MarkupRenderer.plain('https://example.com/') '<br /><a href="https://example.com/">https://example.com/</a>' """ source = safe_str(source) if universal_newline: newline = '\n' source = newline.join(source.splitlines()) def url_func(match_obj): url_full = match_obj.group(0) return '<a href="%(url)s">%(url)s</a>' % ({'url': url_full}) source = url_re.sub(url_func, source) return '<br />' + source.replace("\n", '<br />')
def test_delete(self): self.log_user() repo_name = u'vcs_test_new_to_delete_%s' % self.REPO_TYPE description = u'description for newly created repo' response = self.app.post(url('repos'), fixture._get_repo_create_params(repo_private=False, repo_type=self.REPO_TYPE, repo_name=repo_name, repo_description=description, _authentication_token=self.authentication_token())) ## run the check page that triggers the flash message response = self.app.get(url('repo_check_home', repo_name=repo_name)) self.checkSessionFlash(response, 'Created repository <a href="/%s">%s</a>' % (repo_name, repo_name)) # test if the repo was created in the database new_repo = Session().query(Repository) \ .filter(Repository.repo_name == repo_name).one() assert new_repo.repo_name == repo_name assert new_repo.description == description # test if the repository is visible in the list ? response = self.app.get(url('summary_home', repo_name=repo_name)) response.mustcontain(repo_name) response.mustcontain(self.REPO_TYPE) # test if the repository was created on filesystem try: vcs.get_repo(safe_str(os.path.join(Ui.get_by_key('paths', '/').ui_value, repo_name))) except vcs.exceptions.VCSError: pytest.fail('no repo %s in filesystem' % repo_name) response = self.app.post(url('delete_repo', repo_name=repo_name), params={'_authentication_token': self.authentication_token()}) self.checkSessionFlash(response, 'Deleted repository %s' % (repo_name)) response.follow() #check if repo was deleted from db deleted_repo = Session().query(Repository) \ .filter(Repository.repo_name == repo_name).scalar() assert deleted_repo == None assert os.path.isdir(os.path.join(Ui.get_by_key('paths', '/').ui_value, repo_name)) == False
def is_valid_repo_uri(repo_type, url, ui): """Check if the url seems like a valid remote repo location Raise InvalidCloneUriException if any problems""" if repo_type == 'hg': if url.startswith('http') or url.startswith('ssh'): # initially check if it's at least the proper URL # or does it pass basic auth try: MercurialRepository._check_url(url, ui) except urllib.error.URLError as e: raise InvalidCloneUriException('URI %s URLError: %s' % (url, e)) except mercurial.error.RepoError as e: raise InvalidCloneUriException( 'Mercurial %s: %s' % (type(e).__name__, safe_str(bytes(e)))) elif url.startswith('svn+http'): try: from hgsubversion.svnrepo import svnremoterepo except ImportError: raise InvalidCloneUriException( 'URI type %s not supported - hgsubversion is not available' % (url, )) svnremoterepo(ui, url).svn.uuid elif url.startswith('git+http'): raise InvalidCloneUriException('URI type %s not implemented' % (url, )) else: raise InvalidCloneUriException('URI %s not allowed' % (url, )) elif repo_type == 'git': if url.startswith('http') or url.startswith('git'): # initially check if it's at least the proper URL # or does it pass basic auth try: GitRepository._check_url(url) except urllib.error.URLError as e: raise InvalidCloneUriException('URI %s URLError: %s' % (url, e)) elif url.startswith('svn+http'): raise InvalidCloneUriException('URI type %s not implemented' % (url, )) elif url.startswith('hg+http'): raise InvalidCloneUriException('URI type %s not implemented' % (url, )) else: raise InvalidCloneUriException('URI %s not allowed' % (url))
def __get_cs(rev, repo): """ Safe way to get changeset. If error occur fail with error message. :param rev: revision to fetch :param repo: repo instance """ try: return c.db_repo_scm_instance.get_changeset(rev) except EmptyRepositoryError as e: h.flash(h.literal(_('There are no changesets yet')), category='error') except RepositoryError as e: log.error(traceback.format_exc()) h.flash(safe_str(e), category='error') raise HTTPBadRequest()
def index(self): c.came_from = safe_str(request.GET.get('came_from', '')) if c.came_from: if not self._validate_came_from(c.came_from): log.error('Invalid came_from (not server-relative): %r', c.came_from) raise HTTPBadRequest() else: c.came_from = url('home') ip_allowed = AuthUser.check_ip_allowed(request.authuser, request.ip_addr) # redirect if already logged in if request.authuser.is_authenticated and ip_allowed: raise HTTPFound(location=c.came_from) if request.POST: # import Login Form validator class login_form = LoginForm()() try: c.form_result = login_form.to_python(dict(request.POST)) # form checks for username/password, now we're authenticated username = c.form_result['username'] user = User.get_by_username_or_email(username, case_insensitive=True) except formencode.Invalid as errors: defaults = errors.value # remove password from filling in form again defaults.pop('password', None) return htmlfill.render( render('/login.html'), defaults=errors.value, errors=errors.error_dict or {}, prefix_error=False, encoding="UTF-8", force_defaults=False) except UserCreationError as e: # container auth or other auth functions that create users on # the fly can throw this exception signaling that there's issue # with user creation, explanation should be provided in # Exception itself h.flash(e, 'error') else: log_in_user(user, c.form_result['remember'], is_external_auth=False) raise HTTPFound(location=c.came_from) return render('/login.html')
def update(self, id): c.user_group = UserGroup.get_or_404(id) c.active = 'settings' self.__load_data(id) available_members = [safe_str(x[0]) for x in c.available_members] users_group_form = UserGroupForm(edit=True, old_data=c.user_group.get_dict(), available_members=available_members)() try: form_result = users_group_form.to_python(request.POST) UserGroupModel().update(c.user_group, form_result) gr = form_result['users_group_name'] action_logger(request.authuser, 'admin_updated_users_group:%s' % gr, None, request.ip_addr) h.flash(_('Updated user group %s') % gr, category='success') Session().commit() except formencode.Invalid as errors: ug_model = UserGroupModel() defaults = errors.value e = errors.error_dict or {} defaults.update({ 'create_repo_perm': ug_model.has_perm(id, 'hg.create.repository'), 'fork_repo_perm': ug_model.has_perm(id, 'hg.fork.repository'), }) return htmlfill.render( render('admin/user_groups/user_group_edit.html'), defaults=defaults, errors=e, prefix_error=False, encoding="UTF-8", force_defaults=False) except Exception: log.error(traceback.format_exc()) h.flash(_('Error occurred during update of user group %s') % request.POST.get('users_group_name'), category='error') raise HTTPFound(location=url('edit_users_group', id=id))
def index(self, repo_name, revision=None, f_path=None): limit = 2000 default = 100 if request.GET.get('size'): c.size = max(min(safe_int(request.GET.get('size')), limit), 1) session['changelog_size'] = c.size session.save() else: c.size = int(session.get('changelog_size', default)) # min size must be 1 c.size = max(c.size, 1) p = safe_int(request.GET.get('page', 1), 1) branch_name = request.GET.get('branch', None) if (branch_name and branch_name not in c.db_repo_scm_instance.branches and branch_name not in c.db_repo_scm_instance.closed_branches and not revision): return redirect( url('changelog_file_home', repo_name=c.repo_name, revision=branch_name, f_path=f_path or '')) if revision == 'tip': revision = None c.changelog_for_path = f_path try: if f_path: log.debug('generating changelog for path %s' % f_path) # get the history for the file ! tip_cs = c.db_repo_scm_instance.get_changeset() try: collection = tip_cs.get_file_history(f_path) except (NodeDoesNotExistError, ChangesetError): #this node is not present at tip ! try: cs = self.__get_cs(revision, repo_name) collection = cs.get_file_history(f_path) except RepositoryError, e: h.flash(safe_str(e), category='warning') redirect(h.url('changelog_home', repo_name=repo_name)) collection = list(reversed(collection)) else:
def make_ui(read_from='file', path=None, clear_session=True): """ A function that will read python rc files or database and make an mercurial ui object from read options :param path: path to mercurial config file :param read_from: read from 'file' or 'db' """ baseui = ui.ui() # clean the baseui object baseui._ocfg = config.config() baseui._ucfg = config.config() baseui._tcfg = config.config() if read_from == 'file': if not os.path.isfile(path): log.debug('hgrc file is not present at %s, skipping...', path) return False log.debug('reading hgrc from %s', path) cfg = config.config() cfg.read(path) for section in ui_sections: for k, v in cfg.items(section): log.debug('settings ui from file: [%s] %s=%s', section, k, v) baseui.setconfig(safe_str(section), safe_str(k), safe_str(v)) elif read_from == 'db': sa = meta.Session() ret = sa.query(Ui).all() hg_ui = ret for ui_ in hg_ui: if ui_.ui_active: ui_val = '' if ui_.ui_value is None else safe_str(ui_.ui_value) log.debug('settings ui from db: [%s] %s=%r', ui_.ui_section, ui_.ui_key, ui_val) baseui.setconfig(safe_str(ui_.ui_section), safe_str(ui_.ui_key), ui_val) if clear_session: meta.Session.remove() # force set push_ssl requirement to False, Kallithea handles that baseui.setconfig('web', 'push_ssl', False) baseui.setconfig('web', 'allow_push', '*') # prevent interactive questions for ssh password / passphrase ssh = baseui.config('ui', 'ssh', default='ssh') baseui.setconfig('ui', 'ssh', '%s -oBatchMode=yes -oIdentitiesOnly=yes' % ssh) return baseui
def raw(self, repo_name, revision, f_path): cs = self.__get_cs(revision) file_node = self.__get_filenode(cs, f_path) raw_mimetype_mapping = { # map original mimetype to a mimetype used for "show as raw" # you can also provide a content-disposition to override the # default "attachment" disposition. # orig_type: (new_type, new_dispo) # show images inline: 'image/x-icon': ('image/x-icon', 'inline'), 'image/png': ('image/png', 'inline'), 'image/gif': ('image/gif', 'inline'), 'image/jpeg': ('image/jpeg', 'inline'), 'image/svg+xml': ('image/svg+xml', 'inline'), } mimetype = file_node.mimetype try: mimetype, dispo = raw_mimetype_mapping[mimetype] except KeyError: # we don't know anything special about this, handle it safely if file_node.is_binary: # do same as download raw for binary files mimetype, dispo = 'application/octet-stream', 'attachment' else: # do not just use the original mimetype, but force text/plain, # otherwise it would serve text/html and that might be unsafe. # Note: underlying vcs library fakes text/plain mimetype if the # mimetype can not be determined and it thinks it is not # binary.This might lead to erroneous text display in some # cases, but helps in other cases, like with text files # without extension. mimetype, dispo = 'text/plain', 'inline' if dispo == 'attachment': dispo = 'attachment; filename=%s' % \ safe_str(f_path.split(os.sep)[-1]) response.content_disposition = dispo response.content_type = mimetype return file_node.content
def show(self, gist_id, revision='tip', format='html', f_path=None): c.gist = Gist.get_or_404(gist_id) if c.gist.is_expired: log.error('Gist expired at %s', time_to_datetime(c.gist.gist_expires)) raise HTTPNotFound() try: c.file_changeset, c.files = GistModel().get_gist_files( gist_id, revision=revision) except VCSError: log.error(traceback.format_exc()) raise HTTPNotFound() if format == 'raw': content = '\n\n'.join( safe_str(f.content) for f in c.files if (f_path is None or f.path == f_path)) response.content_type = 'text/plain' return content return render('admin/gists/show.html')
def _get_ref_rev(repo, ref_type, ref_name, returnempty=False): """ Safe way to get changeset. If error occurs show error. """ from kallithea.lib import helpers as h try: return repo.scm_instance.get_ref_revision(ref_type, ref_name) except EmptyRepositoryError as e: if returnempty: return repo.scm_instance.EMPTY_CHANGESET h.flash(h.literal(_('There are no changesets yet')), category='error') raise webob.exc.HTTPNotFound() except ChangesetDoesNotExistError as e: h.flash(h.literal(_('Changeset not found')), category='error') raise webob.exc.HTTPNotFound() except RepositoryError as e: log.error(traceback.format_exc()) h.flash(safe_str(e), category='error') raise webob.exc.HTTPBadRequest()
def _get_instance(self, cls, instance, callback=None): """ Gets instance of given cls using some simple lookup mechanism. :param cls: class to fetch :param instance: int or Instance :param callback: callback to call if all lookups failed """ if isinstance(instance, cls): return instance elif isinstance(instance, (int, long)) or safe_str(instance).isdigit(): return cls.get(instance) else: if instance: if callback is None: raise Exception( 'given object must be int, long or Instance of %s ' 'got %s, no callback provided' % (cls, type(instance))) else: return callback(instance)
def __get_filenode(self, cs, path): """ Returns file_node or raise HTTP error. :param cs: given changeset :param path: path to lookup """ try: file_node = cs.get_node(path) if file_node.is_dir(): raise RepositoryError('given path is a directory') except ChangesetDoesNotExistError: msg = _('Such revision does not exist for this repository') h.flash(msg, category='error') raise HTTPNotFound() except RepositoryError as e: h.flash(safe_str(e), category='error') raise HTTPNotFound() return file_node
def _escaper(string): """ Do HTML escaping/markup """ def substitute(m): groups = m.groups() if groups[0]: return '&' if groups[1]: return '<' if groups[2]: return '>' if groups[3]: return '<u>\t</u>' if groups[4]: return '<u class="cr"></u>' if groups[5]: return ' <i></i>' assert False return _escape_re.sub(substitute, safe_str(string))
class ChangelogController(BaseRepoController): def __before__(self): super(ChangelogController, self).__before__() c.affected_files_cut_off = 60 @staticmethod def __get_cs(rev, repo): """ Safe way to get changeset. If error occur fail with error message. :param rev: revision to fetch :param repo: repo instance """ try: return c.db_repo_scm_instance.get_changeset(rev) except EmptyRepositoryError, e: h.flash(h.literal(_('There are no changesets yet')), category='error') except RepositoryError, e: log.error(traceback.format_exc()) h.flash(safe_str(e), category='error')
def repo_scan(self, repos_path=None): """ Listing of repositories in given path. This path should not be a repository itself. Return a dictionary of repository objects :param repos_path: path to directory containing repositories """ if repos_path is None: repos_path = self.repos_path log.info('scanning for repositories in %s', repos_path) baseui = make_ui('db') repos = {} for name, path in get_filesystem_repos(repos_path): # name need to be decomposed and put back together using the / # since this is internal storage separator for kallithea name = Repository.normalize_repo_name(name) try: if name in repos: raise RepositoryError('Duplicate repository name %s ' 'found in %s' % (name, path)) else: klass = get_backend(path[0]) if path[0] == 'hg' and path[0] in BACKENDS.keys(): repos[name] = klass(safe_str(path[1]), baseui=baseui) if path[0] == 'git' and path[0] in BACKENDS.keys(): repos[name] = klass(path[1]) except OSError: continue log.debug('found %s paths with repositories', len(repos)) return repos
def test_create(self): self.log_user() repo_name = self.NEW_REPO description = u'description for newly created repo' response = self.app.post(url('repos'), fixture._get_repo_create_params(repo_private=False, repo_name=repo_name, repo_type=self.REPO_TYPE, repo_description=description, _authentication_token=self.authentication_token())) ## run the check page that triggers the flash message response = self.app.get(url('repo_check_home', repo_name=repo_name)) assert response.json == {u'result': True} self.checkSessionFlash(response, 'Created repository <a href="/%s">%s</a>' % (repo_name, repo_name)) # test if the repo was created in the database new_repo = Session().query(Repository) \ .filter(Repository.repo_name == repo_name).one() assert new_repo.repo_name == repo_name assert new_repo.description == description # test if the repository is visible in the list ? response = self.app.get(url('summary_home', repo_name=repo_name)) response.mustcontain(repo_name) response.mustcontain(self.REPO_TYPE) # test if the repository was created on filesystem try: vcs.get_repo(safe_str(os.path.join(Ui.get_by_key('paths', '/').ui_value, repo_name))) except vcs.exceptions.VCSError: pytest.fail('no repo %s in filesystem' % repo_name) RepoModel().delete(repo_name) Session().commit()
def add_doc(self, writer, path, repo, repo_name, index_rev=None): """ Adding doc to writer this function itself fetches data from the instance of vcs backend """ try: node = self.get_node(repo, path, index_rev) except (ChangesetError, NodeDoesNotExistError): log.debug(" >> %s - not found in %s %s", path, repo, index_rev) return 0, 0 indexed = indexed_w_content = 0 if self.is_indexable_node(node): bytes_content = node.content if b'\0' in bytes_content: log.warning(' >> %s - no text content', path) u_content = '' else: log.debug(' >> %s', path) u_content = safe_str(bytes_content) indexed_w_content += 1 else: log.debug(' >> %s - not indexable', path) # just index file name without it's content u_content = '' indexed += 1 writer.add_document(fileid=path, owner=repo.contact, repository_rawname=repo_name, repository=repo_name, path=path, content=u_content, modtime=self.get_node_mtime(node), extension=node.extension) return indexed, indexed_w_content
def url_generator(**kw): q = urllib.quote(safe_str(c.cur_query)) return update_params("?q=%s&type=%s" \ % (q, safe_str(c.cur_type)), **kw)
def handle_git_receive(repo_path, revs, env, hook_type): """ A really hacky method that is run by git post-receive hook and logs a push action together with pushed revisions. It's executed by subprocess thus needs all info to be able to create an on the fly app environment, connect to database and run the logging code. Hacky as sh*t but works. :param repo_path: :param revs: :param env: """ from paste.deploy import appconfig from sqlalchemy import engine_from_config from kallithea.config.environment import load_environment from kallithea.model.base import init_model from kallithea.model.db import Ui from kallithea.lib.utils import make_ui, setup_cache_regions extras = _extract_extras(env) repo_path = safe_unicode(repo_path) path, ini_name = os.path.split(extras['config']) conf = appconfig('config:%s' % ini_name, relative_to=path) conf = load_environment(conf.global_conf, conf.local_conf) setup_cache_regions(conf) engine = engine_from_config(conf, 'sqlalchemy.') init_model(engine) baseui = make_ui('db') # fix if it's not a bare repo if repo_path.endswith(os.sep + '.git'): repo_path = repo_path[:-5] repo = Repository.get_by_full_path(repo_path) if not repo: raise OSError('Repository %s not found in database' % (safe_str(repo_path))) _hooks = dict(baseui.configitems('hooks')) or {} if hook_type == 'pre': repo = repo.scm_instance else: # post push shouldn't use the cached instance never repo = repo.scm_instance_no_cache() if hook_type == 'pre': pre_push(baseui, repo) # if push hook is enabled via web interface elif hook_type == 'post' and _hooks.get(Ui.HOOK_PUSH): rev_data = [] for l in revs: old_rev, new_rev, ref = l.strip().split(' ') _ref_data = ref.split('/') if _ref_data[1] in ['tags', 'heads']: rev_data.append({ 'old_rev': old_rev, 'new_rev': new_rev, 'ref': ref, 'type': _ref_data[1], 'name': '/'.join(_ref_data[2:]) }) git_revs = [] for push_ref in rev_data: _type = push_ref['type'] if _type == 'heads': if push_ref['old_rev'] == EmptyChangeset().raw_id: # update the symbolic ref if we push new repo if repo.is_empty(): repo._repo.refs.set_symbolic_ref( 'HEAD', 'refs/heads/%s' % push_ref['name']) cmd = [ 'for-each-ref', '--format=%(refname)', 'refs/heads/*' ] heads = repo.run_git_command(cmd)[0] cmd = [ 'log', push_ref['new_rev'], '--reverse', '--pretty=format:%H', '--not' ] heads = heads.replace(push_ref['ref'], '') for l in heads.splitlines(): cmd.append(l.strip()) git_revs += repo.run_git_command(cmd)[0].splitlines() elif push_ref['new_rev'] == EmptyChangeset().raw_id: #delete branch case git_revs += ['delete_branch=>%s' % push_ref['name']] else: cmd = [ 'log', '%(old_rev)s..%(new_rev)s' % push_ref, '--reverse', '--pretty=format:%H' ] git_revs += repo.run_git_command(cmd)[0].splitlines() elif _type == 'tags': git_revs += ['tag=>%s' % push_ref['name']] log_push_action(baseui, repo, _git_revs=git_revs)
def execute(self): created_by = User.get(request.authuser.user_id) pr = PullRequest() pr.org_repo = self.org_repo pr.org_ref = self.org_ref pr.other_repo = self.other_repo pr.other_ref = self.other_ref pr.revisions = self.revisions pr.title = self.title pr.description = self.description pr.owner = self.owner Session().add(pr) Session().flush() # make database assign pull_request_id if self.org_repo.scm_instance.alias == 'git': # create a ref under refs/pull/ so that commits don't get garbage-collected self.org_repo.scm_instance._repo["refs/pull/%d/head" % pr.pull_request_id] = safe_str(self.org_rev) #reset state to under-review from kallithea.model.changeset_status import ChangesetStatusModel from kallithea.model.comment import ChangesetCommentsModel comment = ChangesetCommentsModel().create( text=u'', repo=self.org_repo, author=created_by, pull_request=pr, send_email=False, status_change=ChangesetStatus.STATUS_UNDER_REVIEW, ) ChangesetStatusModel().set_status( self.org_repo, ChangesetStatus.STATUS_UNDER_REVIEW, created_by, comment, pull_request=pr, ) mention_recipients = extract_mentioned_users(self.description) PullRequestModel().add_reviewers(created_by, pr, self.reviewers, mention_recipients) return pr
def auth(self, userobj, username, password, settings, **kwargs): """ Gets the container_auth username (or email). It tries to get username from REMOTE_USER if this plugin is enabled, if that fails it tries to get username from HTTP_X_FORWARDED_USER if fallback header is set. clean_username extracts the username from this data if it's having @ in it. Return None on failure. On success, return a dictionary of the form: see: KallitheaAuthPluginBase.auth_func_attrs :param userobj: :param username: :param password: :param settings: :param kwargs: """ environ = kwargs.get('environ') if not environ: log.debug('Empty environ data skipping...') return None if not userobj: userobj = self.get_user('', environ=environ, settings=settings) # we don't care passed username/password for container auth plugins. # only way to log in is using environ username = None if userobj: username = safe_str(getattr(userobj, 'username')) if not username: # we don't have any objects in DB, user doesn't exist, extract # username from environ based on the settings username = self._get_username(environ, settings) # if cannot fetch username, it's a no-go for this plugin to proceed if not username: return None # old attrs fetched from Kallithea database admin = getattr(userobj, 'admin', False) active = getattr(userobj, 'active', True) email = environ.get(settings.get('email_header'), getattr(userobj, 'email', '')) firstname = environ.get(settings.get('firstname_header'), getattr(userobj, 'firstname', '')) lastname = environ.get(settings.get('lastname_header'), getattr(userobj, 'lastname', '')) user_data = { 'username': username, 'firstname': safe_unicode(firstname or username), 'lastname': safe_unicode(lastname or ''), 'groups': [], 'email': email or '', 'admin': admin or False, 'active': active, 'active_from_extern': True, 'extern_name': username, } log.info('user `%s` authenticated correctly', user_data['username']) return user_data
def _create_filesystem_repo(self, repo_name, repo_type, repo_group, clone_uri=None, repo_store_location=None): """ Makes repository on filesystem. Operation is group aware, meaning that it will create a repository within a group, and alter the paths accordingly to the group location. :param repo_name: :param alias: :param parent: :param clone_uri: :param repo_store_location: """ from kallithea.lib.utils import is_valid_repo, is_valid_repo_group from kallithea.model.scm import ScmModel if '/' in repo_name: raise ValueError('repo_name must not contain groups got `%s`' % repo_name) if isinstance(repo_group, RepoGroup): new_parent_path = os.sep.join(repo_group.full_path_splitted) else: new_parent_path = repo_group or '' if repo_store_location: _paths = [repo_store_location] else: _paths = [self.repos_path, new_parent_path, repo_name] # we need to make it str for mercurial repo_path = os.path.join(*map(lambda x: safe_str(x), _paths)) # check if this path is not a repository if is_valid_repo(repo_path, self.repos_path): raise Exception('This path %s is a valid repository' % repo_path) # check if this path is a group if is_valid_repo_group(repo_path, self.repos_path): raise Exception('This path %s is a valid group' % repo_path) log.info('creating repo %s in %s from url: `%s`', repo_name, safe_unicode(repo_path), obfuscate_url_pw(clone_uri)) backend = get_backend(repo_type) if repo_type == 'hg': baseui = make_ui('db', clear_session=False) # patch and reset hooks section of UI config to not run any # hooks on creating remote repo for k, v in baseui.configitems('hooks'): baseui.setconfig('hooks', k, None) repo = backend(repo_path, create=True, src_url=clone_uri, baseui=baseui) elif repo_type == 'git': repo = backend(repo_path, create=True, src_url=clone_uri, bare=True) # add kallithea hook into this repo ScmModel().install_git_hooks(repo=repo) else: raise Exception('Not supported repo_type %s expected hg/git' % repo_type) log.debug('Created repo %s with %s backend', safe_unicode(repo_name), safe_unicode(repo_type)) return repo
def index(self, repo_name, revision=None, f_path=None): # Fix URL after page size form submission via GET # TODO: Somehow just don't send this extra junk in the GET URL if request.GET.get('set'): request.GET.pop('set', None) if revision is None: raise HTTPFound(location=url( 'changelog_home', repo_name=repo_name, **request.GET)) raise HTTPFound(location=url('changelog_file_home', repo_name=repo_name, revision=revision, f_path=f_path, **request.GET)) limit = 2000 default = 100 if request.GET.get('size'): c.size = max(min(safe_int(request.GET.get('size')), limit), 1) session['changelog_size'] = c.size session.save() else: c.size = int(session.get('changelog_size', default)) # min size must be 1 c.size = max(c.size, 1) p = safe_int(request.GET.get('page'), 1) branch_name = request.GET.get('branch', None) if (branch_name and branch_name not in c.db_repo_scm_instance.branches and branch_name not in c.db_repo_scm_instance.closed_branches and not revision): raise HTTPFound(location=url('changelog_file_home', repo_name=c.repo_name, revision=branch_name, f_path=f_path or '')) if revision == 'tip': revision = None c.changelog_for_path = f_path try: if f_path: log.debug('generating changelog for path %s', f_path) # get the history for the file ! tip_cs = c.db_repo_scm_instance.get_changeset() try: collection = tip_cs.get_file_history(f_path) except (NodeDoesNotExistError, ChangesetError): #this node is not present at tip ! try: cs = self.__get_cs(revision, repo_name) collection = cs.get_file_history(f_path) except RepositoryError as e: h.flash(safe_str(e), category='warning') raise HTTPFound(location=h.url('changelog_home', repo_name=repo_name)) collection = list(reversed(collection)) else: collection = c.db_repo_scm_instance.get_changesets( start=0, end=revision, branch_name=branch_name) c.total_cs = len(collection) c.pagination = RepoPage( collection, page=p, item_count=c.total_cs, items_per_page=c.size, branch=branch_name, ) page_revisions = [x.raw_id for x in c.pagination] c.comments = c.db_repo.get_comments(page_revisions) c.statuses = c.db_repo.statuses(page_revisions) except EmptyRepositoryError as e: h.flash(safe_str(e), category='warning') raise HTTPFound( location=url('summary_home', repo_name=c.repo_name)) except (RepositoryError, ChangesetDoesNotExistError, Exception) as e: log.error(traceback.format_exc()) h.flash(safe_str(e), category='error') raise HTTPFound( location=url('changelog_home', repo_name=c.repo_name)) c.branch_name = branch_name c.branch_filters = [('', _('None'))] + \ [(k, k) for k in c.db_repo_scm_instance.branches.keys()] if c.db_repo_scm_instance.closed_branches: prefix = _('(closed)') + ' ' c.branch_filters += [('-', '-')] + \ [(k, prefix + k) for k in c.db_repo_scm_instance.closed_branches.keys()] revs = [] if not f_path: revs = [x.revision for x in c.pagination] c.jsdata = graph_data(c.db_repo_scm_instance, revs) c.revision = revision # requested revision ref c.first_revision = c.pagination[0] # pagination is never empty here! return render('changelog/changelog.html')
def setUp(self): self.repo = MercurialRepository(safe_str(TEST_HG_REPO))
def authenticate_ldap(self, username, password): """ Authenticate a user via LDAP and return his/her LDAP properties. Raises AuthenticationError if the credentials are rejected, or EnvironmentError if the LDAP server can't be reached. :param username: username :param password: password """ if not password: log.debug("Attempt to authenticate LDAP user " "with blank password rejected.") raise LdapPasswordError() if "," in username: raise LdapUsernameError("invalid character in username: ,") try: if self.cacertdir: if hasattr(ldap, 'OPT_X_TLS_CACERTDIR'): ldap.set_option(ldap.OPT_X_TLS_CACERTDIR, self.cacertdir) else: log.debug( "OPT_X_TLS_CACERTDIR is not available - can't set %s", self.cacertdir) ldap.set_option(ldap.OPT_REFERRALS, ldap.OPT_OFF) ldap.set_option(ldap.OPT_RESTART, ldap.OPT_ON) ldap.set_option(ldap.OPT_TIMEOUT, 20) ldap.set_option(ldap.OPT_NETWORK_TIMEOUT, 10) ldap.set_option(ldap.OPT_TIMELIMIT, 15) if self.TLS_KIND != 'PLAIN': ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, self.TLS_REQCERT) server = ldap.initialize(self.LDAP_SERVER) if self.ldap_version == 2: server.protocol = ldap.VERSION2 else: server.protocol = ldap.VERSION3 if self.TLS_KIND == 'START_TLS': server.start_tls_s() if self.LDAP_BIND_DN and self.LDAP_BIND_PASS: log.debug('Trying simple_bind with password and given DN: %s', self.LDAP_BIND_DN) server.simple_bind_s(self.LDAP_BIND_DN, self.LDAP_BIND_PASS) filter_ = '(&%s(%s=%s))' % ( self.LDAP_FILTER, ldap.filter.escape_filter_chars(self.attr_login), ldap.filter.escape_filter_chars(username)) log.debug("Authenticating %r filter %s at %s", self.BASE_DN, filter_, self.LDAP_SERVER) lobjects = server.search_ext_s(self.BASE_DN, self.SEARCH_SCOPE, filter_) if not lobjects: raise ldap.NO_SUCH_OBJECT() for (dn, _attrs) in lobjects: if dn is None: continue try: log.debug('Trying simple bind with %s', dn) server.simple_bind_s(dn, safe_str(password)) results = server.search_ext_s(dn, ldap.SCOPE_BASE, '(objectClass=*)') if len(results) == 1: dn_, attrs = results[0] assert dn_ == dn return dn, attrs except ldap.INVALID_CREDENTIALS: log.debug("LDAP rejected password for user '%s': %s", username, dn) continue # accept authentication as another ldap user with same username log.debug("No matching LDAP objects for authentication " "of '%s'", username) raise LdapPasswordError() except ldap.NO_SUCH_OBJECT: log.debug("LDAP says no such user '%s'", username) raise LdapUsernameError() except ldap.SERVER_DOWN: # [0] might be {'info': "TLS error -8179:Peer's Certificate issuer is not recognized.", 'desc': "Can't contact LDAP server"} raise LdapConnectionError( "LDAP can't connect to authentication server")
def _authorize(self, environ, start_response, action, repo_name, ip_addr): """Authenticate and authorize user. Since we're dealing with a VCS client and not a browser, we only support HTTP basic authentication, either directly via raw header inspection, or by using container authentication to delegate the authentication to the web server. Returns (user, None) on successful authentication and authorization. Returns (None, wsgi_app) to send the wsgi_app response to the client. """ # Check if anonymous access is allowed. default_user = User.get_default_user(cache=True) is_default_user_allowed = (default_user.active and self._check_permission(action, default_user, repo_name, ip_addr)) if is_default_user_allowed: return default_user, None if not default_user.active: log.debug('Anonymous access is disabled') else: log.debug('Not authorized to access this ' 'repository as anonymous user') username = None #============================================================== # DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE # NEED TO AUTHENTICATE AND ASK FOR AUTH USER PERMISSIONS #============================================================== # try to auth based on environ, container auth methods log.debug('Running PRE-AUTH for container based authentication') pre_auth = auth_modules.authenticate('', '', environ) if pre_auth is not None and pre_auth.get('username'): username = pre_auth['username'] log.debug('PRE-AUTH got %s as username', username) # If not authenticated by the container, running basic auth if not username: self.authenticate.realm = safe_str(self.config['realm']) result = self.authenticate(environ) if isinstance(result, str): paste.httpheaders.AUTH_TYPE.update(environ, 'basic') paste.httpheaders.REMOTE_USER.update(environ, result) username = result else: return None, result.wsgi_application #============================================================== # CHECK PERMISSIONS FOR THIS REQUEST USING GIVEN USERNAME #============================================================== try: user = User.get_by_username_or_email(username) if user is None or not user.active: return None, webob.exc.HTTPForbidden() except Exception: log.error(traceback.format_exc()) return None, webob.exc.HTTPInternalServerError() #check permissions for this repository perm = self._check_permission(action, user, repo_name, ip_addr) if not perm: return None, webob.exc.HTTPForbidden() return user, None