def _dispatch_call(self):
"""
Implement dispatch interface specified by WSGIController
"""
raw_response = ''
try:
raw_response = self._inspect_call(self._func)
if isinstance(raw_response, HTTPError):
self._error = str(raw_response)
except JSONRPCError as e:
self._error = safe_str(e)
except Exception as e:
log.error('Encountered unhandled exception: %s',
traceback.format_exc(),)
json_exc = JSONRPCError('Internal server error')
self._error = safe_str(json_exc)
if self._error is not None:
raw_response = None
response = dict(id=self._req_id, result=raw_response, error=self._error)
try:
return json.dumps(response)
except TypeError as e:
log.error('API FAILED. Error encoding response: %s', e)
return json.dumps(
dict(
id=self._req_id,
result=None,
error="Error encoding response"
)
)
def _rpc_call(self, action, environ, **rpc_args):
"""
Call the specified RPC Method
"""
raw_response = ''
try:
raw_response = getattr(self, action)(**rpc_args)
if isinstance(raw_response, HTTPError):
self._error = str(raw_response)
except JSONRPCError as e:
self._error = safe_str(e)
except Exception as e:
log.error('Encountered unhandled exception: %s',
traceback.format_exc(),)
json_exc = JSONRPCError('Internal server error')
self._error = safe_str(json_exc)
if self._error is not None:
raw_response = None
response = dict(id=self._req_id, result=raw_response, error=self._error)
try:
return json.dumps(response)
except TypeError as e:
log.error('API FAILED. Error encoding response: %s', e)
return json.dumps(
dict(
id=self._req_id,
result=None,
error="Error encoding response"
)
)
def is_valid_repo_group(repo_group_name, base_path, skip_path_check=False):
"""
Returns True if given path is a repository group False otherwise
:param repo_name:
:param base_path:
"""
full_path = os.path.join(safe_str(base_path), safe_str(repo_group_name))
# check if it's not a repo
if is_valid_repo(repo_group_name, base_path):
return False
try:
# we need to check bare git repos at higher level
# since we might match branches/hooks/info/objects or possible
# other things inside bare git repo
get_scm(os.path.dirname(full_path))
return False
except VCSError:
pass
# check if it's a valid path
if skip_path_check or os.path.isdir(full_path):
return True
return False
def commit_change(self, repo, repo_name, cs, user, author, message,
content, f_path):
"""
Commit a change to a single file
:param repo: a db_repo.scm_instance
"""
user = self._get_user(user)
IMC = self._get_IMC_module(repo.alias)
# decoding here will force that we have proper encoded values
# in any other case this will throw exceptions and deny commit
content = safe_str(content)
path = safe_str(f_path)
# message and author needs to be unicode
# proper backend should then translate that into required type
message = safe_unicode(message)
author = safe_unicode(author)
imc = IMC(repo)
imc.change(FileNode(path, content, mode=cs.get_file_mode(f_path)))
try:
tip = imc.commit(message=message, author=author,
parents=[cs], branch=cs.branch)
except Exception as e:
log.error(traceback.format_exc())
raise IMCCommitError(str(e))
finally:
# always clear caches, if commit fails we want fresh object also
self.mark_for_invalidation(repo_name)
self._handle_push(repo,
username=user.username,
action='push_local',
repo_name=repo_name,
revisions=[tip.raw_id])
return tip
def __init__(self, server, base_dn, port=None, bind_dn='', bind_pass='',
tls_kind='LDAPS', tls_reqcert='DEMAND', cacertdir=None, ldap_version=3,
ldap_filter='(&(objectClass=user)(!(objectClass=computer)))',
search_scope='SUBTREE', attr_login='******'):
if ldap is None:
raise LdapImportError
self.ldap_version = ldap_version
self.TLS_KIND = tls_kind
OPT_X_TLS_DEMAND = 2
self.TLS_REQCERT = getattr(ldap, 'OPT_X_TLS_%s' % tls_reqcert,
OPT_X_TLS_DEMAND)
self.cacertdir = cacertdir
protocol = 'ldaps' if self.TLS_KIND == 'LDAPS' else 'ldap'
if not port:
port = 636 if self.TLS_KIND == 'LDAPS' else 389
self.LDAP_SERVER = str(', '.join(
"%s://%s:%s" % (protocol,
host.strip(),
port)
for host in server.split(',')))
self.LDAP_BIND_DN = safe_str(bind_dn)
self.LDAP_BIND_PASS = safe_str(bind_pass)
self.BASE_DN = safe_str(base_dn)
self.LDAP_FILTER = safe_str(ldap_filter)
self.SEARCH_SCOPE = getattr(ldap, 'SCOPE_%s' % search_scope)
self.attr_login = attr_login
def commit_change(self, repo, repo_name, cs, user, author, message,
content, f_path):
"""
Commits changes
:param repo: SCM instance
"""
user = self._get_user(user)
IMC = self._get_IMC_module(repo.alias)
# decoding here will force that we have proper encoded values
# in any other case this will throw exceptions and deny commit
content = safe_str(content)
path = safe_str(f_path)
# message and author needs to be unicode
# proper backend should then translate that into required type
message = safe_unicode(message)
author = safe_unicode(author)
imc = IMC(repo)
imc.change(FileNode(path, content, mode=cs.get_file_mode(f_path)))
try:
tip = imc.commit(message=message, author=author,
parents=[cs], branch=cs.branch)
except Exception, e:
log.error(traceback.format_exc())
raise IMCCommitError(str(e))
def test_get_repo(self):
alias = 'hg'
path = TEST_HG_REPO
backend = get_backend(alias)
repo = backend(safe_str(path))
self.assertEqual(repo.__class__, get_repo(safe_str(path), alias).__class__)
self.assertEqual(repo.path, get_repo(safe_str(path), alias).path)
def test_get_repo_autoalias_git(self):
alias = 'git'
path = TEST_GIT_REPO
backend = get_backend(alias)
repo = backend(safe_str(path))
self.assertEqual(repo.__class__, get_repo(safe_str(path)).__class__)
self.assertEqual(repo.path, get_repo(safe_str(path)).path)
def log_push_action(ui, repo, **kwargs):
"""
Register that changes have been pushed.
Mercurial invokes this directly as a hook, git uses handle_git_receive.
"""
ex = _extract_extras()
action_tmpl = ex.action + ':%s'
revs = []
if ex.scm == 'hg':
node = kwargs['node']
def get_revs(repo, rev_opt):
if rev_opt:
revs = revrange(repo, rev_opt)
if len(revs) == 0:
return (nullrev, nullrev)
return max(revs), min(revs)
else:
return len(repo) - 1, 0
stop, start = get_revs(repo, [node + ':'])
_h = binascii.hexlify
revs = [_h(repo[r].node()) for r in xrange(start, stop + 1)]
elif ex.scm == 'git':
revs = kwargs.get('_git_revs', [])
if '_git_revs' in kwargs:
kwargs.pop('_git_revs')
action = action_tmpl % ','.join(revs)
action_logger(ex.username, action, ex.repository, ex.ip, commit=True)
# extension hook call
from kallithea import EXTENSIONS
callback = getattr(EXTENSIONS, 'PUSH_HOOK', None)
if callable(callback):
kw = {'pushed_revs': revs}
kw.update(ex)
callback(**kw)
if ex.make_lock is not None and not ex.make_lock:
Repository.unlock(Repository.get_by_repo_name(ex.repository))
ui.status(safe_str('Released lock on repo `%s`\n' % ex.repository))
if ex.locked_by[0]:
locked_by = User.get(ex.locked_by[0]).username
_http_ret = HTTPLockedRC(ex.repository, locked_by)
if str(_http_ret.code).startswith('2'):
#2xx Codes don't raise exceptions
ui.status(safe_str(_http_ret.title))
return 0
def checkSessionFlash(self, response, msg, skip=0):
if 'flash' not in response.session:
self.fail(safe_str(u'msg `%s` not found - session has no flash ' % msg))
try:
level, m = response.session['flash'][-1 - skip]
if msg in m:
return
except IndexError:
pass
self.fail(safe_str(u'msg `%s` not found in session flash (skipping %s): %s' %
(msg, skip,
', '.join('`%s`' % m for level, m in response.session['flash']))))
def take_action(self, args):
_caches = CacheInvalidation.query().order_by(CacheInvalidation.cache_key).all()
if args.show:
for c_obj in _caches:
print 'key:%s active:%s' % (safe_str(c_obj.cache_key), c_obj.cache_active)
elif args.cleanup:
for c_obj in _caches:
Session().delete(c_obj)
print 'Removing key: %s' % (safe_str(c_obj.cache_key))
Session().commit()
else:
print 'Nothing done, exiting...'
def checkSessionFlash(self, response, msg=None, skip=0, _matcher=lambda msg, m: msg in m):
if 'flash' not in response.session:
pytest.fail(safe_str(u'msg `%s` not found - session has no flash:\n%s' % (msg, response)))
try:
level, m = response.session['flash'][-1 - skip]
if _matcher(msg, m):
return
except IndexError:
pass
pytest.fail(safe_str(u'msg `%s` not found in session flash (skipping %s): %s' %
(msg, skip,
', '.join('`%s`' % m for level, m in response.session['flash']))))
def command(self):
#get SqlAlchemy session
self._init_session()
repos_location = Ui.get_repos_location()
to_remove = []
for dn, dirs, f in os.walk(safe_str(repos_location)):
alldirs = list(dirs)
del dirs[:]
if ('.hg' in alldirs or
'objects' in alldirs and ('refs' in alldirs or 'packed-refs' in f)):
continue
for loc in alldirs:
if REMOVED_REPO_PAT.match(loc):
to_remove.append([os.path.join(dn, loc),
self._extract_date(loc)])
else:
dirs.append(loc)
#filter older than (if present)!
now = datetime.datetime.now()
older_than = self.options.older_than
if older_than:
to_remove_filtered = []
older_than_date = self._parse_older_than(older_than)
for name, date_ in to_remove:
repo_age = now - date_
if repo_age > older_than_date:
to_remove_filtered.append([name, date_])
to_remove = to_remove_filtered
print >> sys.stdout, 'removing %s deleted repos older than %s (%s)' \
% (len(to_remove), older_than, older_than_date)
else:
print >> sys.stdout, 'removing all [%s] deleted repos' \
% len(to_remove)
if self.options.dont_ask or not to_remove:
# don't ask just remove !
remove = True
else:
remove = ask_ok('the following repositories will be deleted completely:\n%s\n'
'are you sure you want to remove them [y/n]?'
% ', \n'.join(['%s removed on %s'
% (safe_str(x[0]), safe_str(x[1])) for x in to_remove]))
if remove:
for path, date_ in to_remove:
print >> sys.stdout, 'removing repository %s' % path
shutil.rmtree(path)
else:
print 'nothing done exiting...'
sys.exit(0)
def take_action(self, args):
repos_location = Ui.get_repos_location()
to_remove = []
for dn_, dirs, f in os.walk(safe_str(repos_location)):
alldirs = list(dirs)
del dirs[:]
if ('.hg' in alldirs or
'.git' in alldirs or
'.svn' in alldirs or
'objects' in alldirs and ('refs' in alldirs or 'packed-refs' in f)):
continue
for loc in alldirs:
if REMOVED_REPO_PAT.match(loc):
to_remove.append([os.path.join(dn_, loc),
self._extract_date(loc)])
else:
dirs.append(loc)
if dirs:
print 'Scanning: %s' % dn_
#filter older than (if present)!
now = datetime.datetime.now()
older_than = args.older_than
if older_than:
to_remove_filtered = []
older_than_date = self._parse_older_than(older_than)
for name, date_ in to_remove:
repo_age = now - date_
if repo_age > older_than_date:
to_remove_filtered.append([name, date_])
to_remove = to_remove_filtered
print 'Removing %s deleted repos older than %s (%s)' \
% (len(to_remove), older_than, older_than_date)
else:
print 'Removing all %s deleted repos' % len(to_remove)
if args.dont_ask or not to_remove:
# don't ask just remove !
remove = True
else:
remove = ask_ok('the following repositories will be deleted completely:\n%s\n'
'are you sure you want to remove them [y/n]?'
% '\n'.join(['%s removed on %s' % (safe_str(x[0]), safe_str(x[1]))
for x in to_remove]))
if remove:
for path, date_ in to_remove:
print 'Removing repository %s' % path
shutil.rmtree(path)
else:
print 'Nothing done, exiting...'
def test_fork_unicode(self):
self.log_user()
# create a fork
repo_name = self.REPO
org_repo = Repository.get_by_repo_name(repo_name)
fork_name = safe_str(self.REPO_FORK + u'-rødgrød')
creation_args = {
'repo_name': fork_name,
'repo_group': u'-1',
'fork_parent_id': org_repo.repo_id,
'repo_type': self.REPO_TYPE,
'description': 'unicode repo 1',
'private': 'False',
'landing_rev': 'rev:tip',
'_authentication_token': self.authentication_token()}
self.app.post(url(controller='forks', action='fork_create',
repo_name=repo_name), creation_args)
response = self.app.get(url(controller='forks', action='forks',
repo_name=repo_name))
response.mustcontain(
"""<a href="/%s">%s</a>""" % (urllib.quote(fork_name), fork_name)
)
fork_repo = Repository.get_by_repo_name(safe_unicode(fork_name))
assert fork_repo
# fork the fork
fork_name_2 = safe_str(self.REPO_FORK + u'-blåbærgrød')
creation_args = {
'repo_name': fork_name_2,
'repo_group': u'-1',
'fork_parent_id': fork_repo.repo_id,
'repo_type': self.REPO_TYPE,
'description': 'unicode repo 2',
'private': 'False',
'landing_rev': 'rev:tip',
'_authentication_token': self.authentication_token()}
self.app.post(url(controller='forks', action='fork_create',
repo_name=fork_name), creation_args)
response = self.app.get(url(controller='forks', action='forks',
repo_name=fork_name))
response.mustcontain(
"""<a href="/%s">%s</a>""" % (urllib.quote(fork_name_2), fork_name_2)
)
# remove these forks
response = self.app.post(url('delete_repo', repo_name=fork_name_2),
params={'_authentication_token': self.authentication_token()})
response = self.app.post(url('delete_repo', repo_name=fork_name),
params={'_authentication_token': self.authentication_token()})
def test_delete_non_ascii(self):
self.log_user()
non_ascii = "ąęł"
repo_name = "%s%s" % (safe_str(self.NEW_REPO), non_ascii)
repo_name_unicode = safe_unicode(repo_name)
description = 'description for newly created repo' + non_ascii
description_unicode = safe_unicode(description)
response = self.app.post(url('repos'),
fixture._get_repo_create_params(repo_private=False,
repo_name=repo_name,
repo_type=self.REPO_TYPE,
repo_description=description,
_authentication_token=self.authentication_token()))
## run the check page that triggers the flash message
response = self.app.get(url('repo_check_home', repo_name=repo_name))
self.assertEqual(response.json, {u'result': True})
self.checkSessionFlash(response,
u'Created repository <a href="/%s">%s</a>'
% (urllib.quote(repo_name), repo_name_unicode))
# test if the repo was created in the database
new_repo = Session().query(Repository) \
.filter(Repository.repo_name == repo_name_unicode).one()
self.assertEqual(new_repo.repo_name, repo_name_unicode)
self.assertEqual(new_repo.description, description_unicode)
# test if the repository is visible in the list ?
response = self.app.get(url('summary_home', repo_name=repo_name))
response.mustcontain(repo_name)
response.mustcontain(self.REPO_TYPE)
# test if the repository was created on filesystem
try:
vcs.get_repo(safe_str(os.path.join(Ui.get_by_key('paths', '/').ui_value, repo_name_unicode)))
except vcs.exceptions.VCSError:
pytest.fail('no repo %s in filesystem' % repo_name)
response = self.app.post(url('delete_repo', repo_name=repo_name),
params={'_method': 'delete', '_authentication_token': self.authentication_token()})
self.checkSessionFlash(response, 'Deleted repository %s' % (repo_name_unicode))
response.follow()
#check if repo was deleted from db
deleted_repo = Session().query(Repository) \
.filter(Repository.repo_name == repo_name_unicode).scalar()
self.assertEqual(deleted_repo, None)
self.assertEqual(os.path.isdir(os.path.join(Ui.get_by_key('paths', '/').ui_value, repo_name_unicode)),
False)
def get_paths(self, repo):
"""
recursive walk in root dir and return a set of all path in that dir
based on repository walk function
"""
index_paths_ = set()
try:
cs = self._get_index_changeset(repo)
for _topnode, _dirs, files in cs.walk('/'):
for f in files:
index_paths_.add(jn(safe_str(repo.path), safe_str(f.path)))
except RepositoryError:
log.debug(traceback.format_exc())
pass
return index_paths_
def log_pull_action(ui, repo, **kwargs):
"""
Logs user last pull action
:param ui:
:param repo:
"""
ex = _extract_extras()
user = User.get_by_username(ex.username)
action = 'pull'
action_logger(user, action, ex.repository, ex.ip, commit=True)
# extension hook call
from kallithea import EXTENSIONS
callback = getattr(EXTENSIONS, 'PULL_HOOK', None)
if callable(callback):
kw = {}
kw.update(ex)
callback(**kw)
if ex.make_lock is not None and ex.make_lock:
Repository.lock(Repository.get_by_repo_name(ex.repository), user.user_id)
#msg = 'Made lock on repo `%s`' % repository
#ui.status(msg)
if ex.locked_by[0]:
locked_by = User.get(ex.locked_by[0]).username
_http_ret = HTTPLockedRC(ex.repository, locked_by)
if str(_http_ret.code).startswith('2'):
#2xx Codes don't raise exceptions
ui.status(safe_str(_http_ret.title))
return 0
def _rename_filesystem_repo(self, old, new):
"""
renames repository on filesystem
:param old: old name
:param new: new name
"""
log.info('renaming repo from %s to %s', old, new)
old_path = safe_str(os.path.join(self.repos_path, old))
new_path = safe_str(os.path.join(self.repos_path, new))
if os.path.isdir(new_path):
raise Exception(
'Was trying to rename to already existing dir %s' % new_path
)
shutil.move(old_path, new_path)
def _get_cache_parameters(query):
"""For a query with cache_region and cache_namespace configured,
return the corresponding Cache instance and cache key, based
on this query's current criterion and parameter values.
"""
if not hasattr(query, '_cache_parameters'):
raise ValueError("This Query does not have caching "
"parameters configured.")
region, namespace, cache_key = query._cache_parameters
namespace = _namespace_from_query(namespace, query)
if cache_key is None:
# cache key - the value arguments from this query's parameters.
args = [safe_str(x) for x in _params_from_query(query)]
args.extend(filter(lambda k: k not in ['None', None, u'None'],
[str(query._limit), str(query._offset)]))
cache_key = " ".join(args)
if cache_key is None:
raise Exception('Cache key cannot be None')
# get cache
#cache = query.cache_manager.get_cache_region(namespace, region)
cache = get_cache_region(namespace, region)
# optional - hash the cache_key too for consistent length
# import uuid
# cache_key= str(uuid.uuid5(uuid.NAMESPACE_DNS, cache_key))
return cache, cache_key
def _get_scm_size(alias, root_path):
if not alias.startswith('.'):
alias += '.'
size_scm, size_root = 0, 0
for path, dirs, files in os.walk(safe_str(root_path)):
if path.find(alias) != -1:
for f in files:
try:
size_scm += os.path.getsize(os.path.join(path, f))
except OSError:
pass
else:
for f in files:
try:
size_root += os.path.getsize(os.path.join(path, f))
except OSError:
pass
size_scm_f = h.format_byte_size(size_scm)
size_root_f = h.format_byte_size(size_root)
size_total_f = h.format_byte_size(size_root + size_scm)
return size_scm_f, size_root_f, size_total_f
def __get_cs(self, rev, silent_empty=False):
"""
Safe way to get changeset if error occur it redirects to tip with
proper message
:param rev: revision to fetch
:silent_empty: return None if repository is empty
"""
try:
return c.db_repo_scm_instance.get_changeset(rev)
except EmptyRepositoryError as e:
if silent_empty:
return None
url_ = url('files_add_home',
repo_name=c.repo_name,
revision=0, f_path='', anchor='edit')
add_new = h.link_to(_('Click here to add new file'), url_, class_="alert-link")
h.flash(h.literal(_('There are no files yet. %s') % add_new),
category='warning')
raise HTTPNotFound()
except (ChangesetDoesNotExistError, LookupError):
msg = _('Such revision does not exist for this repository')
h.flash(msg, category='error')
raise HTTPNotFound()
except RepositoryError as e:
h.flash(safe_str(e), category='error')
raise HTTPNotFound()
def index(self, repo_name, revision, f_path, annotate=False):
# redirect to given revision from form if given
post_revision = request.POST.get('at_rev', None)
if post_revision:
cs = self.__get_cs(post_revision) # FIXME - unused!
c.revision = revision
c.changeset = self.__get_cs(revision)
c.branch = request.GET.get('branch', None)
c.f_path = f_path
c.annotate = annotate
cur_rev = c.changeset.revision
# prev link
try:
prev_rev = c.db_repo_scm_instance.get_changeset(cur_rev).prev(c.branch)
c.url_prev = url('files_home', repo_name=c.repo_name,
revision=prev_rev.raw_id, f_path=f_path)
if c.branch:
c.url_prev += '?branch=%s' % c.branch
except (ChangesetDoesNotExistError, VCSError):
c.url_prev = '#'
# next link
try:
next_rev = c.db_repo_scm_instance.get_changeset(cur_rev).next(c.branch)
c.url_next = url('files_home', repo_name=c.repo_name,
revision=next_rev.raw_id, f_path=f_path)
if c.branch:
c.url_next += '?branch=%s' % c.branch
except (ChangesetDoesNotExistError, VCSError):
c.url_next = '#'
# files or dirs
try:
c.file = c.changeset.get_node(f_path)
if c.file.is_file():
c.load_full_history = False
file_last_cs = c.file.last_changeset
c.file_changeset = (c.changeset
if c.changeset.revision < file_last_cs.revision
else file_last_cs)
#determine if we're on branch head
_branches = c.db_repo_scm_instance.branches
c.on_branch_head = revision in _branches.keys() + _branches.values()
_hist = []
c.file_history = []
if c.load_full_history:
c.file_history, _hist = self._get_node_history(c.changeset, f_path)
c.authors = []
for a in set([x.author for x in _hist]):
c.authors.append((h.email(a), h.person(a)))
else:
c.authors = c.file_history = []
except RepositoryError, e:
h.flash(safe_str(e), category='error')
raise HTTPNotFound()
def __init__(self, reponame, username, *args, **kwargs):
from kallithea import CONFIG
from kallithea.lib.utils2 import safe_int
_code = CONFIG.get('lock_ret_code')
self.code = safe_int(_code, self.code)
self.title = self.explanation = safe_str(
'Repository `%s` locked by user `%s`' % (reponame, username))
super(HTTPLockedRC, self).__init__(*args, **kwargs)
def test_repo_clone_without_update(self):
repo = MercurialRepository(safe_str(TEST_HG_REPO))
repo_clone = MercurialRepository(TEST_HG_REPO_CLONE + '_wo_update',
src_url=TEST_HG_REPO, update_after_clone=False)
self.assertEqual(len(repo.revisions), len(repo_clone.revisions))
self.assertEqual(os.path.isfile(os.path.join(TEST_HG_REPO_CLONE \
+ '_wo_update',
'MANIFEST.in')), False,)
def make_ui(read_from='file', path=None, checkpaths=True, clear_session=True):
"""
A function that will read python rc files or database
and make an mercurial ui object from read options
:param path: path to mercurial config file
:param checkpaths: check the path
:param read_from: read from 'file' or 'db'
"""
baseui = ui.ui()
# clean the baseui object
baseui._ocfg = config.config()
baseui._ucfg = config.config()
baseui._tcfg = config.config()
if read_from == 'file':
if not os.path.isfile(path):
log.debug('hgrc file is not present at %s, skipping...', path)
return False
log.debug('reading hgrc from %s', path)
cfg = config.config()
cfg.read(path)
for section in ui_sections:
for k, v in cfg.items(section):
log.debug('settings ui from file: [%s] %s=%s', section, k, v)
baseui.setconfig(safe_str(section), safe_str(k), safe_str(v))
elif read_from == 'db':
sa = meta.Session()
ret = sa.query(Ui).all()
hg_ui = ret
for ui_ in hg_ui:
if ui_.ui_active:
ui_val = safe_str(ui_.ui_value)
if ui_.ui_section == 'hooks' and BRAND != 'kallithea' and ui_val.startswith('python:' + BRAND + '.lib.hooks.'):
ui_val = ui_val.replace('python:' + BRAND + '.lib.hooks.', 'python:kallithea.lib.hooks.')
log.debug('settings ui from db: [%s] %s=%s', ui_.ui_section,
ui_.ui_key, ui_val)
baseui.setconfig(safe_str(ui_.ui_section), safe_str(ui_.ui_key),
ui_val)
if ui_.ui_key == 'push_ssl':
# force set push_ssl requirement to False, kallithea
# handles that
baseui.setconfig(safe_str(ui_.ui_section), safe_str(ui_.ui_key),
False)
if clear_session:
meta.Session.remove()
# prevent interactive questions for ssh password / passphrase
ssh = baseui.config('ui', 'ssh', default='ssh')
baseui.setconfig('ui', 'ssh', '%s -oBatchMode=yes -oIdentitiesOnly=yes' % ssh)
return baseui
def get_filesystem_repos(path):
"""
Scans given path for repos and return (name,(type,path)) tuple
:param path: path to scan for repositories
:param recursive: recursive search and return names with subdirs in front
"""
# remove ending slash for better results
path = safe_str(path.rstrip(os.sep))
log.debug('now scanning in %s', path)
def isdir(*n):
return os.path.isdir(os.path.join(*n))
for root, dirs, _files in os.walk(path):
recurse_dirs = []
for subdir in dirs:
# skip removed repos
if REMOVED_REPO_PAT.match(subdir):
continue
#skip .<something> dirs TODO: rly? then we should prevent creating them ...
if subdir.startswith('.'):
continue
cur_path = os.path.join(root, subdir)
if isdir(cur_path, '.git'):
log.warning('ignoring non-bare Git repo: %s', cur_path)
continue
if (isdir(cur_path, '.hg') or
isdir(cur_path, '.svn') or
isdir(cur_path, 'objects') and (isdir(cur_path, 'refs') or
os.path.isfile(os.path.join(cur_path, 'packed-refs')))):
if not os.access(cur_path, os.R_OK) or not os.access(cur_path, os.X_OK):
log.warning('ignoring repo path without access: %s', cur_path)
continue
if not os.access(cur_path, os.W_OK):
log.warning('repo path without write access: %s', cur_path)
try:
scm_info = get_scm(cur_path)
assert cur_path.startswith(path)
repo_path = cur_path[len(path) + 1:]
yield repo_path, scm_info
continue # no recursion
except VCSError:
# We should perhaps ignore such broken repos, but especially
# the bare git detection is unreliable so we dive into it
pass
recurse_dirs.append(subdir)
dirs[:] = recurse_dirs
def _set_cache_parameters(query, region, namespace, cache_key):
if hasattr(query, '_cache_parameters'):
region, namespace, cache_key = query._cache_parameters
raise ValueError("This query is already configured "
"for region %r namespace %r" %
(region, namespace)
)
query._cache_parameters = region, safe_str(namespace), cache_key
def get_node(self, repo, path, index_rev=None):
"""
gets a filenode based on given full path. It operates on string for
hg git compatibility.
:param repo: scm repo instance
:param path: full path including root location
:return: FileNode
"""
# FIXME: paths should be normalized ... or even better: don't include repo.path
path = safe_str(path)
repo_path = safe_str(repo.path)
assert path.startswith(repo_path)
assert path[len(repo_path)] in (os.path.sep, os.path.altsep)
node_path = path[len(repo_path) + 1:]
cs = self._get_index_changeset(repo, index_rev=index_rev)
node = cs.get_node(node_path)
return node
def rawfile(self, repo_name, revision, f_path):
cs = self.__get_cs(revision)
file_node = self.__get_filenode(cs, f_path)
response.content_disposition = 'attachment; filename=%s' % \
safe_str(f_path.split(Repository.url_sep())[-1])
response.content_type = file_node.mimetype
return file_node.content
def get_crypt_password(password):
"""
Cryptographic function used for password hashing based on pybcrypt
or Python's own OpenSSL wrapper on windows
:param password: password to hash
"""
if is_windows:
return hashlib.sha256(password).hexdigest()
elif is_unix:
import bcrypt
return bcrypt.hashpw(safe_str(password), bcrypt.gensalt(10))
else:
raise Exception('Unknown or unsupported platform %s' \
% __platform__)
def pre_push(ui, repo, **kwargs):
# pre push function, currently used to ban pushing when
# repository is locked
ex = _extract_extras()
usr = User.get_by_username(ex.username)
if ex.locked_by[0] and usr.user_id != int(ex.locked_by[0]):
locked_by = User.get(ex.locked_by[0]).username
# this exception is interpreted in git/hg middlewares and based
# on that proper return code is server to client
_http_ret = HTTPLockedRC(ex.repository, locked_by)
if str(_http_ret.code).startswith('2'):
#2xx Codes don't raise exceptions
ui.status(safe_str(_http_ret.title))
else:
raise _http_ret
def _get_by_id(self, repo_name):
"""
Gets a special pattern _<ID> from clone url and tries to replace it
with a repository_name for support of _<ID> permanent URLs
:param repo_name:
"""
data = repo_name.split('/')
if len(data) >= 2:
from kallithea.lib.utils import get_repo_by_id
by_id_match = get_repo_by_id(repo_name)
if by_id_match:
data[1] = safe_str(by_id_match)
return '/'.join(data)
def plain(cls, source, universal_newline=True):
"""
>>> MarkupRenderer.plain('https://example.com/')
'<br /><a href="https://example.com/">https://example.com/</a>'
"""
source = safe_str(source)
if universal_newline:
newline = '\n'
source = newline.join(source.splitlines())
def url_func(match_obj):
url_full = match_obj.group(0)
return '<a href="%(url)s">%(url)s</a>' % ({'url': url_full})
source = url_re.sub(url_func, source)
return '<br />' + source.replace("\n", '<br />')
def test_delete(self):
self.log_user()
repo_name = u'vcs_test_new_to_delete_%s' % self.REPO_TYPE
description = u'description for newly created repo'
response = self.app.post(url('repos'),
fixture._get_repo_create_params(repo_private=False,
repo_type=self.REPO_TYPE,
repo_name=repo_name,
repo_description=description,
_authentication_token=self.authentication_token()))
## run the check page that triggers the flash message
response = self.app.get(url('repo_check_home', repo_name=repo_name))
self.checkSessionFlash(response,
'Created repository <a href="/%s">%s</a>'
% (repo_name, repo_name))
# test if the repo was created in the database
new_repo = Session().query(Repository) \
.filter(Repository.repo_name == repo_name).one()
assert new_repo.repo_name == repo_name
assert new_repo.description == description
# test if the repository is visible in the list ?
response = self.app.get(url('summary_home', repo_name=repo_name))
response.mustcontain(repo_name)
response.mustcontain(self.REPO_TYPE)
# test if the repository was created on filesystem
try:
vcs.get_repo(safe_str(os.path.join(Ui.get_by_key('paths', '/').ui_value, repo_name)))
except vcs.exceptions.VCSError:
pytest.fail('no repo %s in filesystem' % repo_name)
response = self.app.post(url('delete_repo', repo_name=repo_name),
params={'_authentication_token': self.authentication_token()})
self.checkSessionFlash(response, 'Deleted repository %s' % (repo_name))
response.follow()
#check if repo was deleted from db
deleted_repo = Session().query(Repository) \
.filter(Repository.repo_name == repo_name).scalar()
assert deleted_repo == None
assert os.path.isdir(os.path.join(Ui.get_by_key('paths', '/').ui_value, repo_name)) == False
def is_valid_repo_uri(repo_type, url, ui):
"""Check if the url seems like a valid remote repo location
Raise InvalidCloneUriException if any problems"""
if repo_type == 'hg':
if url.startswith('http') or url.startswith('ssh'):
# initially check if it's at least the proper URL
# or does it pass basic auth
try:
MercurialRepository._check_url(url, ui)
except urllib.error.URLError as e:
raise InvalidCloneUriException('URI %s URLError: %s' %
(url, e))
except mercurial.error.RepoError as e:
raise InvalidCloneUriException(
'Mercurial %s: %s' %
(type(e).__name__, safe_str(bytes(e))))
elif url.startswith('svn+http'):
try:
from hgsubversion.svnrepo import svnremoterepo
except ImportError:
raise InvalidCloneUriException(
'URI type %s not supported - hgsubversion is not available'
% (url, ))
svnremoterepo(ui, url).svn.uuid
elif url.startswith('git+http'):
raise InvalidCloneUriException('URI type %s not implemented' %
(url, ))
else:
raise InvalidCloneUriException('URI %s not allowed' % (url, ))
elif repo_type == 'git':
if url.startswith('http') or url.startswith('git'):
# initially check if it's at least the proper URL
# or does it pass basic auth
try:
GitRepository._check_url(url)
except urllib.error.URLError as e:
raise InvalidCloneUriException('URI %s URLError: %s' %
(url, e))
elif url.startswith('svn+http'):
raise InvalidCloneUriException('URI type %s not implemented' %
(url, ))
elif url.startswith('hg+http'):
raise InvalidCloneUriException('URI type %s not implemented' %
(url, ))
else:
raise InvalidCloneUriException('URI %s not allowed' % (url))
def __get_cs(rev, repo):
"""
Safe way to get changeset. If error occur fail with error message.
:param rev: revision to fetch
:param repo: repo instance
"""
try:
return c.db_repo_scm_instance.get_changeset(rev)
except EmptyRepositoryError as e:
h.flash(h.literal(_('There are no changesets yet')),
category='error')
except RepositoryError as e:
log.error(traceback.format_exc())
h.flash(safe_str(e), category='error')
raise HTTPBadRequest()
def index(self):
c.came_from = safe_str(request.GET.get('came_from', ''))
if c.came_from:
if not self._validate_came_from(c.came_from):
log.error('Invalid came_from (not server-relative): %r', c.came_from)
raise HTTPBadRequest()
else:
c.came_from = url('home')
ip_allowed = AuthUser.check_ip_allowed(request.authuser, request.ip_addr)
# redirect if already logged in
if request.authuser.is_authenticated and ip_allowed:
raise HTTPFound(location=c.came_from)
if request.POST:
# import Login Form validator class
login_form = LoginForm()()
try:
c.form_result = login_form.to_python(dict(request.POST))
# form checks for username/password, now we're authenticated
username = c.form_result['username']
user = User.get_by_username_or_email(username, case_insensitive=True)
except formencode.Invalid as errors:
defaults = errors.value
# remove password from filling in form again
defaults.pop('password', None)
return htmlfill.render(
render('/login.html'),
defaults=errors.value,
errors=errors.error_dict or {},
prefix_error=False,
encoding="UTF-8",
force_defaults=False)
except UserCreationError as e:
# container auth or other auth functions that create users on
# the fly can throw this exception signaling that there's issue
# with user creation, explanation should be provided in
# Exception itself
h.flash(e, 'error')
else:
log_in_user(user, c.form_result['remember'],
is_external_auth=False)
raise HTTPFound(location=c.came_from)
return render('/login.html')
def update(self, id):
c.user_group = UserGroup.get_or_404(id)
c.active = 'settings'
self.__load_data(id)
available_members = [safe_str(x[0]) for x in c.available_members]
users_group_form = UserGroupForm(edit=True,
old_data=c.user_group.get_dict(),
available_members=available_members)()
try:
form_result = users_group_form.to_python(request.POST)
UserGroupModel().update(c.user_group, form_result)
gr = form_result['users_group_name']
action_logger(request.authuser,
'admin_updated_users_group:%s' % gr, None,
request.ip_addr)
h.flash(_('Updated user group %s') % gr, category='success')
Session().commit()
except formencode.Invalid as errors:
ug_model = UserGroupModel()
defaults = errors.value
e = errors.error_dict or {}
defaults.update({
'create_repo_perm':
ug_model.has_perm(id, 'hg.create.repository'),
'fork_repo_perm':
ug_model.has_perm(id, 'hg.fork.repository'),
})
return htmlfill.render(
render('admin/user_groups/user_group_edit.html'),
defaults=defaults,
errors=e,
prefix_error=False,
encoding="UTF-8",
force_defaults=False)
except Exception:
log.error(traceback.format_exc())
h.flash(_('Error occurred during update of user group %s') %
request.POST.get('users_group_name'),
category='error')
raise HTTPFound(location=url('edit_users_group', id=id))
def index(self, repo_name, revision=None, f_path=None):
limit = 2000
default = 100
if request.GET.get('size'):
c.size = max(min(safe_int(request.GET.get('size')), limit), 1)
session['changelog_size'] = c.size
session.save()
else:
c.size = int(session.get('changelog_size', default))
# min size must be 1
c.size = max(c.size, 1)
p = safe_int(request.GET.get('page', 1), 1)
branch_name = request.GET.get('branch', None)
if (branch_name and branch_name not in c.db_repo_scm_instance.branches
and branch_name not in c.db_repo_scm_instance.closed_branches
and not revision):
return redirect(
url('changelog_file_home',
repo_name=c.repo_name,
revision=branch_name,
f_path=f_path or ''))
if revision == 'tip':
revision = None
c.changelog_for_path = f_path
try:
if f_path:
log.debug('generating changelog for path %s' % f_path)
# get the history for the file !
tip_cs = c.db_repo_scm_instance.get_changeset()
try:
collection = tip_cs.get_file_history(f_path)
except (NodeDoesNotExistError, ChangesetError):
#this node is not present at tip !
try:
cs = self.__get_cs(revision, repo_name)
collection = cs.get_file_history(f_path)
except RepositoryError, e:
h.flash(safe_str(e), category='warning')
redirect(h.url('changelog_home', repo_name=repo_name))
collection = list(reversed(collection))
else:
def make_ui(read_from='file', path=None, clear_session=True):
"""
A function that will read python rc files or database
and make an mercurial ui object from read options
:param path: path to mercurial config file
:param read_from: read from 'file' or 'db'
"""
baseui = ui.ui()
# clean the baseui object
baseui._ocfg = config.config()
baseui._ucfg = config.config()
baseui._tcfg = config.config()
if read_from == 'file':
if not os.path.isfile(path):
log.debug('hgrc file is not present at %s, skipping...', path)
return False
log.debug('reading hgrc from %s', path)
cfg = config.config()
cfg.read(path)
for section in ui_sections:
for k, v in cfg.items(section):
log.debug('settings ui from file: [%s] %s=%s', section, k, v)
baseui.setconfig(safe_str(section), safe_str(k), safe_str(v))
elif read_from == 'db':
sa = meta.Session()
ret = sa.query(Ui).all()
hg_ui = ret
for ui_ in hg_ui:
if ui_.ui_active:
ui_val = '' if ui_.ui_value is None else safe_str(ui_.ui_value)
log.debug('settings ui from db: [%s] %s=%r', ui_.ui_section,
ui_.ui_key, ui_val)
baseui.setconfig(safe_str(ui_.ui_section), safe_str(ui_.ui_key),
ui_val)
if clear_session:
meta.Session.remove()
# force set push_ssl requirement to False, Kallithea handles that
baseui.setconfig('web', 'push_ssl', False)
baseui.setconfig('web', 'allow_push', '*')
# prevent interactive questions for ssh password / passphrase
ssh = baseui.config('ui', 'ssh', default='ssh')
baseui.setconfig('ui', 'ssh', '%s -oBatchMode=yes -oIdentitiesOnly=yes' % ssh)
return baseui
def raw(self, repo_name, revision, f_path):
cs = self.__get_cs(revision)
file_node = self.__get_filenode(cs, f_path)
raw_mimetype_mapping = {
# map original mimetype to a mimetype used for "show as raw"
# you can also provide a content-disposition to override the
# default "attachment" disposition.
# orig_type: (new_type, new_dispo)
# show images inline:
'image/x-icon': ('image/x-icon', 'inline'),
'image/png': ('image/png', 'inline'),
'image/gif': ('image/gif', 'inline'),
'image/jpeg': ('image/jpeg', 'inline'),
'image/svg+xml': ('image/svg+xml', 'inline'),
}
mimetype = file_node.mimetype
try:
mimetype, dispo = raw_mimetype_mapping[mimetype]
except KeyError:
# we don't know anything special about this, handle it safely
if file_node.is_binary:
# do same as download raw for binary files
mimetype, dispo = 'application/octet-stream', 'attachment'
else:
# do not just use the original mimetype, but force text/plain,
# otherwise it would serve text/html and that might be unsafe.
# Note: underlying vcs library fakes text/plain mimetype if the
# mimetype can not be determined and it thinks it is not
# binary.This might lead to erroneous text display in some
# cases, but helps in other cases, like with text files
# without extension.
mimetype, dispo = 'text/plain', 'inline'
if dispo == 'attachment':
dispo = 'attachment; filename=%s' % \
safe_str(f_path.split(os.sep)[-1])
response.content_disposition = dispo
response.content_type = mimetype
return file_node.content
def show(self, gist_id, revision='tip', format='html', f_path=None):
c.gist = Gist.get_or_404(gist_id)
if c.gist.is_expired:
log.error('Gist expired at %s',
time_to_datetime(c.gist.gist_expires))
raise HTTPNotFound()
try:
c.file_changeset, c.files = GistModel().get_gist_files(
gist_id, revision=revision)
except VCSError:
log.error(traceback.format_exc())
raise HTTPNotFound()
if format == 'raw':
content = '\n\n'.join(
safe_str(f.content) for f in c.files
if (f_path is None or f.path == f_path))
response.content_type = 'text/plain'
return content
return render('admin/gists/show.html')
def _get_ref_rev(repo, ref_type, ref_name, returnempty=False):
"""
Safe way to get changeset. If error occurs show error.
"""
from kallithea.lib import helpers as h
try:
return repo.scm_instance.get_ref_revision(ref_type, ref_name)
except EmptyRepositoryError as e:
if returnempty:
return repo.scm_instance.EMPTY_CHANGESET
h.flash(h.literal(_('There are no changesets yet')),
category='error')
raise webob.exc.HTTPNotFound()
except ChangesetDoesNotExistError as e:
h.flash(h.literal(_('Changeset not found')), category='error')
raise webob.exc.HTTPNotFound()
except RepositoryError as e:
log.error(traceback.format_exc())
h.flash(safe_str(e), category='error')
raise webob.exc.HTTPBadRequest()
def _get_instance(self, cls, instance, callback=None):
"""
Gets instance of given cls using some simple lookup mechanism.
:param cls: class to fetch
:param instance: int or Instance
:param callback: callback to call if all lookups failed
"""
if isinstance(instance, cls):
return instance
elif isinstance(instance, (int, long)) or safe_str(instance).isdigit():
return cls.get(instance)
else:
if instance:
if callback is None:
raise Exception(
'given object must be int, long or Instance of %s '
'got %s, no callback provided' % (cls, type(instance)))
else:
return callback(instance)
def __get_filenode(self, cs, path):
"""
Returns file_node or raise HTTP error.
:param cs: given changeset
:param path: path to lookup
"""
try:
file_node = cs.get_node(path)
if file_node.is_dir():
raise RepositoryError('given path is a directory')
except ChangesetDoesNotExistError:
msg = _('Such revision does not exist for this repository')
h.flash(msg, category='error')
raise HTTPNotFound()
except RepositoryError as e:
h.flash(safe_str(e), category='error')
raise HTTPNotFound()
return file_node
def _escaper(string):
"""
Do HTML escaping/markup
"""
def substitute(m):
groups = m.groups()
if groups[0]:
return '&'
if groups[1]:
return '<'
if groups[2]:
return '>'
if groups[3]:
return '<u>\t</u>'
if groups[4]:
return '<u class="cr"></u>'
if groups[5]:
return ' <i></i>'
assert False
return _escape_re.sub(substitute, safe_str(string))
class ChangelogController(BaseRepoController):
def __before__(self):
super(ChangelogController, self).__before__()
c.affected_files_cut_off = 60
@staticmethod
def __get_cs(rev, repo):
"""
Safe way to get changeset. If error occur fail with error message.
:param rev: revision to fetch
:param repo: repo instance
"""
try:
return c.db_repo_scm_instance.get_changeset(rev)
except EmptyRepositoryError, e:
h.flash(h.literal(_('There are no changesets yet')),
category='error')
except RepositoryError, e:
log.error(traceback.format_exc())
h.flash(safe_str(e), category='error')
def repo_scan(self, repos_path=None):
"""
Listing of repositories in given path. This path should not be a
repository itself. Return a dictionary of repository objects
:param repos_path: path to directory containing repositories
"""
if repos_path is None:
repos_path = self.repos_path
log.info('scanning for repositories in %s', repos_path)
baseui = make_ui('db')
repos = {}
for name, path in get_filesystem_repos(repos_path):
# name need to be decomposed and put back together using the /
# since this is internal storage separator for kallithea
name = Repository.normalize_repo_name(name)
try:
if name in repos:
raise RepositoryError('Duplicate repository name %s '
'found in %s' % (name, path))
else:
klass = get_backend(path[0])
if path[0] == 'hg' and path[0] in BACKENDS.keys():
repos[name] = klass(safe_str(path[1]), baseui=baseui)
if path[0] == 'git' and path[0] in BACKENDS.keys():
repos[name] = klass(path[1])
except OSError:
continue
log.debug('found %s paths with repositories', len(repos))
return repos
def test_create(self):
self.log_user()
repo_name = self.NEW_REPO
description = u'description for newly created repo'
response = self.app.post(url('repos'),
fixture._get_repo_create_params(repo_private=False,
repo_name=repo_name,
repo_type=self.REPO_TYPE,
repo_description=description,
_authentication_token=self.authentication_token()))
## run the check page that triggers the flash message
response = self.app.get(url('repo_check_home', repo_name=repo_name))
assert response.json == {u'result': True}
self.checkSessionFlash(response,
'Created repository <a href="/%s">%s</a>'
% (repo_name, repo_name))
# test if the repo was created in the database
new_repo = Session().query(Repository) \
.filter(Repository.repo_name == repo_name).one()
assert new_repo.repo_name == repo_name
assert new_repo.description == description
# test if the repository is visible in the list ?
response = self.app.get(url('summary_home', repo_name=repo_name))
response.mustcontain(repo_name)
response.mustcontain(self.REPO_TYPE)
# test if the repository was created on filesystem
try:
vcs.get_repo(safe_str(os.path.join(Ui.get_by_key('paths', '/').ui_value, repo_name)))
except vcs.exceptions.VCSError:
pytest.fail('no repo %s in filesystem' % repo_name)
RepoModel().delete(repo_name)
Session().commit()
def add_doc(self, writer, path, repo, repo_name, index_rev=None):
"""
Adding doc to writer this function itself fetches data from
the instance of vcs backend
"""
try:
node = self.get_node(repo, path, index_rev)
except (ChangesetError, NodeDoesNotExistError):
log.debug(" >> %s - not found in %s %s", path, repo, index_rev)
return 0, 0
indexed = indexed_w_content = 0
if self.is_indexable_node(node):
bytes_content = node.content
if b'\0' in bytes_content:
log.warning(' >> %s - no text content', path)
u_content = ''
else:
log.debug(' >> %s', path)
u_content = safe_str(bytes_content)
indexed_w_content += 1
else:
log.debug(' >> %s - not indexable', path)
# just index file name without it's content
u_content = ''
indexed += 1
writer.add_document(fileid=path,
owner=repo.contact,
repository_rawname=repo_name,
repository=repo_name,
path=path,
content=u_content,
modtime=self.get_node_mtime(node),
extension=node.extension)
return indexed, indexed_w_content
def url_generator(**kw):
q = urllib.quote(safe_str(c.cur_query))
return update_params("?q=%s&type=%s" \
% (q, safe_str(c.cur_type)), **kw)
def handle_git_receive(repo_path, revs, env, hook_type):
"""
A really hacky method that is run by git post-receive hook and logs
a push action together with pushed revisions. It's executed by subprocess
thus needs all info to be able to create an on the fly app environment,
connect to database and run the logging code. Hacky as sh*t but works.
:param repo_path:
:param revs:
:param env:
"""
from paste.deploy import appconfig
from sqlalchemy import engine_from_config
from kallithea.config.environment import load_environment
from kallithea.model.base import init_model
from kallithea.model.db import Ui
from kallithea.lib.utils import make_ui, setup_cache_regions
extras = _extract_extras(env)
repo_path = safe_unicode(repo_path)
path, ini_name = os.path.split(extras['config'])
conf = appconfig('config:%s' % ini_name, relative_to=path)
conf = load_environment(conf.global_conf, conf.local_conf)
setup_cache_regions(conf)
engine = engine_from_config(conf, 'sqlalchemy.')
init_model(engine)
baseui = make_ui('db')
# fix if it's not a bare repo
if repo_path.endswith(os.sep + '.git'):
repo_path = repo_path[:-5]
repo = Repository.get_by_full_path(repo_path)
if not repo:
raise OSError('Repository %s not found in database' %
(safe_str(repo_path)))
_hooks = dict(baseui.configitems('hooks')) or {}
if hook_type == 'pre':
repo = repo.scm_instance
else:
# post push shouldn't use the cached instance never
repo = repo.scm_instance_no_cache()
if hook_type == 'pre':
pre_push(baseui, repo)
# if push hook is enabled via web interface
elif hook_type == 'post' and _hooks.get(Ui.HOOK_PUSH):
rev_data = []
for l in revs:
old_rev, new_rev, ref = l.strip().split(' ')
_ref_data = ref.split('/')
if _ref_data[1] in ['tags', 'heads']:
rev_data.append({
'old_rev': old_rev,
'new_rev': new_rev,
'ref': ref,
'type': _ref_data[1],
'name': '/'.join(_ref_data[2:])
})
git_revs = []
for push_ref in rev_data:
_type = push_ref['type']
if _type == 'heads':
if push_ref['old_rev'] == EmptyChangeset().raw_id:
# update the symbolic ref if we push new repo
if repo.is_empty():
repo._repo.refs.set_symbolic_ref(
'HEAD', 'refs/heads/%s' % push_ref['name'])
cmd = [
'for-each-ref', '--format=%(refname)', 'refs/heads/*'
]
heads = repo.run_git_command(cmd)[0]
cmd = [
'log', push_ref['new_rev'], '--reverse',
'--pretty=format:%H', '--not'
]
heads = heads.replace(push_ref['ref'], '')
for l in heads.splitlines():
cmd.append(l.strip())
git_revs += repo.run_git_command(cmd)[0].splitlines()
elif push_ref['new_rev'] == EmptyChangeset().raw_id:
#delete branch case
git_revs += ['delete_branch=>%s' % push_ref['name']]
else:
cmd = [
'log',
'%(old_rev)s..%(new_rev)s' % push_ref, '--reverse',
'--pretty=format:%H'
]
git_revs += repo.run_git_command(cmd)[0].splitlines()
elif _type == 'tags':
git_revs += ['tag=>%s' % push_ref['name']]
log_push_action(baseui, repo, _git_revs=git_revs)
def execute(self):
created_by = User.get(request.authuser.user_id)
pr = PullRequest()
pr.org_repo = self.org_repo
pr.org_ref = self.org_ref
pr.other_repo = self.other_repo
pr.other_ref = self.other_ref
pr.revisions = self.revisions
pr.title = self.title
pr.description = self.description
pr.owner = self.owner
Session().add(pr)
Session().flush() # make database assign pull_request_id
if self.org_repo.scm_instance.alias == 'git':
# create a ref under refs/pull/ so that commits don't get garbage-collected
self.org_repo.scm_instance._repo["refs/pull/%d/head" % pr.pull_request_id] = safe_str(self.org_rev)
#reset state to under-review
from kallithea.model.changeset_status import ChangesetStatusModel
from kallithea.model.comment import ChangesetCommentsModel
comment = ChangesetCommentsModel().create(
text=u'',
repo=self.org_repo,
author=created_by,
pull_request=pr,
send_email=False,
status_change=ChangesetStatus.STATUS_UNDER_REVIEW,
)
ChangesetStatusModel().set_status(
self.org_repo,
ChangesetStatus.STATUS_UNDER_REVIEW,
created_by,
comment,
pull_request=pr,
)
mention_recipients = extract_mentioned_users(self.description)
PullRequestModel().add_reviewers(created_by, pr, self.reviewers, mention_recipients)
return pr
def auth(self, userobj, username, password, settings, **kwargs):
"""
Gets the container_auth username (or email). It tries to get username
from REMOTE_USER if this plugin is enabled, if that fails
it tries to get username from HTTP_X_FORWARDED_USER if fallback header
is set. clean_username extracts the username from this data if it's
having @ in it.
Return None on failure. On success, return a dictionary of the form:
see: KallitheaAuthPluginBase.auth_func_attrs
:param userobj:
:param username:
:param password:
:param settings:
:param kwargs:
"""
environ = kwargs.get('environ')
if not environ:
log.debug('Empty environ data skipping...')
return None
if not userobj:
userobj = self.get_user('', environ=environ, settings=settings)
# we don't care passed username/password for container auth plugins.
# only way to log in is using environ
username = None
if userobj:
username = safe_str(getattr(userobj, 'username'))
if not username:
# we don't have any objects in DB, user doesn't exist, extract
# username from environ based on the settings
username = self._get_username(environ, settings)
# if cannot fetch username, it's a no-go for this plugin to proceed
if not username:
return None
# old attrs fetched from Kallithea database
admin = getattr(userobj, 'admin', False)
active = getattr(userobj, 'active', True)
email = environ.get(settings.get('email_header'),
getattr(userobj, 'email', ''))
firstname = environ.get(settings.get('firstname_header'),
getattr(userobj, 'firstname', ''))
lastname = environ.get(settings.get('lastname_header'),
getattr(userobj, 'lastname', ''))
user_data = {
'username': username,
'firstname': safe_unicode(firstname or username),
'lastname': safe_unicode(lastname or ''),
'groups': [],
'email': email or '',
'admin': admin or False,
'active': active,
'active_from_extern': True,
'extern_name': username,
}
log.info('user `%s` authenticated correctly', user_data['username'])
return user_data
def _create_filesystem_repo(self,
repo_name,
repo_type,
repo_group,
clone_uri=None,
repo_store_location=None):
"""
Makes repository on filesystem. Operation is group aware, meaning that it will create
a repository within a group, and alter the paths accordingly to the group location.
:param repo_name:
:param alias:
:param parent:
:param clone_uri:
:param repo_store_location:
"""
from kallithea.lib.utils import is_valid_repo, is_valid_repo_group
from kallithea.model.scm import ScmModel
if '/' in repo_name:
raise ValueError('repo_name must not contain groups got `%s`' %
repo_name)
if isinstance(repo_group, RepoGroup):
new_parent_path = os.sep.join(repo_group.full_path_splitted)
else:
new_parent_path = repo_group or ''
if repo_store_location:
_paths = [repo_store_location]
else:
_paths = [self.repos_path, new_parent_path, repo_name]
# we need to make it str for mercurial
repo_path = os.path.join(*map(lambda x: safe_str(x), _paths))
# check if this path is not a repository
if is_valid_repo(repo_path, self.repos_path):
raise Exception('This path %s is a valid repository' % repo_path)
# check if this path is a group
if is_valid_repo_group(repo_path, self.repos_path):
raise Exception('This path %s is a valid group' % repo_path)
log.info('creating repo %s in %s from url: `%s`', repo_name,
safe_unicode(repo_path), obfuscate_url_pw(clone_uri))
backend = get_backend(repo_type)
if repo_type == 'hg':
baseui = make_ui('db', clear_session=False)
# patch and reset hooks section of UI config to not run any
# hooks on creating remote repo
for k, v in baseui.configitems('hooks'):
baseui.setconfig('hooks', k, None)
repo = backend(repo_path,
create=True,
src_url=clone_uri,
baseui=baseui)
elif repo_type == 'git':
repo = backend(repo_path,
create=True,
src_url=clone_uri,
bare=True)
# add kallithea hook into this repo
ScmModel().install_git_hooks(repo=repo)
else:
raise Exception('Not supported repo_type %s expected hg/git' %
repo_type)
log.debug('Created repo %s with %s backend', safe_unicode(repo_name),
safe_unicode(repo_type))
return repo
def index(self, repo_name, revision=None, f_path=None):
# Fix URL after page size form submission via GET
# TODO: Somehow just don't send this extra junk in the GET URL
if request.GET.get('set'):
request.GET.pop('set', None)
if revision is None:
raise HTTPFound(location=url(
'changelog_home', repo_name=repo_name, **request.GET))
raise HTTPFound(location=url('changelog_file_home',
repo_name=repo_name,
revision=revision,
f_path=f_path,
**request.GET))
limit = 2000
default = 100
if request.GET.get('size'):
c.size = max(min(safe_int(request.GET.get('size')), limit), 1)
session['changelog_size'] = c.size
session.save()
else:
c.size = int(session.get('changelog_size', default))
# min size must be 1
c.size = max(c.size, 1)
p = safe_int(request.GET.get('page'), 1)
branch_name = request.GET.get('branch', None)
if (branch_name and branch_name not in c.db_repo_scm_instance.branches
and branch_name not in c.db_repo_scm_instance.closed_branches
and not revision):
raise HTTPFound(location=url('changelog_file_home',
repo_name=c.repo_name,
revision=branch_name,
f_path=f_path or ''))
if revision == 'tip':
revision = None
c.changelog_for_path = f_path
try:
if f_path:
log.debug('generating changelog for path %s', f_path)
# get the history for the file !
tip_cs = c.db_repo_scm_instance.get_changeset()
try:
collection = tip_cs.get_file_history(f_path)
except (NodeDoesNotExistError, ChangesetError):
#this node is not present at tip !
try:
cs = self.__get_cs(revision, repo_name)
collection = cs.get_file_history(f_path)
except RepositoryError as e:
h.flash(safe_str(e), category='warning')
raise HTTPFound(location=h.url('changelog_home',
repo_name=repo_name))
collection = list(reversed(collection))
else:
collection = c.db_repo_scm_instance.get_changesets(
start=0, end=revision, branch_name=branch_name)
c.total_cs = len(collection)
c.pagination = RepoPage(
collection,
page=p,
item_count=c.total_cs,
items_per_page=c.size,
branch=branch_name,
)
page_revisions = [x.raw_id for x in c.pagination]
c.comments = c.db_repo.get_comments(page_revisions)
c.statuses = c.db_repo.statuses(page_revisions)
except EmptyRepositoryError as e:
h.flash(safe_str(e), category='warning')
raise HTTPFound(
location=url('summary_home', repo_name=c.repo_name))
except (RepositoryError, ChangesetDoesNotExistError, Exception) as e:
log.error(traceback.format_exc())
h.flash(safe_str(e), category='error')
raise HTTPFound(
location=url('changelog_home', repo_name=c.repo_name))
c.branch_name = branch_name
c.branch_filters = [('', _('None'))] + \
[(k, k) for k in c.db_repo_scm_instance.branches.keys()]
if c.db_repo_scm_instance.closed_branches:
prefix = _('(closed)') + ' '
c.branch_filters += [('-', '-')] + \
[(k, prefix + k) for k in c.db_repo_scm_instance.closed_branches.keys()]
revs = []
if not f_path:
revs = [x.revision for x in c.pagination]
c.jsdata = graph_data(c.db_repo_scm_instance, revs)
c.revision = revision # requested revision ref
c.first_revision = c.pagination[0] # pagination is never empty here!
return render('changelog/changelog.html')
def setUp(self):
self.repo = MercurialRepository(safe_str(TEST_HG_REPO))
def authenticate_ldap(self, username, password):
"""
Authenticate a user via LDAP and return his/her LDAP properties.
Raises AuthenticationError if the credentials are rejected, or
EnvironmentError if the LDAP server can't be reached.
:param username: username
:param password: password
"""
if not password:
log.debug("Attempt to authenticate LDAP user "
"with blank password rejected.")
raise LdapPasswordError()
if "," in username:
raise LdapUsernameError("invalid character in username: ,")
try:
if self.cacertdir:
if hasattr(ldap, 'OPT_X_TLS_CACERTDIR'):
ldap.set_option(ldap.OPT_X_TLS_CACERTDIR, self.cacertdir)
else:
log.debug(
"OPT_X_TLS_CACERTDIR is not available - can't set %s",
self.cacertdir)
ldap.set_option(ldap.OPT_REFERRALS, ldap.OPT_OFF)
ldap.set_option(ldap.OPT_RESTART, ldap.OPT_ON)
ldap.set_option(ldap.OPT_TIMEOUT, 20)
ldap.set_option(ldap.OPT_NETWORK_TIMEOUT, 10)
ldap.set_option(ldap.OPT_TIMELIMIT, 15)
if self.TLS_KIND != 'PLAIN':
ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, self.TLS_REQCERT)
server = ldap.initialize(self.LDAP_SERVER)
if self.ldap_version == 2:
server.protocol = ldap.VERSION2
else:
server.protocol = ldap.VERSION3
if self.TLS_KIND == 'START_TLS':
server.start_tls_s()
if self.LDAP_BIND_DN and self.LDAP_BIND_PASS:
log.debug('Trying simple_bind with password and given DN: %s',
self.LDAP_BIND_DN)
server.simple_bind_s(self.LDAP_BIND_DN, self.LDAP_BIND_PASS)
filter_ = '(&%s(%s=%s))' % (
self.LDAP_FILTER,
ldap.filter.escape_filter_chars(self.attr_login),
ldap.filter.escape_filter_chars(username))
log.debug("Authenticating %r filter %s at %s", self.BASE_DN,
filter_, self.LDAP_SERVER)
lobjects = server.search_ext_s(self.BASE_DN, self.SEARCH_SCOPE,
filter_)
if not lobjects:
raise ldap.NO_SUCH_OBJECT()
for (dn, _attrs) in lobjects:
if dn is None:
continue
try:
log.debug('Trying simple bind with %s', dn)
server.simple_bind_s(dn, safe_str(password))
results = server.search_ext_s(dn, ldap.SCOPE_BASE,
'(objectClass=*)')
if len(results) == 1:
dn_, attrs = results[0]
assert dn_ == dn
return dn, attrs
except ldap.INVALID_CREDENTIALS:
log.debug("LDAP rejected password for user '%s': %s",
username, dn)
continue # accept authentication as another ldap user with same username
log.debug("No matching LDAP objects for authentication "
"of '%s'", username)
raise LdapPasswordError()
except ldap.NO_SUCH_OBJECT:
log.debug("LDAP says no such user '%s'", username)
raise LdapUsernameError()
except ldap.SERVER_DOWN:
# [0] might be {'info': "TLS error -8179:Peer's Certificate issuer is not recognized.", 'desc': "Can't contact LDAP server"}
raise LdapConnectionError(
"LDAP can't connect to authentication server")
def _authorize(self, environ, start_response, action, repo_name, ip_addr):
"""Authenticate and authorize user.
Since we're dealing with a VCS client and not a browser, we only
support HTTP basic authentication, either directly via raw header
inspection, or by using container authentication to delegate the
authentication to the web server.
Returns (user, None) on successful authentication and authorization.
Returns (None, wsgi_app) to send the wsgi_app response to the client.
"""
# Check if anonymous access is allowed.
default_user = User.get_default_user(cache=True)
is_default_user_allowed = (default_user.active and
self._check_permission(action, default_user, repo_name, ip_addr))
if is_default_user_allowed:
return default_user, None
if not default_user.active:
log.debug('Anonymous access is disabled')
else:
log.debug('Not authorized to access this '
'repository as anonymous user')
username = None
#==============================================================
# DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE
# NEED TO AUTHENTICATE AND ASK FOR AUTH USER PERMISSIONS
#==============================================================
# try to auth based on environ, container auth methods
log.debug('Running PRE-AUTH for container based authentication')
pre_auth = auth_modules.authenticate('', '', environ)
if pre_auth is not None and pre_auth.get('username'):
username = pre_auth['username']
log.debug('PRE-AUTH got %s as username', username)
# If not authenticated by the container, running basic auth
if not username:
self.authenticate.realm = safe_str(self.config['realm'])
result = self.authenticate(environ)
if isinstance(result, str):
paste.httpheaders.AUTH_TYPE.update(environ, 'basic')
paste.httpheaders.REMOTE_USER.update(environ, result)
username = result
else:
return None, result.wsgi_application
#==============================================================
# CHECK PERMISSIONS FOR THIS REQUEST USING GIVEN USERNAME
#==============================================================
try:
user = User.get_by_username_or_email(username)
if user is None or not user.active:
return None, webob.exc.HTTPForbidden()
except Exception:
log.error(traceback.format_exc())
return None, webob.exc.HTTPInternalServerError()
#check permissions for this repository
perm = self._check_permission(action, user, repo_name, ip_addr)
if not perm:
return None, webob.exc.HTTPForbidden()
return user, None