def _commit_change(repo,
filename,
content,
message,
vcs_type,
parent=None,
newfile=False):
repo = Repository.get_by_repo_name(repo)
_cs = parent
if not parent:
_cs = EmptyChangeset(alias=vcs_type)
if newfile:
cs = ScmModel().create_node(repo=repo.scm_instance,
repo_name=repo.repo_name,
cs=_cs,
user=TEST_USER_ADMIN_LOGIN,
author=TEST_USER_ADMIN_LOGIN,
message=message,
content=content,
f_path=filename)
else:
cs = ScmModel().commit_change(repo=repo.scm_instance,
repo_name=repo.repo_name,
cs=parent,
user=TEST_USER_ADMIN_LOGIN,
author=TEST_USER_ADMIN_LOGIN,
message=message,
content=content,
f_path=filename)
return cs
def _commit_change(
repo, filename, content, message, vcs_type, parent=None,
newfile=False):
repo = Repository.get_by_repo_name(repo)
_commit = parent
if not parent:
_commit = EmptyCommit(alias=vcs_type)
if newfile:
nodes = {
filename: {
'content': content
}
}
commit = ScmModel().create_nodes(
user=TEST_USER_ADMIN_LOGIN, repo=repo,
message=message,
nodes=nodes,
parent_commit=_commit,
author=TEST_USER_ADMIN_LOGIN,
)
else:
commit = ScmModel().commit_change(
repo=repo.scm_instance(), repo_name=repo.repo_name,
commit=parent, user=TEST_USER_ADMIN_LOGIN,
author=TEST_USER_ADMIN_LOGIN,
message=message,
content=content,
f_path=filename
)
return commit
def __init__(self, indexname=IDX_NAME, index_location=None,
repo_location=None, sa=None, repo_list=None):
self.indexname = indexname
self.index_location = index_location
if not index_location:
raise Exception('You have to provide index location')
self.repo_location = repo_location
if not repo_location:
raise Exception('You have to provide repositories location')
self.repo_paths = ScmModel(sa).repo_scan(self.repo_location)
if repo_list:
filtered_repo_paths = {}
for repo_name, repo in self.repo_paths.items():
if repo_name in repo_list:
filtered_repo_paths[repo_name] = repo
self.repo_paths = filtered_repo_paths
self.initial = False
if not os.path.isdir(self.index_location):
os.makedirs(self.index_location)
log.info('Cannot run incremental index since it does not'
' yet exist running full build')
self.initial = True
def __init__(self,
indexname=IDX_NAME,
index_location=None,
repo_location=None,
sa=None,
repo_list=None,
repo_update_list=None):
self.indexname = indexname
self.index_location = index_location
if not index_location:
raise Exception('You have to provide index location')
self.repo_location = repo_location
if not repo_location:
raise Exception('You have to provide repositories location')
self.repo_paths = ScmModel(sa).repo_scan(self.repo_location)
#filter repo list
if repo_list:
#Fix non-ascii repo names to unicode
repo_list = map(safe_unicode, repo_list)
self.filtered_repo_paths = {}
for repo_name, repo in self.repo_paths.items():
if repo_name in repo_list:
self.filtered_repo_paths[repo_name] = repo
self.repo_paths = self.filtered_repo_paths
#filter update repo list
self.filtered_repo_update_paths = {}
if repo_update_list:
self.filtered_repo_update_paths = {}
for repo_name, repo in self.repo_paths.items():
if repo_name in repo_update_list:
self.filtered_repo_update_paths[repo_name] = repo
self.repo_paths = self.filtered_repo_update_paths
self.initial = True
if not os.path.isdir(self.index_location):
os.makedirs(self.index_location)
log.info('Cannot run incremental index since it does not '
'yet exist running full build')
elif not exists_in(self.index_location, IDX_NAME):
log.info('Running full index build as the file content '
'index does not exist')
elif not exists_in(self.index_location, CHGSET_IDX_NAME):
log.info('Running full index build as the changeset '
'index does not exist')
else:
self.initial = False
def __before__(self):
"""
__before__ is called before controller methods and after __call__
"""
# on each call propagate settings calls into global settings.
set_rhodecode_config(config)
attach_context_attributes(c)
# TODO: Remove this when fixed in attach_context_attributes()
c.repo_name = get_repo_slug(request) # can be empty
self.cut_off_limit_diff = safe_int(config.get('cut_off_limit_diff'))
self.cut_off_limit_file = safe_int(config.get('cut_off_limit_file'))
self.sa = meta.Session
self.scm_model = ScmModel(self.sa)
default_lang = c.language
user_lang = c.language
try:
user_obj = self._rhodecode_user.get_instance()
if user_obj:
user_lang = user_obj.user_data.get('language')
except Exception:
log.exception('Failed to fetch user language for user %s',
self._rhodecode_user)
if user_lang and user_lang != default_lang:
log.debug('set language to %s for user %s', user_lang,
self._rhodecode_user)
translation.set_lang(user_lang)
def invalidate_cache(self, apiuser, repoid):
"""
Dispatch cache invalidation action on given repo
:param apiuser:
:param repoid:
"""
repo = get_repo_or_error(repoid)
if HasPermissionAnyApi('hg.admin')(user=apiuser) is False:
# check if we have admin permission for this repo !
if HasRepoPermissionAnyApi('repository.admin',
'repository.write')(user=apiuser,
repo_name=repo.repo_name) is False:
raise JSONRPCError('repository `%s` does not exist' % (repoid))
try:
invalidated_keys = ScmModel().mark_for_invalidation(repo.repo_name)
Session().commit()
return ('Cache for repository `%s` was invalidated: '
'invalidated cache keys: %s' % (repoid, invalidated_keys))
except Exception:
log.error(traceback.format_exc())
raise JSONRPCError(
'Error occurred during cache invalidation action'
)
def extract_svn_repo_from_dump(dump_name, repo_name):
"""Create a svn repo `repo_name` from dump `dump_name`."""
repos_path = ScmModel().repos_path
target_path = os.path.join(repos_path, repo_name)
SubversionRepository(target_path, create=True)
_load_svn_dump_into_repo(dump_name, target_path)
return target_path
def __before__(self):
"""
__before__ is called before controller methods and after __call__
"""
c.rhodecode_version = __version__
c.rhodecode_instanceid = config.get('instance_id')
c.rhodecode_name = config.get('rhodecode_title')
c.rhodecode_bugtracker = config.get('bugtracker', 'http://bitbucket.org/marcinkuzminski/rhodecode/issues')
c.use_gravatar = str2bool(config.get('use_gravatar'))
c.ga_code = config.get('rhodecode_ga_code')
# Visual options
c.visual = AttributeDict({})
rc_config = RhodeCodeSetting.get_app_settings()
## DB stored
c.visual.show_public_icon = str2bool(rc_config.get('rhodecode_show_public_icon'))
c.visual.show_private_icon = str2bool(rc_config.get('rhodecode_show_private_icon'))
c.visual.stylify_metatags = str2bool(rc_config.get('rhodecode_stylify_metatags'))
c.visual.dashboard_items = safe_int(rc_config.get('rhodecode_dashboard_items', 100))
c.visual.repository_fields = str2bool(rc_config.get('rhodecode_repository_fields'))
c.visual.show_version = str2bool(rc_config.get('rhodecode_show_version'))
## INI stored
self.cut_off_limit = int(config.get('cut_off_limit'))
c.visual.allow_repo_location_change = str2bool(config.get('allow_repo_location_change', True))
c.visual.allow_custom_hooks_settings = str2bool(config.get('allow_custom_hooks_settings', True))
c.repo_name = get_repo_slug(request) # can be empty
c.backends = BACKENDS.keys()
c.unread_notifications = NotificationModel()\
.get_unread_cnt_for_user(c.rhodecode_user.user_id)
self.sa = meta.Session
self.scm_model = ScmModel(self.sa)
def get_repo_nodes(self, apiuser, repo_name, revision, root_path,
ret_type='all'):
"""
returns a list of nodes and it's children
for a given path at given revision. It's possible to specify ret_type
to show only files or dirs
:param apiuser:
:param repo_name: name of repository
:param revision: revision for which listing should be done
:param root_path: path from which start displaying
:param ret_type: return type 'all|files|dirs' nodes
"""
try:
_d, _f = ScmModel().get_nodes(repo_name, revision, root_path,
flat=False)
_map = {
'all': _d + _f,
'files': _f,
'dirs': _d,
}
return _map[ret_type]
except KeyError:
raise JSONRPCError('ret_type must be one of %s' % _map.keys())
except Exception, e:
raise JSONRPCError(e)
def index(self):
# Return a rendered template
p = int(request.params.get('page', 1))
c.user = User.get(self.rhodecode_user.user_id)
all_repos = self.sa.query(Repository)\
.filter(Repository.user_id == c.user.user_id)\
.order_by(func.lower(Repository.repo_name)).all()
c.user_repos = ScmModel().get_repos(all_repos)
c.following = self.sa.query(UserFollowing)\
.filter(UserFollowing.user_id == self.rhodecode_user.user_id)\
.options(joinedload(UserFollowing.follows_repository))\
.all()
journal = self._get_journal_data(c.following)
c.journal_pager = Page(journal, page=p, items_per_page=20)
c.journal_day_aggreagate = self._get_daily_aggregate(c.journal_pager)
c.journal_data = render('journal/journal_data.html')
if request.environ.get('HTTP_X_PARTIAL_XHR'):
return c.journal_data
return render('journal/journal.html')
def _invalidate_cache(self, repo_name):
"""
Set's cache for this repository for invalidation on next access
:param repo_name: full repo name, also a cache key
"""
ScmModel().mark_for_invalidation(repo_name)
def edit_advanced_fork(self, repo_name):
"""
Mark given repository as a fork of another
:param repo_name:
"""
new_fork_id = request.POST.get('id_fork_of')
try:
if new_fork_id and not new_fork_id.isdigit():
log.error('Given fork id %s is not an INT', new_fork_id)
fork_id = safe_int(new_fork_id)
repo = ScmModel().mark_as_fork(repo_name, fork_id,
c.rhodecode_user.username)
fork = repo.fork.repo_name if repo.fork else _('Nothing')
Session().commit()
h.flash(_('Marked repo %s as fork of %s') % (repo_name, fork),
category='success')
except RepositoryError as e:
log.exception("Repository Error occurred")
h.flash(str(e), category='error')
except Exception as e:
log.exception("Exception while editing fork")
h.flash(_('An error occurred during this operation'),
category='error')
return redirect(url('edit_repo_advanced', repo_name=repo_name))
def get_server_info(request, apiuser):
"""
Returns the |RCE| server information.
This includes the running version of |RCE| and all installed
packages. This command takes the following options:
:param apiuser: This is filled automatically from the |authtoken|.
:type apiuser: AuthUser
Example output:
.. code-block:: bash
id : <id_given_in_input>
result : {
'modules': [<module name>,...]
'py_version': <python version>,
'platform': <platform type>,
'rhodecode_version': <rhodecode version>
}
error : null
"""
if not has_superadmin_permission(apiuser):
raise JSONRPCForbidden()
return ScmModel().get_server_info(request.environ)
def create_repository(self):
"""GET /_admin/create_repository: Form to create a new item"""
new_repo = request.GET.get('repo', '')
parent_group = request.GET.get('parent_group')
if not HasPermissionAny('hg.admin', 'hg.create.repository')():
#you're not super admin nor have global create permissions,
#but maybe you have at least write permission to a parent group ?
_gr = RepoGroup.get(parent_group)
gr_name = _gr.group_name if _gr else None
if not HasReposGroupPermissionAny(
'group.admin', 'group.write')(group_name=gr_name):
raise HTTPForbidden
acl_groups = GroupList(RepoGroup.query().all(),
perm_set=['group.write', 'group.admin'])
c.repo_groups = RepoGroup.groups_choices(groups=acl_groups)
c.repo_groups_choices = map(lambda k: unicode(k[0]), c.repo_groups)
choices, c.landing_revs = ScmModel().get_repo_landing_revs()
c.new_repo = repo_name_slug(new_repo)
## apply the defaults from defaults page
defaults = RhodeCodeSetting.get_default_repo_settings(
strip_prefix=True)
if parent_group:
defaults.update({'repo_group': parent_group})
return htmlfill.render(render('admin/repos/repo_add.html'),
defaults=defaults,
errors={},
prefix_error=False,
encoding="UTF-8")
def update(self, gist, description, owner, gist_mapping, gist_type,
lifetime, gist_acl_level):
gist = self._get_gist(gist)
gist_repo = gist.scm_instance()
lifetime = safe_int(lifetime, -1)
if lifetime == 0: # preserve old value
gist_expires = gist.gist_expires
else:
gist_expires = (
time.time() + (lifetime * 60) if lifetime != -1 else -1)
# calculate operation type based on given data
gist_mapping_op = {}
for k, v in gist_mapping.items():
# add, mod, del
if not v['org_filename'] and v['filename']:
op = 'add'
elif v['org_filename'] and not v['filename']:
op = 'del'
else:
op = 'mod'
v['op'] = op
gist_mapping_op[k] = v
gist.gist_description = description
gist.gist_expires = gist_expires
gist.owner = owner
gist.gist_type = gist_type
gist.acl_level = gist_acl_level
self.sa.add(gist)
self.sa.flush()
message = 'updated file'
message += 's: ' if len(gist_mapping) > 1 else ': '
message += ', '.join([x for x in gist_mapping])
# fake RhodeCode Repository object
fake_repo = AttributeDict({
'repo_name': gist_repo.path,
'scm_instance': lambda *args, **kwargs: gist_repo,
})
self._store_metadata(gist_repo, gist.gist_id, gist.gist_access_id,
owner.user_id, owner.username, gist.gist_type,
gist.gist_expires, gist_acl_level)
# this can throw NodeNotChangedError, if changes we're trying to commit
# are not actually changes...
ScmModel().update_nodes(
user=owner.user_id,
repo=fake_repo,
message=message,
nodes=gist_mapping_op,
trigger_push_hook=False
)
return gist
def index(self, format='html'):
"""GET /repos: All items in the collection"""
# url('repos')
c.repos_list = ScmModel().get_repos(Repository.query()
.order_by(Repository.repo_name)
.all(), sort_key='name_sort')
return render('admin/repos/repos.html')
def _cached_nodes():
log.debug('Generating cached nodelist for %s, %s, %s', repo_name,
commit_id, f_path)
_d, _f = ScmModel().get_nodes(repo_name,
commit_id,
f_path,
flat=False)
return _d + _f
def nodelist(self, repo_name, revision, f_path):
if request.environ.get('HTTP_X_PARTIAL_XHR'):
cs = self.__get_cs_or_redirect(revision, repo_name)
_d, _f = ScmModel().get_nodes(repo_name,
cs.raw_id,
f_path,
flat=False)
return {'nodes': _d + _f}
def update(self, repo_name):
"""
PUT /repos/repo_name: Update an existing item"""
# Forms posted to this method should contain a hidden field:
# <input type="hidden" name="_method" value="PUT" />
# Or using helpers:
# h.form(url('repo', repo_name=ID),
# method='put')
# url('repo', repo_name=ID)
self.__load_data(repo_name)
c.active = 'settings'
c.repo_fields = RepositoryField.query()\
.filter(RepositoryField.repository == c.repo_info).all()
repo_model = RepoModel()
changed_name = repo_name
# override the choices with extracted revisions !
c.personal_repo_group = RepoGroup.get_by_group_name(
c.rhodecode_user.username)
repo = Repository.get_by_repo_name(repo_name)
old_data = {
'repo_name': repo_name,
'repo_group': repo.group.get_dict() if repo.group else {},
'repo_type': repo.repo_type,
}
_form = RepoForm(edit=True,
old_data=old_data,
repo_groups=c.repo_groups_choices,
landing_revs=c.landing_revs_choices,
allow_disabled=True)()
try:
form_result = _form.to_python(dict(request.POST))
repo = repo_model.update(repo_name, **form_result)
ScmModel().mark_for_invalidation(repo_name)
h.flash(_('Repository %s updated successfully') % repo_name,
category='success')
changed_name = repo.repo_name
action_logger(c.rhodecode_user, 'admin_updated_repo', changed_name,
self.ip_addr, self.sa)
Session().commit()
except formencode.Invalid as errors:
defaults = self.__load_data(repo_name)
defaults.update(errors.value)
return htmlfill.render(render('admin/repos/repo_edit.html'),
defaults=defaults,
errors=errors.error_dict or {},
prefix_error=False,
encoding="UTF-8",
force_defaults=False)
except Exception:
log.exception("Exception during update of repository")
h.flash(_('Error occurred during update of repository %s') \
% repo_name, category='error')
return redirect(url('edit_repo', repo_name=changed_name))
def __load_defaults(self):
acl_groups = RepoGroupList(RepoGroup.query().all(),
perm_set=['group.write', 'group.admin'])
c.repo_groups = RepoGroup.groups_choices(groups=acl_groups)
c.repo_groups_choices = map(lambda k: unicode(k[0]), c.repo_groups)
choices, c.landing_revs = ScmModel().get_repo_landing_revs()
c.landing_revs_choices = choices
c.can_update = RhodeCodeUi.get_by_key(
RhodeCodeUi.HOOK_UPDATE).ui_active
def update(self, repo_name):
"""
PUT /repos/repo_name: Update an existing item"""
# Forms posted to this method should contain a hidden field:
# <input type="hidden" name="_method" value="PUT" />
# Or using helpers:
# h.form(url('repo', repo_name=ID),
# method='put')
# url('repo', repo_name=ID)
self.__load_defaults()
repo_model = RepoModel()
changed_name = repo_name
#override the choices with extracted revisions !
choices, c.landing_revs = ScmModel().get_repo_landing_revs(repo_name)
c.landing_revs_choices = choices
repo = Repository.get_by_repo_name(repo_name)
old_data = {
'repo_name': repo_name,
'repo_group': repo.group.get_dict() if repo.group else {},
'repo_type': repo.repo_type,
}
_form = RepoForm(edit=True,
old_data=old_data,
repo_groups=c.repo_groups_choices,
landing_revs=c.landing_revs_choices)()
try:
form_result = _form.to_python(dict(request.POST))
repo = repo_model.update(repo_name, **form_result)
ScmModel().mark_for_invalidation(repo_name)
h.flash(_('Repository %s updated successfully') % repo_name,
category='success')
changed_name = repo.repo_name
action_logger(self.rhodecode_user, 'admin_updated_repo',
changed_name, self.ip_addr, self.sa)
Session().commit()
except formencode.Invalid, errors:
defaults = self.__load_data(repo_name)
defaults.update(errors.value)
return htmlfill.render(render('admin/repos/repo_edit.html'),
defaults=defaults,
errors=errors.error_dict or {},
prefix_error=False,
encoding="UTF-8")
def test_remove_repo(self, backend):
repo = backend.create_repo()
Session().commit()
RepoModel().delete(repo=repo)
Session().commit()
repos = ScmModel().repo_scan()
assert Repository.get_by_repo_name(repo_name=backend.repo_name) is None
assert repo.repo_name not in repos
def command(self):
#get SqlAlchemy session
self._init_session()
rm_obsolete = self.options.delete_obsolete
log.info('Now scanning root location for new repos...')
added, removed = repo2db_mapper(ScmModel().repo_scan(),
remove_obsolete=rm_obsolete)
added = ', '.join(added) or '-'
removed = ', '.join(removed) or '-'
log.info('Scan completed added: %s removed: %s' % (added, removed))
def edit_remote(self, repo_name):
"""PUT /{repo_name}/settings/remote: edit the repo remote."""
try:
ScmModel().pull_changes(repo_name, c.rhodecode_user.username)
h.flash(_('Pulled from remote location'), category='success')
except Exception:
log.exception("Exception during pull from remote")
h.flash(_('An error occurred during pull from remote location'),
category='error')
return redirect(url('edit_repo_remote', repo_name=c.repo_name))
def __load_defaults(self):
acl_groups = RepoGroupList(
RepoGroup.query().all(),
perm_set=['group.write', 'group.admin'])
c.repo_groups = RepoGroup.groups_choices(groups=acl_groups)
c.repo_groups_choices = map(lambda k: unicode(k[0]), c.repo_groups)
choices, c.landing_revs = ScmModel().get_repo_landing_revs()
c.landing_revs_choices = choices
c.personal_repo_group = RepoGroup.get_by_group_name(
c.rhodecode_user.username)
def __create_repo(self,
repo_name,
alias,
parent,
clone_uri=False,
repo_store_location=None):
"""
makes repository on filesystem. It's group aware means it'll create
a repository within a group, and alter the paths accordingly of
group location
:param repo_name:
:param alias:
:param parent_id:
:param clone_uri:
:param repo_path:
"""
from rhodecode.lib.utils import is_valid_repo, is_valid_repos_group
from rhodecode.model.scm import ScmModel
if parent:
new_parent_path = os.sep.join(parent.full_path_splitted)
else:
new_parent_path = ''
if repo_store_location:
_paths = [repo_store_location]
else:
_paths = [self.repos_path, new_parent_path, repo_name]
# we need to make it str for mercurial
repo_path = os.path.join(*map(lambda x: safe_str(x), _paths))
# check if this path is not a repository
if is_valid_repo(repo_path, self.repos_path):
raise Exception('This path %s is a valid repository' % repo_path)
# check if this path is a group
if is_valid_repos_group(repo_path, self.repos_path):
raise Exception('This path %s is a valid group' % repo_path)
log.info(
'creating repo %s in %s @ %s' %
(repo_name, safe_unicode(repo_path), obfuscate_url_pw(clone_uri)))
backend = get_backend(alias)
if alias == 'hg':
repo = backend(repo_path, create=True, src_url=clone_uri)
elif alias == 'git':
repo = backend(repo_path,
create=True,
src_url=clone_uri,
bare=True)
# add rhodecode hook into this repo
ScmModel().install_git_hook(repo=repo)
else:
raise Exception('Undefined alias %s' % alias)
return repo
def invalidate_cache(cache_key, *args):
"""
Puts cache invalidation task into db for
further global cache invalidation
"""
from rhodecode.model.scm import ScmModel
if cache_key.startswith('get_repo_cached_'):
name = cache_key.split('get_repo_cached_')[-1]
ScmModel().mark_for_invalidation(name)
def __load_defaults(self):
acl_groups = GroupList(RepoGroup.query().all(),
perm_set=['group.write', 'group.admin'])
c.repo_groups = RepoGroup.groups_choices(groups=acl_groups)
c.repo_groups_choices = map(lambda k: unicode(k[0]), c.repo_groups)
repo_model = RepoModel()
c.users_array = repo_model.get_users_js()
c.users_groups_array = repo_model.get_users_groups_js()
choices, c.landing_revs = ScmModel().get_repo_landing_revs()
c.landing_revs_choices = choices
def rescan_repos(request, apiuser, remove_obsolete=Optional(False)):
"""
Triggers a rescan of the specified repositories.
* If the ``remove_obsolete`` option is set, it also deletes repositories
that are found in the database but not on the file system, so called
"clean zombies".
This command can only be run using an |authtoken| with admin rights to
the specified repository.
This command takes the following options:
:param apiuser: This is filled automatically from the |authtoken|.
:type apiuser: AuthUser
:param remove_obsolete: Deletes repositories from the database that
are not found on the filesystem.
:type remove_obsolete: Optional(``True`` | ``False``)
Example output:
.. code-block:: bash
id : <id_given_in_input>
result : {
'added': [<added repository name>,...]
'removed': [<removed repository name>,...]
}
error : null
Example error output:
.. code-block:: bash
id : <id_given_in_input>
result : null
error : {
'Error occurred during rescan repositories action'
}
"""
if not has_superadmin_permission(apiuser):
raise JSONRPCForbidden()
try:
rm_obsolete = Optional.extract(remove_obsolete)
added, removed = repo2db_mapper(ScmModel().repo_scan(),
remove_obsolete=rm_obsolete)
return {'added': added, 'removed': removed}
except Exception:
log.exception('Failed to run repo rescann')
raise JSONRPCError(
'Error occurred during rescan repositories action'
)
def __load_data(self, repo_name=None):
"""
Load defaults settings for edit, and update
:param repo_name:
"""
self.__load_defaults()
c.repo_info = db_repo = Repository.get_by_repo_name(repo_name)
repo = db_repo.scm_instance
if c.repo_info is None:
h.not_mapped_error(repo_name)
return redirect(url('repos'))
##override defaults for exact repo info here git/hg etc
choices, c.landing_revs = ScmModel().get_repo_landing_revs(c.repo_info)
c.landing_revs_choices = choices
c.default_user_id = User.get_by_username('default').user_id
c.in_public_journal = UserFollowing.query()\
.filter(UserFollowing.user_id == c.default_user_id)\
.filter(UserFollowing.follows_repository == c.repo_info).scalar()
if c.repo_info.stats:
# this is on what revision we ended up so we add +1 for count
last_rev = c.repo_info.stats.stat_on_revision + 1
else:
last_rev = 0
c.stats_revision = last_rev
c.repo_last_rev = repo.count() if repo.revisions else 0
if last_rev == 0 or c.repo_last_rev == 0:
c.stats_percentage = 0
else:
c.stats_percentage = '%.2f' % ((float(
(last_rev)) / c.repo_last_rev) * 100)
c.repo_fields = RepositoryField.query()\
.filter(RepositoryField.repository == db_repo).all()
defaults = RepoModel()._get_defaults(repo_name)
c.repos_list = [('', _('--REMOVE FORK--'))]
c.repos_list += [
(x.repo_id, x.repo_name)
for x in Repository.query().order_by(Repository.repo_name).all()
if x.repo_id != c.repo_info.repo_id
]
defaults['id_fork_of'] = db_repo.fork.repo_id if db_repo.fork else ''
return defaults
class WhooshIndexingDaemon(object):
"""
Daemon for atomic jobs
"""
def __init__(self, indexname=IDX_NAME, index_location=None,
repo_location=None, sa=None, repo_list=None):
self.indexname = indexname
self.index_location = index_location
if not index_location:
raise Exception('You have to provide index location')
self.repo_location = repo_location
if not repo_location:
raise Exception('You have to provide repositories location')
self.repo_paths = ScmModel(sa).repo_scan(self.repo_location)
if repo_list:
filtered_repo_paths = {}
for repo_name, repo in self.repo_paths.items():
if repo_name in repo_list:
filtered_repo_paths[repo_name] = repo
self.repo_paths = filtered_repo_paths
self.initial = False
if not os.path.isdir(self.index_location):
os.makedirs(self.index_location)
log.info('Cannot run incremental index since it does not'
' yet exist running full build')
self.initial = True
def get_paths(self, repo):
"""
recursive walk in root dir and return a set of all path in that dir
based on repository walk function
"""
index_paths_ = set()
try:
tip = repo.get_changeset('tip')
for topnode, dirs, files in tip.walk('/'):
for f in files:
index_paths_.add(jn(repo.path, f.path))
except RepositoryError, e:
log.debug(traceback.format_exc())
pass
return index_paths_
def __init__(self, indexname=IDX_NAME, index_location=None,
repo_location=None, sa=None, repo_list=None,
repo_update_list=None):
self.indexname = indexname
self.index_location = index_location
if not index_location:
raise Exception('You have to provide index location')
self.repo_location = repo_location
if not repo_location:
raise Exception('You have to provide repositories location')
self.repo_paths = ScmModel(sa).repo_scan(self.repo_location)
#filter repo list
if repo_list:
#Fix non-ascii repo names to unicode
repo_list = map(safe_unicode, repo_list)
self.filtered_repo_paths = {}
for repo_name, repo in self.repo_paths.items():
if repo_name in repo_list:
self.filtered_repo_paths[repo_name] = repo
self.repo_paths = self.filtered_repo_paths
#filter update repo list
self.filtered_repo_update_paths = {}
if repo_update_list:
self.filtered_repo_update_paths = {}
for repo_name, repo in self.repo_paths.items():
if repo_name in repo_update_list:
self.filtered_repo_update_paths[repo_name] = repo
self.repo_paths = self.filtered_repo_update_paths
self.initial = True
if not os.path.isdir(self.index_location):
os.makedirs(self.index_location)
log.info('Cannot run incremental index since it does not '
'yet exist running full build')
elif not exists_in(self.index_location, IDX_NAME):
log.info('Running full index build as the file content '
'index does not exist')
elif not exists_in(self.index_location, CHGSET_IDX_NAME):
log.info('Running full index build as the changeset '
'index does not exist')
else:
self.initial = False
class WhooshIndexingDaemon(object):
"""
Daemon for atomic indexing jobs
"""
def __init__(self, indexname=IDX_NAME, index_location=None,
repo_location=None, sa=None, repo_list=None,
repo_update_list=None):
self.indexname = indexname
self.index_location = index_location
if not index_location:
raise Exception('You have to provide index location')
self.repo_location = repo_location
if not repo_location:
raise Exception('You have to provide repositories location')
self.repo_paths = ScmModel(sa).repo_scan(self.repo_location)
#filter repo list
if repo_list:
#Fix non-ascii repo names to unicode
repo_list = map(safe_unicode, repo_list)
self.filtered_repo_paths = {}
for repo_name, repo in self.repo_paths.items():
if repo_name in repo_list:
self.filtered_repo_paths[repo_name] = repo
self.repo_paths = self.filtered_repo_paths
#filter update repo list
self.filtered_repo_update_paths = {}
if repo_update_list:
self.filtered_repo_update_paths = {}
for repo_name, repo in self.repo_paths.items():
if repo_name in repo_update_list:
self.filtered_repo_update_paths[repo_name] = repo
self.repo_paths = self.filtered_repo_update_paths
self.initial = True
if not os.path.isdir(self.index_location):
os.makedirs(self.index_location)
log.info('Cannot run incremental index since it does not '
'yet exist running full build')
elif not exists_in(self.index_location, IDX_NAME):
log.info('Running full index build as the file content '
'index does not exist')
elif not exists_in(self.index_location, CHGSET_IDX_NAME):
log.info('Running full index build as the changeset '
'index does not exist')
else:
self.initial = False
def _get_index_revision(self, repo):
db_repo = Repository.get_by_repo_name(repo.name)
landing_rev = 'tip'
if db_repo:
landing_rev = db_repo.landing_rev
return landing_rev
def _get_index_changeset(self, repo):
index_rev = self._get_index_revision(repo)
cs = repo.get_changeset(index_rev)
return cs
def get_paths(self, repo):
"""
recursive walk in root dir and return a set of all path in that dir
based on repository walk function
"""
index_paths_ = set()
try:
cs = self._get_index_changeset(repo)
for _topnode, _dirs, files in cs.walk('/'):
for f in files:
index_paths_.add(jn(safe_str(repo.path), safe_str(f.path)))
except RepositoryError:
log.debug(traceback.format_exc())
pass
return index_paths_
def get_node(self, repo, path):
"""
gets a filenode based on given full path.It operates on string for
hg git compatability.
:param repo: scm repo instance
:param path: full path including root location
:return: FileNode
"""
root_path = safe_str(repo.path)+'/'
parts = safe_str(path).partition(root_path)
cs = self._get_index_changeset(repo)
node = cs.get_node(parts[-1])
return node
def get_node_mtime(self, node):
return mktime(node.last_changeset.date.timetuple())
def add_doc(self, writer, path, repo, repo_name):
"""
Adding doc to writer this function itself fetches data from
the instance of vcs backend
"""
node = self.get_node(repo, path)
indexed = indexed_w_content = 0
# we just index the content of chosen files, and skip binary files
if node.extension in INDEX_EXTENSIONS and not node.is_binary:
u_content = node.content
if not isinstance(u_content, unicode):
log.warning(' >> %s Could not get this content as unicode '
'replacing with empty content' % path)
u_content = u''
else:
log.debug(' >> %s [WITH CONTENT]' % path)
indexed_w_content += 1
else:
log.debug(' >> %s' % path)
# just index file name without it's content
u_content = u''
indexed += 1
p = safe_unicode(path)
writer.add_document(
fileid=p,
owner=unicode(repo.contact),
repository=safe_unicode(repo_name),
path=p,
content=u_content,
modtime=self.get_node_mtime(node),
extension=node.extension
)
return indexed, indexed_w_content
def index_changesets(self, writer, repo_name, repo, start_rev=None):
"""
Add all changeset in the vcs repo starting at start_rev
to the index writer
:param writer: the whoosh index writer to add to
:param repo_name: name of the repository from whence the
changeset originates including the repository group
:param repo: the vcs repository instance to index changesets for,
the presumption is the repo has changesets to index
:param start_rev=None: the full sha id to start indexing from
if start_rev is None then index from the first changeset in
the repo
"""
if start_rev is None:
start_rev = repo[0].raw_id
log.debug('indexing changesets in %s starting at rev: %s' %
(repo_name, start_rev))
indexed = 0
cs_iter = repo.get_changesets(start=start_rev)
total = len(cs_iter)
for cs in cs_iter:
log.debug(' >> %s/%s' % (cs, total))
writer.add_document(
raw_id=unicode(cs.raw_id),
owner=unicode(repo.contact),
date=cs._timestamp,
repository=safe_unicode(repo_name),
author=cs.author,
message=cs.message,
last=cs.last,
added=u' '.join([safe_unicode(node.path) for node in cs.added]).lower(),
removed=u' '.join([safe_unicode(node.path) for node in cs.removed]).lower(),
changed=u' '.join([safe_unicode(node.path) for node in cs.changed]).lower(),
parents=u' '.join([cs.raw_id for cs in cs.parents]),
)
indexed += 1
log.debug('indexed %d changesets for repo %s' % (indexed, repo_name))
return indexed
def index_files(self, file_idx_writer, repo_name, repo):
"""
Index files for given repo_name
:param file_idx_writer: the whoosh index writer to add to
:param repo_name: name of the repository we're indexing
:param repo: instance of vcs repo
"""
i_cnt = iwc_cnt = 0
log.debug('building index for %s @revision:%s' % (repo.path,
self._get_index_revision(repo)))
for idx_path in self.get_paths(repo):
i, iwc = self.add_doc(file_idx_writer, idx_path, repo, repo_name)
i_cnt += i
iwc_cnt += iwc
log.debug('added %s files %s with content for repo %s' %
(i_cnt + iwc_cnt, iwc_cnt, repo.path))
return i_cnt, iwc_cnt
def update_changeset_index(self):
idx = open_dir(self.index_location, indexname=CHGSET_IDX_NAME)
with idx.searcher() as searcher:
writer = idx.writer()
writer_is_dirty = False
try:
indexed_total = 0
repo_name = None
for repo_name, repo in self.repo_paths.items():
# skip indexing if there aren't any revs in the repo
num_of_revs = len(repo)
if num_of_revs < 1:
continue
qp = QueryParser('repository', schema=CHGSETS_SCHEMA)
q = qp.parse(u"last:t AND %s" % repo_name)
results = searcher.search(q)
# default to scanning the entire repo
last_rev = 0
start_id = None
if len(results) > 0:
# assuming that there is only one result, if not this
# may require a full re-index.
start_id = results[0]['raw_id']
last_rev = repo.get_changeset(revision=start_id).revision
# there are new changesets to index or a new repo to index
if last_rev == 0 or num_of_revs > last_rev + 1:
# delete the docs in the index for the previous
# last changeset(s)
for hit in results:
q = qp.parse(u"last:t AND %s AND raw_id:%s" %
(repo_name, hit['raw_id']))
writer.delete_by_query(q)
# index from the previous last changeset + all new ones
indexed_total += self.index_changesets(writer,
repo_name, repo, start_id)
writer_is_dirty = True
log.debug('indexed %s changesets for repo %s' % (
indexed_total, repo_name)
)
finally:
if writer_is_dirty:
log.debug('>> COMMITING CHANGES TO CHANGESET INDEX<<')
writer.commit(merge=True)
log.debug('>>> FINISHED REBUILDING CHANGESET INDEX <<<')
else:
log.debug('>> NOTHING TO COMMIT TO CHANGESET INDEX<<')
def update_file_index(self):
log.debug((u'STARTING INCREMENTAL INDEXING UPDATE FOR EXTENSIONS %s '
'AND REPOS %s') % (INDEX_EXTENSIONS, self.repo_paths.keys()))
idx = open_dir(self.index_location, indexname=self.indexname)
# The set of all paths in the index
indexed_paths = set()
# The set of all paths we need to re-index
to_index = set()
writer = idx.writer()
writer_is_dirty = False
try:
with idx.reader() as reader:
# Loop over the stored fields in the index
for fields in reader.all_stored_fields():
indexed_path = fields['path']
indexed_repo_path = fields['repository']
indexed_paths.add(indexed_path)
if not indexed_repo_path in self.filtered_repo_update_paths:
continue
repo = self.repo_paths[indexed_repo_path]
try:
node = self.get_node(repo, indexed_path)
# Check if this file was changed since it was indexed
indexed_time = fields['modtime']
mtime = self.get_node_mtime(node)
if mtime > indexed_time:
# The file has changed, delete it and add it to
# the list of files to reindex
log.debug(
'adding to reindex list %s mtime: %s vs %s' % (
indexed_path, mtime, indexed_time)
)
writer.delete_by_term('fileid', indexed_path)
writer_is_dirty = True
to_index.add(indexed_path)
except (ChangesetError, NodeDoesNotExistError):
# This file was deleted since it was indexed
log.debug('removing from index %s' % indexed_path)
writer.delete_by_term('path', indexed_path)
writer_is_dirty = True
# Loop over the files in the filesystem
# Assume we have a function that gathers the filenames of the
# documents to be indexed
ri_cnt_total = 0 # indexed
riwc_cnt_total = 0 # indexed with content
for repo_name, repo in self.repo_paths.items():
# skip indexing if there aren't any revisions
if len(repo) < 1:
continue
ri_cnt = 0 # indexed
riwc_cnt = 0 # indexed with content
for path in self.get_paths(repo):
path = safe_unicode(path)
if path in to_index or path not in indexed_paths:
# This is either a file that's changed, or a new file
# that wasn't indexed before. So index it!
i, iwc = self.add_doc(writer, path, repo, repo_name)
writer_is_dirty = True
log.debug('re indexing %s' % path)
ri_cnt += i
ri_cnt_total += 1
riwc_cnt += iwc
riwc_cnt_total += iwc
log.debug('added %s files %s with content for repo %s' % (
ri_cnt + riwc_cnt, riwc_cnt, repo.path)
)
log.debug('indexed %s files in total and %s with content' % (
ri_cnt_total, riwc_cnt_total)
)
finally:
if writer_is_dirty:
log.debug('>> COMMITING CHANGES TO FILE INDEX <<')
writer.commit(merge=True)
log.debug('>>> FINISHED REBUILDING FILE INDEX <<<')
else:
log.debug('>> NOTHING TO COMMIT TO FILE INDEX <<')
writer.cancel()
def build_indexes(self):
if os.path.exists(self.index_location):
log.debug('removing previous index')
rmtree(self.index_location)
if not os.path.exists(self.index_location):
os.mkdir(self.index_location)
chgset_idx = create_in(self.index_location, CHGSETS_SCHEMA,
indexname=CHGSET_IDX_NAME)
chgset_idx_writer = chgset_idx.writer()
file_idx = create_in(self.index_location, SCHEMA, indexname=IDX_NAME)
file_idx_writer = file_idx.writer()
log.debug('BUILDING INDEX FOR EXTENSIONS %s '
'AND REPOS %s' % (INDEX_EXTENSIONS, self.repo_paths.keys()))
for repo_name, repo in self.repo_paths.items():
# skip indexing if there aren't any revisions
if len(repo) < 1:
continue
self.index_files(file_idx_writer, repo_name, repo)
self.index_changesets(chgset_idx_writer, repo_name, repo)
log.debug('>> COMMITING CHANGES <<')
file_idx_writer.commit(merge=True)
chgset_idx_writer.commit(merge=True)
log.debug('>>> FINISHED BUILDING INDEX <<<')
def update_indexes(self):
self.update_file_index()
self.update_changeset_index()
def run(self, full_index=False):
"""Run daemon"""
if full_index or self.initial:
self.build_indexes()
else:
self.update_indexes()
def update(self, setting_id):
"""PUT /admin/settings/setting_id: Update an existing item"""
# Forms posted to this method should contain a hidden field:
# <input type="hidden" name="_method" value="PUT" />
# Or using helpers:
# h.form(url('admin_setting', setting_id=ID),
# method='put')
# url('admin_setting', setting_id=ID)
if setting_id == 'mapping':
rm_obsolete = request.POST.get('destroy', False)
log.debug('Rescanning directories with destroy=%s' % rm_obsolete)
initial = ScmModel().repo_scan()
log.debug('invalidating all repositories')
for repo_name in initial.keys():
invalidate_cache('get_repo_cached_%s' % repo_name)
added, removed = repo2db_mapper(initial, rm_obsolete)
h.flash(_('Repositories successfully'
' rescanned added: %s,removed: %s') % (added, removed),
category='success')
if setting_id == 'whoosh':
repo_location = self.get_hg_ui_settings()['paths_root_path']
full_index = request.POST.get('full_index', False)
run_task(tasks.whoosh_index, repo_location, full_index)
h.flash(_('Whoosh reindex task scheduled'), category='success')
if setting_id == 'global':
application_form = ApplicationSettingsForm()()
try:
form_result = application_form.to_python(dict(request.POST))
try:
hgsettings1 = RhodeCodeSetting.get_by_name('title')
hgsettings1.app_settings_value = \
form_result['rhodecode_title']
hgsettings2 = RhodeCodeSetting.get_by_name('realm')
hgsettings2.app_settings_value = \
form_result['rhodecode_realm']
hgsettings3 = RhodeCodeSetting.get_by_name('ga_code')
hgsettings3.app_settings_value = \
form_result['rhodecode_ga_code']
self.sa.add(hgsettings1)
self.sa.add(hgsettings2)
self.sa.add(hgsettings3)
self.sa.commit()
set_rhodecode_config(config)
h.flash(_('Updated application settings'),
category='success')
except Exception:
log.error(traceback.format_exc())
h.flash(_('error occurred during updating '
'application settings'),
category='error')
self.sa.rollback()
except formencode.Invalid, errors:
return htmlfill.render(
render('admin/settings/settings.html'),
defaults=errors.value,
errors=errors.error_dict or {},
prefix_error=False,
encoding="UTF-8")
