def commit_change(self, repo, repo_name, cs, user, author, message, content,
f_path):
if repo.alias == 'hg':
from vcs.backends.hg import MercurialInMemoryChangeset as IMC
elif repo.alias == 'git':
from vcs.backends.git import GitInMemoryChangeset as IMC
# decoding here will force that we have proper encoded values
# in any other case this will throw exceptions and deny commit
content = safe_str(content)
message = safe_str(message)
path = safe_str(f_path)
author = safe_str(author)
m = IMC(repo)
m.change(FileNode(path, content))
tip = m.commit(message=message,
author=author,
parents=[cs], branch=cs.branch)
new_cs = tip.short_id
action = 'push_local:%s' % new_cs
action_logger(user, action, repo_name)
self.mark_for_invalidation(repo_name)
def create_node(self, repo, repo_name, cs, user, author, message, content,
f_path):
if repo.alias == 'hg':
from vcs.backends.hg import MercurialInMemoryChangeset as IMC
elif repo.alias == 'git':
from vcs.backends.git import GitInMemoryChangeset as IMC
# decoding here will force that we have proper encoded values
# in any other case this will throw exceptions and deny commit
if isinstance(content, (basestring,)):
content = safe_str(content)
elif isinstance(content, file):
content = content.read()
message = safe_str(message)
path = safe_str(f_path)
author = safe_str(author)
m = IMC(repo)
if isinstance(cs, EmptyChangeset):
# Emptychangeset means we we're editing empty repository
parents = None
else:
parents = [cs]
m.add(FileNode(path, content=content))
tip = m.commit(message=message,
author=author,
parents=parents, branch=cs.branch)
new_cs = tip.short_id
action = 'push_local:%s' % new_cs
action_logger(user, action, repo_name)
self.mark_for_invalidation(repo_name)
def __get_instance(self):
repo_full_path = self.repo_full_path
try:
alias = get_scm(repo_full_path)[0]
log.debug('Creating instance of %s repository', alias)
backend = get_backend(alias)
except VCSError:
log.error(traceback.format_exc())
log.error('Perhaps this repository is in db and not in '
'filesystem run rescan repositories with '
'"destroy old data " option from admin panel')
return
if alias == 'hg':
repo = backend(safe_str(repo_full_path), create=False,
baseui=self._ui)
#skip hidden web repository
if repo._get_hidden():
return
else:
repo = backend(repo_full_path, create=False)
return repo
def __create_repo(self, repo_name, alias, new_parent_id, clone_uri=False):
"""
makes repository on filesystem. It's group aware means it'll create
a repository within a group, and alter the paths accordingly of
group location
:param repo_name:
:param alias:
:param parent_id:
:param clone_uri:
"""
from rhodecode.lib.utils import check_repo
if new_parent_id:
paths = Group.get(new_parent_id).full_path.split(Group.url_sep())
new_parent_path = os.sep.join(paths)
else:
new_parent_path = ""
repo_path = os.path.join(*map(lambda x: safe_str(x), [self.repos_path, new_parent_path, repo_name]))
if check_repo(repo_path, self.repos_path):
log.info("creating repo %s in %s @ %s", repo_name, repo_path, clone_uri)
backend = get_backend(alias)
backend(repo_path, create=True, src_url=clone_uri)
def rawfile(self, repo_name, revision, f_path):
cs = self.__get_cs_or_redirect(revision, repo_name)
file_node = self.__get_filenode_or_redirect(repo_name, cs, f_path)
response.content_disposition = 'attachment; filename=%s' % \
safe_str(f_path.split(os.sep)[-1])
response.content_type = file_node.mimetype
return file_node.content
def __get_codes_stats(repo_name):
repos_path = get_repos_path()
repo = get_repo(safe_str(os.path.join(repos_path, repo_name)))
tip = repo.get_changeset()
code_stats = {}
def aggregate(cs):
for f in cs[2]:
ext = lower(f.extension)
if ext in LANGUAGES_EXTENSIONS_MAP.keys() and not f.is_binary:
if ext in code_stats:
code_stats[ext] += 1
else:
code_stats[ext] = 1
map(aggregate, tip.walk('/'))
return code_stats or {}
def scm_instance_cached(self):
@cache_region('long_term')
def _c(repo_name):
return self.__get_instance()
# TODO: remove this trick when beaker 1.6 is released
# and have fixed this issue with not supporting unicode keys
rn = safe_str(self.repo_name)
inv = self.invalidate
if inv is not None:
region_invalidate(_c, None, rn)
# update our cache
inv.cache_active = True
Session.add(inv)
Session.commit()
return _c(rn)
def raw(self, repo_name, revision, f_path):
cs = self.__get_cs_or_redirect(revision, repo_name)
file_node = self.__get_filenode_or_redirect(repo_name, cs, f_path)
raw_mimetype_mapping = {
# map original mimetype to a mimetype used for "show as raw"
# you can also provide a content-disposition to override the
# default "attachment" disposition.
# orig_type: (new_type, new_dispo)
# show images inline:
'image/x-icon': ('image/x-icon', 'inline'),
'image/png': ('image/png', 'inline'),
'image/gif': ('image/gif', 'inline'),
'image/jpeg': ('image/jpeg', 'inline'),
'image/svg+xml': ('image/svg+xml', 'inline'),
}
mimetype = file_node.mimetype
try:
mimetype, dispo = raw_mimetype_mapping[mimetype]
except KeyError:
# we don't know anything special about this, handle it safely
if file_node.is_binary:
# do same as download raw for binary files
mimetype, dispo = 'application/octet-stream', 'attachment'
else:
# do not just use the original mimetype, but force text/plain,
# otherwise it would serve text/html and that might be unsafe.
# Note: underlying vcs library fakes text/plain mimetype if the
# mimetype can not be determined and it thinks it is not
# binary.This might lead to erroneous text display in some
# cases, but helps in other cases, like with text files
# without extension.
mimetype, dispo = 'text/plain', 'inline'
if dispo == 'attachment':
dispo = 'attachment; filename=%s' % \
safe_str(f_path.split(os.sep)[-1])
response.content_disposition = dispo
response.content_type = mimetype
return file_node.content
def gravatar_url(email_address, size=30):
if (not str2bool(config['app_conf'].get('use_gravatar')) or
not email_address or email_address == '*****@*****.**'):
return url("/images/user%s.png" % size)
ssl_enabled = 'https' == request.environ.get('wsgi.url_scheme')
default = 'identicon'
baseurl_nossl = "http://www.gravatar.com/avatar/"
baseurl_ssl = "https://secure.gravatar.com/avatar/"
baseurl = baseurl_ssl if ssl_enabled else baseurl_nossl
if isinstance(email_address, unicode):
#hashlib crashes on unicode items
email_address = safe_str(email_address)
# construct the url
gravatar_url = baseurl + hashlib.md5(email_address.lower()).hexdigest() + "?"
gravatar_url += urllib.urlencode({'d':default, 's':str(size)})
return gravatar_url
def repo_scan(self, repos_path=None):
"""Listing of repositories in given path. This path should not be a
repository itself. Return a dictionary of repository objects
:param repos_path: path to directory containing repositories
"""
if repos_path is None:
repos_path = self.repos_path
log.info('scanning for repositories in %s', repos_path)
baseui = make_ui('db')
repos_list = {}
for name, path in get_filesystem_repos(repos_path, recursive=True):
# name need to be decomposed and put back together using the /
# since this is internal storage separator for rhodecode
name = Repository.url_sep().join(name.split(os.sep))
try:
if name in repos_list:
raise RepositoryError('Duplicate repository name %s '
'found in %s' % (name, path))
else:
klass = get_backend(path[0])
if path[0] == 'hg' and path[0] in BACKENDS.keys():
# for mercurial we need to have an str path
repos_list[name] = klass(safe_str(path[1]),
baseui=baseui)
if path[0] == 'git' and path[0] in BACKENDS.keys():
repos_list[name] = klass(path[1])
except OSError:
continue
return repos_list
def repo_scan(self, repos_path=None):
"""Listing of repositories in given path. This path should not be a
repository itself. Return a dictionary of repository objects
:param repos_path: path to directory containing repositories
"""
log.info('scanning for repositories in %s', repos_path)
if repos_path is None:
repos_path = self.repos_path
baseui = make_ui('db')
repos_list = {}
for name, path in get_filesystem_repos(repos_path, recursive=True):
try:
if name in repos_list:
raise RepositoryError('Duplicate repository name %s '
'found in %s' % (name, path))
else:
klass = get_backend(path[0])
if path[0] == 'hg' and path[0] in BACKENDS.keys():
# for mercurial we need to have an str path
repos_list[name] = klass(safe_str(path[1]),
baseui=baseui)
if path[0] == 'git' and path[0] in BACKENDS.keys():
repos_list[name] = klass(path[1])
except OSError:
continue
return repos_list
def __create_repo(self, repo_name, alias, new_parent_id, clone_uri=False):
"""
makes repository on filesystem. It's group aware means it'll create
a repository within a group, and alter the paths accordingly of
group location
:param repo_name:
:param alias:
:param parent_id:
:param clone_uri:
"""
from rhodecode.lib.utils import is_valid_repo,is_valid_repos_group
if new_parent_id:
paths = Group.get(new_parent_id).full_path.split(Group.url_sep())
new_parent_path = os.sep.join(paths)
else:
new_parent_path = ''
repo_path = os.path.join(*map(lambda x:safe_str(x),
[self.repos_path, new_parent_path, repo_name]))
# check if this path is not a repository
if is_valid_repo(repo_path, self.repos_path):
raise Exception('This path %s is a valid repository' % repo_path)
# check if this path is a group
if is_valid_repos_group(repo_path, self.repos_path):
raise Exception('This path %s is a valid group' % repo_path)
log.info('creating repo %s in %s @ %s', repo_name, repo_path,
clone_uri)
backend = get_backend(alias)
backend(repo_path, create=True, src_url=clone_uri)
def __call__(self, environ, start_response):
if not is_git(environ):
return self.application(environ, start_response)
proxy_key = 'HTTP_X_REAL_IP'
def_key = 'REMOTE_ADDR'
ipaddr = environ.get(proxy_key, environ.get(def_key, '0.0.0.0'))
username = None
# skip passing error to error controller
environ['pylons.status_code_redirect'] = True
#======================================================================
# EXTRACT REPOSITORY NAME FROM ENV
#======================================================================
try:
repo_name = self.__get_repository(environ)
log.debug('Extracted repo name is %s' % repo_name)
except:
return HTTPInternalServerError()(environ, start_response)
#======================================================================
# GET ACTION PULL or PUSH
#======================================================================
action = self.__get_action(environ)
#======================================================================
# CHECK ANONYMOUS PERMISSION
#======================================================================
if action in ['pull', 'push']:
anonymous_user = self.__get_user('default')
username = anonymous_user.username
anonymous_perm = self.__check_permission(action,
anonymous_user,
repo_name)
if anonymous_perm is not True or anonymous_user.active is False:
if anonymous_perm is not True:
log.debug('Not enough credentials to access this '
'repository as anonymous user')
if anonymous_user.active is False:
log.debug('Anonymous access is disabled, running '
'authentication')
#==============================================================
# DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE
# NEED TO AUTHENTICATE AND ASK FOR AUTH USER PERMISSIONS
#==============================================================
if not REMOTE_USER(environ):
self.authenticate.realm = \
safe_str(self.config['rhodecode_realm'])
result = self.authenticate(environ)
if isinstance(result, str):
AUTH_TYPE.update(environ, 'basic')
REMOTE_USER.update(environ, result)
else:
return result.wsgi_application(environ, start_response)
#==============================================================
# CHECK PERMISSIONS FOR THIS REQUEST USING GIVEN USERNAME FROM
# BASIC AUTH
#==============================================================
if action in ['pull', 'push']:
username = REMOTE_USER(environ)
try:
user = self.__get_user(username)
username = user.username
except:
log.error(traceback.format_exc())
return HTTPInternalServerError()(environ,
start_response)
#check permissions for this repository
perm = self.__check_permission(action, user,
repo_name)
if perm is not True:
return HTTPForbidden()(environ, start_response)
extras = {'ip': ipaddr,
'username': username,
'action': action,
'repository': repo_name}
#===================================================================
# GIT REQUEST HANDLING
#===================================================================
repo_path = safe_str(os.path.join(self.basepath, repo_name))
log.debug('Repository path is %s' % repo_path)
# quick check if that dir exists...
if is_valid_repo(repo_name, self.basepath) is False:
return HTTPNotFound()(environ, start_response)
try:
#invalidate cache on push
if action == 'push':
self.__invalidate_cache(repo_name)
app = self.__make_app(repo_name, repo_path)
return app(environ, start_response)
except Exception:
log.error(traceback.format_exc())
return HTTPInternalServerError()(environ, start_response)
def get_commits_stats(repo_name, ts_min_y, ts_max_y):
try:
log = get_commits_stats.get_logger()
except:
log = logging.getLogger(__name__)
lockkey = __get_lockkey('get_commits_stats', repo_name, ts_min_y,
ts_max_y)
lockkey_path = config['here']
log.info('running task with lockkey %s', lockkey)
try:
lock = l = DaemonLock(file_=jn(lockkey_path, lockkey))
#for js data compatibilty cleans the key for person from '
akc = lambda k: person(k).replace('"', "")
co_day_auth_aggr = {}
commits_by_day_aggregate = {}
repos_path = get_repos_path()
repo = get_repo(safe_str(os.path.join(repos_path, repo_name)))
repo_size = len(repo.revisions)
#return if repo have no revisions
if repo_size < 1:
lock.release()
return True
skip_date_limit = True
parse_limit = int(config['app_conf'].get('commit_parse_limit'))
last_rev = 0
last_cs = None
timegetter = itemgetter('time')
sa = get_session()
dbrepo = sa.query(Repository)\
.filter(Repository.repo_name == repo_name).scalar()
cur_stats = sa.query(Statistics)\
.filter(Statistics.repository == dbrepo).scalar()
if cur_stats is not None:
last_rev = cur_stats.stat_on_revision
if last_rev == repo.get_changeset().revision and repo_size > 1:
#pass silently without any work if we're not on first revision or
#current state of parsing revision(from db marker) is the
#last revision
lock.release()
return True
if cur_stats:
commits_by_day_aggregate = OrderedDict(json.loads(
cur_stats.commit_activity_combined))
co_day_auth_aggr = json.loads(cur_stats.commit_activity)
log.debug('starting parsing %s', parse_limit)
lmktime = mktime
last_rev = last_rev + 1 if last_rev > 0 else last_rev
for cs in repo[last_rev:last_rev + parse_limit]:
last_cs = cs # remember last parsed changeset
k = lmktime([cs.date.timetuple()[0], cs.date.timetuple()[1],
cs.date.timetuple()[2], 0, 0, 0, 0, 0, 0])
if akc(cs.author) in co_day_auth_aggr:
try:
l = [timegetter(x) for x in
co_day_auth_aggr[akc(cs.author)]['data']]
time_pos = l.index(k)
except ValueError:
time_pos = False
if time_pos >= 0 and time_pos is not False:
datadict = \
co_day_auth_aggr[akc(cs.author)]['data'][time_pos]
datadict["commits"] += 1
datadict["added"] += len(cs.added)
datadict["changed"] += len(cs.changed)
datadict["removed"] += len(cs.removed)
else:
if k >= ts_min_y and k <= ts_max_y or skip_date_limit:
datadict = {"time": k,
"commits": 1,
"added": len(cs.added),
"changed": len(cs.changed),
"removed": len(cs.removed),
}
co_day_auth_aggr[akc(cs.author)]['data']\
.append(datadict)
else:
if k >= ts_min_y and k <= ts_max_y or skip_date_limit:
co_day_auth_aggr[akc(cs.author)] = {
"label": akc(cs.author),
"data": [{"time":k,
"commits":1,
"added":len(cs.added),
"changed":len(cs.changed),
"removed":len(cs.removed),
}],
"schema": ["commits"],
}
#gather all data by day
if k in commits_by_day_aggregate:
commits_by_day_aggregate[k] += 1
else:
commits_by_day_aggregate[k] = 1
overview_data = sorted(commits_by_day_aggregate.items(),
key=itemgetter(0))
if not co_day_auth_aggr:
co_day_auth_aggr[akc(repo.contact)] = {
"label": akc(repo.contact),
"data": [0, 1],
"schema": ["commits"],
}
stats = cur_stats if cur_stats else Statistics()
stats.commit_activity = json.dumps(co_day_auth_aggr)
stats.commit_activity_combined = json.dumps(overview_data)
log.debug('last revison %s', last_rev)
leftovers = len(repo.revisions[last_rev:])
log.debug('revisions to parse %s', leftovers)
if last_rev == 0 or leftovers < parse_limit:
log.debug('getting code trending stats')
stats.languages = json.dumps(__get_codes_stats(repo_name))
try:
stats.repository = dbrepo
stats.stat_on_revision = last_cs.revision if last_cs else 0
sa.add(stats)
sa.commit()
except:
log.error(traceback.format_exc())
sa.rollback()
lock.release()
return False
#final release
lock.release()
#execute another task if celery is enabled
if len(repo.revisions) > 1 and CELERY_ON:
run_task(get_commits_stats, repo_name, ts_min_y, ts_max_y)
return True
except LockHeld:
log.info('LockHeld')
return 'Task with key %s already running' % lockkey
def __call__(self, environ, start_response):
if not is_mercurial(environ):
return self.application(environ, start_response)
proxy_key = "HTTP_X_REAL_IP"
def_key = "REMOTE_ADDR"
ipaddr = environ.get(proxy_key, environ.get(def_key, "0.0.0.0"))
# skip passing error to error controller
environ["pylons.status_code_redirect"] = True
# ======================================================================
# EXTRACT REPOSITORY NAME FROM ENV
# ======================================================================
try:
repo_name = environ["REPO_NAME"] = self.__get_repository(environ)
log.debug("Extracted repo name is %s" % repo_name)
except:
return HTTPInternalServerError()(environ, start_response)
# ======================================================================
# GET ACTION PULL or PUSH
# ======================================================================
action = self.__get_action(environ)
# ======================================================================
# CHECK ANONYMOUS PERMISSION
# ======================================================================
if action in ["pull", "push"]:
anonymous_user = self.__get_user("default")
username = anonymous_user.username
anonymous_perm = self.__check_permission(action, anonymous_user, repo_name)
if anonymous_perm is not True or anonymous_user.active is False:
if anonymous_perm is not True:
log.debug("Not enough credentials to access this " "repository as anonymous user")
if anonymous_user.active is False:
log.debug("Anonymous access is disabled, running " "authentication")
# ==============================================================
# DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE
# NEED TO AUTHENTICATE AND ASK FOR AUTH USER PERMISSIONS
# ==============================================================
if not REMOTE_USER(environ):
self.authenticate.realm = safe_str(self.config["rhodecode_realm"])
result = self.authenticate(environ)
if isinstance(result, str):
AUTH_TYPE.update(environ, "basic")
REMOTE_USER.update(environ, result)
else:
return result.wsgi_application(environ, start_response)
# ==============================================================
# CHECK PERMISSIONS FOR THIS REQUEST USING GIVEN USERNAME FROM
# BASIC AUTH
# ==============================================================
if action in ["pull", "push"]:
username = REMOTE_USER(environ)
try:
user = self.__get_user(username)
username = user.username
except:
log.error(traceback.format_exc())
return HTTPInternalServerError()(environ, start_response)
# check permissions for this repository
perm = self.__check_permission(action, user, repo_name)
if perm is not True:
return HTTPForbidden()(environ, start_response)
extras = {"ip": ipaddr, "username": username, "action": action, "repository": repo_name}
# ======================================================================
# MERCURIAL REQUEST HANDLING
# ======================================================================
repo_path = safe_str(os.path.join(self.basepath, repo_name))
log.debug("Repository path is %s" % repo_path)
baseui = make_ui("db")
self.__inject_extras(repo_path, baseui, extras)
# quick check if that dir exists...
if is_valid_repo(repo_name, self.basepath) is False:
return HTTPNotFound()(environ, start_response)
try:
# invalidate cache on push
if action == "push":
self.__invalidate_cache(repo_name)
app = self.__make_app(repo_path, baseui, extras)
return app(environ, start_response)
except RepoError, e:
if str(e).find("not found") != -1:
return HTTPNotFound()(environ, start_response)
def __call__(self, environ, start_response):
if not is_mercurial(environ):
return self.application(environ, start_response)
proxy_key = 'HTTP_X_REAL_IP'
def_key = 'REMOTE_ADDR'
self.ipaddr = environ.get(proxy_key, environ.get(def_key, '0.0.0.0'))
# skip passing error to error controller
environ['pylons.status_code_redirect'] = True
#======================================================================
# GET ACTION PULL or PUSH
#======================================================================
self.action = self.__get_action(environ)
try:
#==================================================================
# GET REPOSITORY NAME
#==================================================================
self.repo_name = self.__get_repository(environ)
except:
return HTTPInternalServerError()(environ, start_response)
#======================================================================
# CHECK ANONYMOUS PERMISSION
#======================================================================
if self.action in ['pull', 'push']:
anonymous_user = self.__get_user('default')
self.username = anonymous_user.username
anonymous_perm = self.__check_permission(self.action,
anonymous_user,
self.repo_name)
if anonymous_perm is not True or anonymous_user.active is False:
if anonymous_perm is not True:
log.debug('Not enough credentials to access this '
'repository as anonymous user')
if anonymous_user.active is False:
log.debug('Anonymous access is disabled, running '
'authentication')
#==============================================================
# DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE
# NEED TO AUTHENTICATE AND ASK FOR AUTH USER PERMISSIONS
#==============================================================
if not REMOTE_USER(environ):
self.authenticate.realm = \
safe_str(self.config['rhodecode_realm'])
result = self.authenticate(environ)
if isinstance(result, str):
AUTH_TYPE.update(environ, 'basic')
REMOTE_USER.update(environ, result)
else:
return result.wsgi_application(environ, start_response)
#==============================================================
# CHECK PERMISSIONS FOR THIS REQUEST USING GIVEN USERNAME FROM
# BASIC AUTH
#==============================================================
if self.action in ['pull', 'push']:
username = REMOTE_USER(environ)
try:
user = self.__get_user(username)
self.username = user.username
except:
log.error(traceback.format_exc())
return HTTPInternalServerError()(environ,
start_response)
#check permissions for this repository
perm = self.__check_permission(self.action, user,
self.repo_name)
if perm is not True:
return HTTPForbidden()(environ, start_response)
self.extras = {'ip': self.ipaddr,
'username': self.username,
'action': self.action,
'repository': self.repo_name}
#======================================================================
# MERCURIAL REQUEST HANDLING
#======================================================================
environ['PATH_INFO'] = '/' # since we wrap into hgweb, reset the path
self.baseui = make_ui('db')
self.basepath = self.config['base_path']
self.repo_path = os.path.join(self.basepath, self.repo_name)
#quick check if that dir exists...
if check_repo_fast(self.repo_name, self.basepath):
return HTTPNotFound()(environ, start_response)
try:
app = wsgiapplication(self.__make_app)
except RepoError, e:
if str(e).find('not found') != -1:
return HTTPNotFound()(environ, start_response)