def after_set_privacy(self, node, permissions):
"""
:param Node node:
:param str permissions:
:return str: Alert message
"""
if not github_settings.SET_PRIVACY:
return
connect = GitHub.from_settings(self.user_settings)
data = connect.set_privacy(self.user, self.repo, permissions == "private")
if data is None or "errors" in data:
repo = connect.repo(self.user, self.repo)
if repo is not None:
current_privacy = "private" if repo.private else "public"
else:
current_privacy = "unknown"
return "Could not set privacy for repo {user}::{repo}. " "Current privacy status is {perm}.".format(
user=self.user, repo=self.repo, perm=current_privacy
)
return "GitHub repo {user}::{repo} made {perm}.".format(user=self.user, repo=self.repo, perm=permissions)
def add_hook(self, save=True):
if self.user_settings:
connect = GitHub.from_settings(self.user_settings)
secret = utils.make_hook_secret()
hook = connect.add_hook(
self.user, self.repo,
'web',
{
'url': urlparse.urljoin(
hook_domain,
os.path.join(
self.owner.api_url, 'github', 'hook/'
)
),
'content_type': github_settings.HOOK_CONTENT_TYPE,
'secret': secret,
}
)
if hook:
self.hook_id = hook.id
self.hook_secret = secret
if save:
self.save()
def after_register(self, node, registration, user, save=True):
"""
:param Node node: Original node
:param Node registration: Registered node
:param User user: User creating registration
:param bool save: Save settings after callback
:return tuple: Tuple of cloned settings and alert message
"""
clone, message = super(AddonGitHubNodeSettings,
self).after_register(node,
registration,
user,
save=False)
# Copy foreign fields from current add-on
clone.user_settings = self.user_settings
# Store current branch data
if self.user and self.repo:
connect = GitHub.from_settings(self.user_settings)
try:
branches = [
branch.to_json()
for branch in connect.branches(self.user, self.repo)
]
clone.registration_data['branches'] = branches
except ApiError:
pass
if save:
clone.save()
return clone, message
def update_hook(node_settings):
"""Discard the existing webhook for a GitHub node add-on and create a new
one.
"""
logger.warn(
'Updating GitHub hook on node {0}'.format(
node_settings.owner._id
)
)
connection = GitHub.from_settings(node_settings.user_settings)
repo = connection.repo(node_settings.user, node_settings.repo)
hook = repo.hook(node_settings.hook_id)
if hook is None:
logger.warn('Hook {0} not found'.format(node_settings.hook_id))
return
secret = utils.make_hook_secret()
config = hook.config
config['content_type'] = github_settings.HOOK_CONTENT_TYPE
config['secret'] = secret
hook.edit(config=config)
node_settings.hook_secret = secret
node_settings.save()
def update_hook(node_settings):
"""Discard the existing webhook for a GitHub node add-on and create a new
one.
"""
logger.warn('Updating GitHub hook on node {0}'.format(
node_settings.owner._id))
connection = GitHub.from_settings(node_settings.user_settings)
repo = connection.repo(node_settings.user, node_settings.repo)
hook = repo.hook(node_settings.hook_id)
if hook is None:
logger.warn('Hook {0} not found'.format(node_settings.hook_id))
return
secret = utils.make_hook_secret()
config = hook.config
config['content_type'] = github_settings.HOOK_CONTENT_TYPE
config['secret'] = secret
hook.edit(config=config)
node_settings.hook_secret = secret
node_settings.save()
def add_hook(self, save=True):
if self.user_settings:
connect = GitHub.from_settings(self.user_settings)
secret = utils.make_hook_secret()
hook = connect.add_hook(
self.user, self.repo,
'web',
{
'url': urlparse.urljoin(
hook_domain,
os.path.join(
self.owner.api_url, 'github', 'hook/'
)
),
'content_type': github_settings.HOOK_CONTENT_TYPE,
'secret': secret,
}
)
if hook:
self.hook_id = hook.id
self.hook_secret = secret
if save:
self.save()
def after_set_privacy(self, node, permissions):
"""
:param Node node:
:param str permissions:
:return str: Alert message
"""
if not github_settings.SET_PRIVACY:
return
connect = GitHub.from_settings(self.user_settings)
data = connect.set_privacy(self.user, self.repo,
permissions == 'private')
if data is None or 'errors' in data:
repo = connect.repo(self.user, self.repo)
if repo is not None:
current_privacy = 'private' if repo.private else 'public'
else:
current_privacy = 'unknown'
return ('Could not set privacy for repo {user}::{repo}. '
'Current privacy status is {perm}.'.format(
user=self.user,
repo=self.repo,
perm=current_privacy,
))
return ('GitHub repo {user}::{repo} made {perm}.'.format(
user=self.user,
repo=self.repo,
perm=permissions,
))
def tests_none_trees_raise_empty_repo(self, mock_repo):
mock_tree = mock.Mock()
mock_tree.tree.return_value = None
mock_repo.return_value = mock_tree
with assert_raises(EmptyRepoError):
GitHub().tree('', '', '')
def after_register(self, node, registration, user, save=True):
"""
:param Node node: Original node
:param Node registration: Registered node
:param User user: User creating registration
:param bool save: Save settings after callback
:return tuple: Tuple of cloned settings and alert message
"""
clone, message = super(AddonGitHubNodeSettings, self).after_register(
node, registration, user, save=False
)
# Copy foreign fields from current add-on
clone.user_settings = self.user_settings
# Store current branch data
if self.user and self.repo:
connect = GitHub.from_settings(self.user_settings)
try:
branches = [
branch.to_json()
for branch in connect.branches(self.user, self.repo)
]
clone.registration_data['branches'] = branches
except ApiError:
pass
if save:
clone.save()
return clone, message
def to_json(self, user):
ret = super(AddonGitHubNodeSettings, self).to_json(user)
user_settings = user.get_addon('github')
ret.update({
'user_has_auth': user_settings and user_settings.has_auth,
'is_registration': self.owner.is_registration,
})
if self.user_settings and self.user_settings.has_auth:
valid_credentials = False
owner = self.user_settings.owner
if user_settings and user_settings.owner == owner:
connection = GitHub.from_settings(user_settings)
# TODO: Fetch repo list client-side
# Since /user/repos excludes organization repos to which the
# current user has push access, we have to make extra requests to
# find them
valid_credentials = True
try:
repos = itertools.chain.from_iterable(
(connection.repos(), connection.my_org_repos()))
repo_names = [
'{0} / {1}'.format(repo.owner.login, repo.name)
for repo in repos
]
except GitHubError as error:
if error.code == http.UNAUTHORIZED:
repo_names = []
valid_credentials = False
ret.update({'repo_names': repo_names})
ret.update({
'node_has_auth':
True,
'github_user':
self.user or '',
'github_repo':
self.repo or '',
'github_repo_full_name':
'{0} / {1}'.format(self.user, self.repo),
'auth_osf_name':
owner.fullname,
'auth_osf_url':
owner.url,
'auth_osf_id':
owner._id,
'github_user_name':
self.user_settings.github_user_name,
'github_user_url':
'https://github.com/{0}'.format(
self.user_settings.github_user_name),
'is_owner':
owner == user,
'valid_credentials':
valid_credentials,
'addons_url':
web_url_for('user_addons'),
})
return ret
def before_page_load(self, node, user):
"""
:param Node node:
:param User user:
:return str: Alert message
"""
messages = []
# Quit if not contributor
if not node.is_contributor(user):
return messages
# Quit if not configured
if self.user is None or self.repo is None:
return messages
# Quit if no user authorization
if self.user_settings is None:
return messages
connect = GitHub.from_settings(self.user_settings)
try:
repo = connect.repo(self.user, self.repo)
except ApiError:
return
node_permissions = 'public' if node.is_public else 'private'
repo_permissions = 'private' if repo.private else 'public'
if repo_permissions != node_permissions:
message = (
'Warnings: This OSF {category} is {node_perm}, but the GitHub '
'repo {user} / {repo} is {repo_perm}.'.format(
category=node.project_or_component,
node_perm=node_permissions,
repo_perm=repo_permissions,
user=self.user,
repo=self.repo,
)
)
if repo_permissions == 'private':
message += (
' Users can view the contents of this private GitHub '
'repository through this public project.'
)
else:
message += (
' The files in this GitHub repo can be viewed on GitHub '
'<a href="https://github.com/{user}/{repo}/">here</a>.'
).format(
user=self.user,
repo=self.repo,
)
messages.append(message)
return messages
def before_page_load(self, node, user):
"""
:param Node node:
:param User user:
:return str: Alert message
"""
messages = []
# Quit if not contributor
if not node.is_contributor(user):
return messages
# Quit if not configured
if self.user is None or self.repo is None:
return messages
# Quit if no user authorization
if self.user_settings is None:
return messages
connect = GitHub.from_settings(self.user_settings)
try:
repo = connect.repo(self.user, self.repo)
except (ApiError, GitHubError):
return
node_permissions = 'public' if node.is_public else 'private'
repo_permissions = 'private' if repo.private else 'public'
if repo_permissions != node_permissions:
message = (
'Warnings: This OSF {category} is {node_perm}, but the GitHub '
'repo {user} / {repo} is {repo_perm}.'.format(
category=node.project_or_component,
node_perm=node_permissions,
repo_perm=repo_permissions,
user=self.user,
repo=self.repo,
)
)
if repo_permissions == 'private':
message += (
' Users can view the contents of this private GitHub '
'repository through this public project.'
)
else:
message += (
' The files in this GitHub repo can be viewed on GitHub '
'<a href="https://github.com/{user}/{repo}/">here</a>.'
).format(
user=self.user,
repo=self.repo,
)
messages.append(message)
return messages
def revoke_token(self):
connection = GitHub.from_settings(self)
try:
connection.revoke_token()
except GitHubError as error:
if error.code == http.UNAUTHORIZED:
return (
'Your GitHub credentials were removed from the OSF, but we '
'were unable to revoke your access token from GitHub. Your '
'GitHub credentials may no longer be valid.')
raise
def revoke_token(self):
connection = GitHub.from_settings(self)
try:
connection.revoke_token()
except GitHubError as error:
if error.code == http.UNAUTHORIZED:
return (
'Your GitHub credentials were removed from the OSF, but we '
'were unable to revoke your access token from GitHub. Your '
'GitHub credentials may no longer be valid.'
)
raise
def github_delete_file(auth, node_addon, **kwargs):
node = kwargs['node'] or kwargs['project']
now = datetime.datetime.utcnow()
# Must remove trailing slash, else GitHub fails silently on delete
path = get_path(kwargs).rstrip('/')
sha = request.args.get('sha')
if sha is None:
raise HTTPError(http.BAD_REQUEST)
branch = request.args.get('branch')
author = {
'name': auth.user.fullname,
'email': '{0}@osf.io'.format(auth.user._id),
}
connection = GitHub.from_settings(node_addon.user_settings)
data = connection.delete_file(
node_addon.user,
node_addon.repo,
path,
MESSAGES['delete'],
sha=sha,
branch=branch,
author=author,
)
if data is None:
raise HTTPError(http.BAD_REQUEST)
node.add_log(
action='github_' + models.NodeLog.FILE_REMOVED,
params={
'project': node.parent_id,
'node': node._primary_key,
'path': path,
'github': {
'user': node_addon.user,
'repo': node_addon.repo,
},
},
auth=auth,
log_date=now,
)
return {}
def github_download_starball(node_addon, **kwargs):
archive = kwargs.get('archive', 'tar')
ref = request.args.get('sha', 'master')
connection = GitHub.from_settings(node_addon.user_settings)
headers, data = connection.starball(node_addon.user, node_addon.repo,
archive, ref)
resp = make_response(data)
for key, value in headers.iteritems():
resp.headers[key] = value
return resp
def github_download_starball(node_addon, **kwargs):
archive = kwargs.get('archive', 'tar')
ref = request.args.get('sha', 'master')
connection = GitHub.from_settings(node_addon.user_settings)
headers, data = connection.starball(
node_addon.user, node_addon.repo, archive, ref
)
resp = make_response(data)
for key, value in headers.iteritems():
resp.headers[key] = value
return resp
def delete_hook(self, save=True):
"""
:return bool: Hook was deleted
"""
if self.user_settings and self.hook_id:
connection = GitHub.from_settings(self.user_settings)
try:
response = connection.delete_hook(self.user, self.repo, self.hook_id)
except (GitHubError, NotFoundError):
return False
if response:
self.hook_id = None
if save:
self.save()
return True
return False
def delete_hook(self, save=True):
"""
:return bool: Hook was deleted
"""
if self.user_settings and self.hook_id:
connection = GitHub.from_settings(self.user_settings)
try:
response = connection.delete_hook(self.user, self.repo, self.hook_id)
except (GitHubError, NotFoundError):
return False
if response:
self.hook_id = None
if save:
self.save()
return True
return False
def github_delete_file(auth, node_addon, **kwargs):
node = kwargs['node'] or kwargs['project']
now = datetime.datetime.utcnow()
# Must remove trailing slash, else GitHub fails silently on delete
path = get_path(kwargs).rstrip('/')
sha = request.args.get('sha')
if sha is None:
raise HTTPError(http.BAD_REQUEST)
branch = request.args.get('branch')
author = {
'name': auth.user.fullname,
'email': '{0}@osf.io'.format(auth.user._id),
}
connection = GitHub.from_settings(node_addon.user_settings)
data = connection.delete_file(
node_addon.user, node_addon.repo, path, MESSAGES['delete'],
sha=sha, branch=branch, author=author,
)
if data is None:
raise HTTPError(http.BAD_REQUEST)
node.add_log(
action='github_' + models.NodeLog.FILE_REMOVED,
params={
'project': node.parent_id,
'node': node._primary_key,
'path': path,
'github': {
'user': node_addon.user,
'repo': node_addon.repo,
},
},
auth=auth,
log_date=now,
)
return {}
def get_refs(addon, branch=None, sha=None, connection=None):
"""Get the appropriate branch name and sha given the addon settings object,
and optionally the branch and sha from the request arguments.
:param str branch: Branch name. If None, return the default branch from the
repo settings.
:param str sha: The SHA.
:param GitHub connection: GitHub API object. If None, one will be created
from the addon's user settings.
"""
connection = connection or GitHub.from_settings(addon.user_settings)
if sha and not branch:
raise HTTPError(http.BAD_REQUEST)
# Get default branch if not provided
if not branch:
repo = connection.repo(addon.user, addon.repo)
if repo is None:
return None, None, None
branch = repo.default_branch
# Get registered branches if provided
registered_branches = (
[Branch.from_json(b) for b in addon.registration_data.get('branches', [])]
if addon.owner.is_registration
else []
)
registered_branch_names = [
each.name
for each in registered_branches
]
# Fail if registered and branch not in registration data
if registered_branches and branch not in registered_branch_names:
raise HTTPError(http.BAD_REQUEST)
# Get data from GitHub API if not registered
branches = registered_branches or connection.branches(addon.user, addon.repo)
# Use registered SHA if provided
for each in branches:
if branch == each.name:
sha = each.commit.sha
break
return branch, sha, branches
def github_hgrid_data_contents(**kwargs):
"""Return a repo's file tree as a dict formatted for HGrid.
"""
auth = kwargs['auth']
node = kwargs['node'] or kwargs['project']
node_addon = kwargs['node_addon']
path = kwargs.get('path', '')
connection = GitHub.from_settings(node_addon.user_settings)
# The requested branch and sha
req_branch, req_sha = request.args.get('branch'), request.args.get('sha')
# The actual branch and sha to use, given the addon settings
branch, sha, branches = get_refs(node_addon,
req_branch,
req_sha,
connection=connection)
# Get file tree
try:
contents = connection.contents(
user=node_addon.user,
repo=node_addon.repo,
path=path,
ref=sha or branch,
)
except ApiError:
raise HTTPError(http.NOT_FOUND)
can_edit = check_permissions(node_addon, auth, connection, branch, sha)
if contents:
hgrid_tree = to_hgrid(
contents,
node_url=node.url,
node_api_url=node.api_url,
branch=branch,
sha=sha,
can_edit=can_edit,
parent=path,
max_size=node_addon.config.max_file_size,
accepted_files=node_addon.config.accept_extensions)
else:
hgrid_tree = []
return hgrid_tree
def to_json(self, user):
ret = super(AddonGitHubNodeSettings, self).to_json(user)
user_settings = user.get_addon('github')
ret.update({
'user_has_auth': user_settings and user_settings.has_auth,
'is_registration': self.owner.is_registration,
})
if self.user_settings and self.user_settings.has_auth:
valid_credentials = False
owner = self.user_settings.owner
if user_settings and user_settings.owner == owner:
connection = GitHub.from_settings(user_settings)
# TODO: Fetch repo list client-side
# Since /user/repos excludes organization repos to which the
# current user has push access, we have to make extra requests to
# find them
valid_credentials = True
try:
repos = itertools.chain.from_iterable((connection.repos(), connection.my_org_repos()))
repo_names = [
'{0} / {1}'.format(repo.owner.login, repo.name)
for repo in repos
]
except GitHubError as error:
if error.code == http.UNAUTHORIZED:
repo_names = []
valid_credentials = False
ret.update({'repo_names': repo_names})
ret.update({
'node_has_auth': True,
'github_user': self.user or '',
'github_repo': self.repo or '',
'github_repo_full_name': '{0} / {1}'.format(self.user, self.repo),
'auth_osf_name': owner.fullname,
'auth_osf_url': owner.url,
'auth_osf_id': owner._id,
'github_user_name': self.user_settings.github_user_name,
'github_user_url': 'https://github.com/{0}'.format(self.user_settings.github_user_name),
'is_owner': owner == user,
'valid_credentials': valid_credentials,
'addons_url': web_url_for('user_addons'),
})
return ret
def get_refs(addon, branch=None, sha=None, connection=None):
"""Get the appropriate branch name and sha given the addon settings object,
and optionally the branch and sha from the request arguments.
:param str branch: Branch name. If None, return the default branch from the
repo settings.
:param str sha: The SHA.
:param GitHub connection: GitHub API object. If None, one will be created
from the addon's user settings.
"""
connection = connection or GitHub.from_settings(addon.user_settings)
if sha and not branch:
raise HTTPError(http.BAD_REQUEST)
# Get default branch if not provided
if not branch:
repo = connection.repo(addon.user, addon.repo)
if repo is None:
return None, None, None
branch = repo.default_branch
# Get registered branches if provided
registered_branches = ([
Branch.from_json(b)
for b in addon.registration_data.get('branches', [])
] if addon.owner.is_registration else [])
registered_branch_names = [each.name for each in registered_branches]
# Fail if registered and branch not in registration data
if registered_branches and branch not in registered_branch_names:
raise HTTPError(http.BAD_REQUEST)
# Get data from GitHub API if not registered
branches = registered_branches or connection.branches(
addon.user, addon.repo)
# Use registered SHA if provided
for each in branches:
if branch == each.name:
sha = each.commit.sha
break
return branch, sha, branches
def github_download_file(**kwargs):
node_settings = kwargs['node_addon']
path = get_path(kwargs)
ref = request.args.get('sha')
connection = GitHub.from_settings(node_settings.user_settings)
try:
name, data, _ = connection.file(node_settings.user,
node_settings.repo,
path,
ref=ref)
except TooBigError:
raise HTTPError(
http.BAD_REQUEST,
data={
'message_short':
'File too large',
'message_long':
'This file is too large to download through '
'the GitHub API.',
},
)
if data is None:
raise HTTPError(http.NOT_FOUND)
# Build response
resp = make_response(data)
mimetype = get_mimetype(path, data)
# Add binary MIME type if mimetype not found
if mimetype is None:
resp.headers['Content-Type'] = 'application/octet-stream'
else:
resp.headers['Content-Type'] = mimetype
resp.headers['Content-Disposition'] = 'attachment; filename={0}'.format(
name)
return resp
def to_json(self, user):
ret = super(AddonGitHubNodeSettings, self).to_json(user)
user_settings = user.get_addon("github")
ret.update(
{"user_has_auth": user_settings and user_settings.has_auth, "is_registration": self.owner.is_registration}
)
if self.user_settings and self.user_settings.has_auth:
valid_credentials = False
owner = self.user_settings.owner
if user_settings and user_settings.owner == owner:
connection = GitHub.from_settings(user_settings)
# TODO: Fetch repo list client-side
# Since /user/repos excludes organization repos to which the
# current user has push access, we have to make extra requests to
# find them
valid_credentials = True
try:
repos = itertools.chain.from_iterable((connection.repos(), connection.my_org_repos()))
repo_names = ["{0} / {1}".format(repo.owner.login, repo.name) for repo in repos]
except GitHubError as error:
if error.code == http.UNAUTHORIZED:
repo_names = []
valid_credentials = False
ret.update({"repo_names": repo_names})
ret.update(
{
"node_has_auth": True,
"github_user": self.user or "",
"github_repo": self.repo or "",
"github_repo_full_name": "{0} / {1}".format(self.user, self.repo),
"auth_osf_name": owner.fullname,
"auth_osf_url": owner.url,
"auth_osf_id": owner._id,
"github_user_name": self.user_settings.github_user_name,
"github_user_url": "https://github.com/{0}".format(self.user_settings.github_user_name),
"is_owner": owner == user,
"valid_credentials": valid_credentials,
"addons_url": web_url_for("user_addons"),
}
)
return ret
def add_hook(self, save=True):
if self.user_settings:
connect = GitHub.from_settings(self.user_settings)
secret = utils.make_hook_secret()
hook = connect.add_hook(
self.user,
self.repo,
"web",
{
"url": urlparse.urljoin(hook_domain, os.path.join(self.owner.api_url, "github", "hook/")),
"content_type": github_settings.HOOK_CONTENT_TYPE,
"secret": secret,
},
)
if hook:
self.hook_id = hook.id
self.hook_secret = secret
if save:
self.save()
def after_set_privacy(self, node, permissions):
"""
:param Node node:
:param str permissions:
:return str: Alert message
"""
if not github_settings.SET_PRIVACY:
return
connect = GitHub.from_settings(self.user_settings)
data = connect.set_privacy(
self.user, self.repo, permissions == 'private'
)
if data is None or 'errors' in data:
repo = connect.repo(self.user, self.repo)
if repo is not None:
current_privacy = 'private' if repo.private else 'public'
else:
current_privacy = 'unknown'
return (
'Could not set privacy for repo {user}::{repo}. '
'Current privacy status is {perm}.'.format(
user=self.user,
repo=self.repo,
perm=current_privacy,
)
)
return (
'GitHub repo {user}::{repo} made {perm}.'.format(
user=self.user,
repo=self.repo,
perm=permissions,
)
)
def github_download_file(**kwargs):
node_settings = kwargs['node_addon']
path = get_path(kwargs)
ref = request.args.get('sha')
connection = GitHub.from_settings(node_settings.user_settings)
try:
name, data, _ = connection.file(
node_settings.user, node_settings.repo, path, ref=ref
)
except TooBigError:
raise HTTPError(
http.BAD_REQUEST,
data={
'message_short': 'File too large',
'message_long': 'This file is too large to download through '
'the GitHub API.',
},
)
if data is None:
raise HTTPError(http.NOT_FOUND)
# Build response
resp = make_response(data)
mimetype = get_mimetype(path, data)
# Add binary MIME type if mimetype not found
if mimetype is None:
resp.headers['Content-Type'] = 'application/octet-stream'
else:
resp.headers['Content-Type'] = mimetype
resp.headers['Content-Disposition'] = 'attachment; filename={0}'.format(
name)
return resp
def github_hgrid_data_contents(**kwargs):
"""Return a repo's file tree as a dict formatted for HGrid.
"""
auth = kwargs['auth']
node = kwargs['node'] or kwargs['project']
node_addon = kwargs['node_addon']
path = kwargs.get('path', '')
connection = GitHub.from_settings(node_addon.user_settings)
# The requested branch and sha
req_branch, req_sha = request.args.get('branch'), request.args.get('sha')
# The actual branch and sha to use, given the addon settings
branch, sha, branches = get_refs(
node_addon, req_branch, req_sha, connection=connection
)
# Get file tree
try:
contents = connection.contents(
user=node_addon.user, repo=node_addon.repo, path=path,
ref=sha or branch,
)
except ApiError:
raise HTTPError(http.NOT_FOUND)
can_edit = check_permissions(node_addon, auth, connection, branch, sha)
if contents:
hgrid_tree = to_hgrid(
contents, node_url=node.url, node_api_url=node.api_url,
branch=branch, sha=sha, can_edit=can_edit, parent=path,
max_size=node_addon.config.max_file_size,
accepted_files=node_addon.config.accept_extensions
)
else:
hgrid_tree = []
return hgrid_tree
def is_private(self):
connection = GitHub.from_settings(self.user_settings)
return connection.repo(user=self.user, repo=self.repo).private
def github_hgrid_data(node_settings, auth, **kwargs):
# Quit if no repo linked
if not node_settings.complete:
return
connection = GitHub.from_settings(node_settings.user_settings)
# Initialize repo here in the event that it is set in the privacy check
# below. This potentially saves an API call in _check_permissions, below.
repo = None
# Quit if privacy mismatch and not contributor
node = node_settings.owner
if node.is_public and not node.is_contributor(auth.user):
try:
repo = connection.repo(node_settings.user, node_settings.repo)
except NotFoundError:
# TODO: Test me @jmcarp
# TODO: Add warning message
logger.error('Could not access GitHub repo')
return None
if repo.private:
return None
try:
branch, sha, branches = get_refs(
node_settings,
branch=kwargs.get('branch'),
sha=kwargs.get('sha'),
connection=connection,
)
except NotFoundError:
# TODO: Show an alert or change GitHub configuration?
logger.error('GitHub repo not found')
return
if branch is not None:
ref = ref_to_params(branch, sha)
can_edit = check_permissions(
node_settings,
auth,
connection,
branch,
sha,
repo=repo,
)
else:
ref = None
can_edit = False
name_tpl = '{user}/{repo}'.format(user=node_settings.user,
repo=node_settings.repo)
permissions = {'edit': can_edit, 'view': True}
urls = {
'upload':
node_settings.owner.api_url + 'github/file/' + (ref or ''),
'fetch':
node_settings.owner.api_url + 'github/hgrid/' + (ref or ''),
'branch':
node_settings.owner.api_url + 'github/hgrid/root/',
'zip':
node_settings.owner.api_url + 'github/zipball/' + (ref or ''),
'repo':
github_repo_url(owner=node_settings.user,
repo=node_settings.repo,
branch=branch)
}
return [
rubeus.build_addon_root(
node_settings,
name_tpl,
urls=urls,
permissions=permissions,
branches=[each.name for each in branches],
defaultBranch=branch,
)
]
def do_migration(records, dry=True):
count, inval_cred_handled = 0, 0
for raw_user_settings in records:
# False if missing, None if field exists
access_token = raw_user_settings.get('oauth_access_token', False)
token_type = raw_user_settings.get('oauth_token_type', False)
github_user_name = raw_user_settings.get('github_user', False)
if access_token and token_type and github_user_name:
if not dry:
gh = GitHub(access_token, token_type)
try:
github_user = gh.user()
except github3.models.GitHubError:
AddonGitHubUserSettings._storage[0].store.update(
{'_id': raw_user_settings['_id']},
{
'$unset': {
"oauth_access_token" : True,
"oauth_token_type" : True,
"github_user" : True,
},
}
)
inval_cred_handled += 1
print('invalidated credentials handled record: {}'.format(raw_user_settings['_id']))
continue
oauth_settings = AddonGitHubOauthSettings()
oauth_settings.github_user_id = str(github_user.id)
oauth_settings.save()
oauth_settings.oauth_access_token = access_token
oauth_settings.oauth_token_type = token_type
oauth_settings.github_user_name = github_user_name
oauth_settings.save()
AddonGitHubUserSettings._storage[0].store.update(
{'_id': raw_user_settings['_id']},
{
'$unset': {
'oauth_access_token': True,
'oauth_token_type': True,
'github_user': True,
},
'$set': {
'oauth_settings': oauth_settings.github_user_id,
}
}
)
AddonGitHubOauthSettings._storage[0].store.update(
{'github_user_id': oauth_settings.github_user_id},
{
'$push': {
'__backrefs.accessed.addongithubusersettings.oauth_settings': raw_user_settings['_id'],
}
}
)
print('Finished migrating AddonGithubUserSettings record: {}'.format(raw_user_settings['_id']))
count += 1
# Old fields have not yet been unset
elif None in set([access_token, token_type, github_user_name]):
if not dry:
AddonGitHubUserSettings._storage[0].store.update(
{'_id': raw_user_settings['_id']},
{
'$unset': {
'oauth_access_token': True,
'oauth_token_type': True,
'github_user': True,
},
}
)
print('Unset oauth_access_token and oauth_token_type: {0}'.format(raw_user_settings['_id']))
count += 1
return count, inval_cred_handled
def github_hgrid_data(node_settings, auth, **kwargs):
# Quit if no repo linked
if not node_settings.complete:
return
connection = GitHub.from_settings(node_settings.user_settings)
# Initialize repo here in the event that it is set in the privacy check
# below. This potentially saves an API call in _check_permissions, below.
repo = None
# Quit if privacy mismatch and not contributor
node = node_settings.owner
if node.is_public and not node.is_contributor(auth.user):
try:
repo = connection.repo(node_settings.user, node_settings.repo)
except NotFoundError:
# TODO: Test me @jmcarp
# TODO: Add warning message
logger.error('Could not access GitHub repo')
return None
if repo.private:
return None
try:
branch, sha, branches = get_refs(
node_settings,
branch=kwargs.get('branch'),
sha=kwargs.get('sha'),
connection=connection,
)
except NotFoundError:
# TODO: Show an alert or change GitHub configuration?
logger.error('GitHub repo not found')
return
if branch is not None:
ref = ref_to_params(branch, sha)
can_edit = check_permissions(
node_settings, auth, connection, branch, sha, repo=repo,
)
else:
ref = None
can_edit = False
name_tpl = '{user}/{repo}'.format(
user=node_settings.user, repo=node_settings.repo
)
permissions = {
'edit': can_edit,
'view': True
}
urls = {
'upload': node_settings.owner.api_url + 'github/file/' + (ref or ''),
'fetch': node_settings.owner.api_url + 'github/hgrid/' + (ref or ''),
'branch': node_settings.owner.api_url + 'github/hgrid/root/',
'zip': node_settings.owner.api_url + 'github/zipball/' + (ref or ''),
'repo': github_repo_url(owner=node_settings.user, repo=node_settings.repo, branch=branch)
}
return [rubeus.build_addon_root(
node_settings,
name_tpl,
urls=urls,
permissions=permissions,
branches=[each.name for each in branches],
defaultBranch=branch,
)]
def github_upload_file(auth, node_addon, **kwargs):
node = kwargs['node'] or kwargs['project']
user = auth.user
now = datetime.datetime.utcnow()
path = get_path(kwargs, required=False) or ''
branch = request.args.get('branch')
sha = request.args.get('sha')
if branch is None:
raise HTTPError(http.BAD_REQUEST)
connection = GitHub.from_settings(node_addon.user_settings)
upload = request.files.get('file')
filename = secure_filename(upload.filename)
content = upload.read()
# Check max file size
upload.seek(0, os.SEEK_END)
size = upload.tell()
if size > node_addon.config.max_file_size * 1024 * 1024:
raise HTTPError(http.BAD_REQUEST)
# Get SHA of existing file if present; requires an additional call to the
# GitHub API
try:
tree = connection.tree(node_addon.user,
node_addon.repo,
sha=sha or branch).tree
except EmptyRepoError:
tree = []
except NotFoundError:
raise HTTPError(http.BAD_REQUEST)
existing = [
thing for thing in tree if thing.path == os.path.join(path, filename)
]
sha = existing[0].sha if existing else None
author = {
'name': user.fullname,
'email': '{0}@osf.io'.format(user._id),
}
if existing:
data = connection.update_file(node_addon.user,
node_addon.repo,
os.path.join(path, filename),
MESSAGES['update'],
content,
sha=sha,
branch=branch,
author=author)
else:
data = connection.create_file(node_addon.user,
node_addon.repo,
os.path.join(path, filename),
MESSAGES['add'],
content,
branch=branch,
author=author)
if data is not None:
ref = ref_to_params(sha=data['commit'].sha)
view_url = os.path.join(node.url, 'github', 'file', path,
filename) + '/' + ref
download_url = os.path.join(node.url, 'github', 'file', path, filename,
'download') + '/' + ref
node.add_log(
action=('github_' + (models.NodeLog.FILE_UPDATED
if sha else models.NodeLog.FILE_ADDED)),
params={
'project': node.parent_id,
'node': node._primary_key,
'path': os.path.join(path, filename),
'urls': {
'view': view_url,
'download': download_url,
},
'github': {
'user': node_addon.user,
'repo': node_addon.repo,
'sha': data['commit'].sha,
},
},
auth=auth,
log_date=now,
)
# Fail if file size is not provided; this happens when the file was
# too large to upload to GitHub
if data['content'].size is None:
logger.error(
'Could not upload file {0} to GitHub: No size provided'.format(
filename))
raise HTTPError(http.BAD_REQUEST)
info = {
'addon':
'github',
'name':
filename,
'size': [
data['content'].size,
rubeus.format_filesize(data['content'].size),
],
'kind':
'file',
'urls':
build_github_urls(
data['content'],
node.url,
node.api_url,
branch,
sha,
),
'permissions': {
'view': True,
'edit': True,
},
}
return info, 201
raise HTTPError(http.BAD_REQUEST)
def do_migration(records, dry=True):
count, inval_cred_handled = 0, 0
for raw_user_settings in records:
# False if missing, None if field exists
access_token = raw_user_settings.get('oauth_access_token', False)
token_type = raw_user_settings.get('oauth_token_type', False)
github_user_name = raw_user_settings.get('github_user', False)
if access_token and token_type and github_user_name:
if not dry:
gh = GitHub(access_token, token_type)
try:
github_user = gh.user()
except github3.models.GitHubError:
AddonGitHubUserSettings._storage[0].store.update(
{'_id': raw_user_settings['_id']}, {
'$unset': {
"oauth_access_token": True,
"oauth_token_type": True,
"github_user": True,
},
})
inval_cred_handled += 1
print('invalidated credentials handled record: {}'.format(
raw_user_settings['_id']))
continue
oauth_settings = AddonGitHubOauthSettings()
oauth_settings.github_user_id = str(github_user.id)
oauth_settings.save()
oauth_settings.oauth_access_token = access_token
oauth_settings.oauth_token_type = token_type
oauth_settings.github_user_name = github_user_name
oauth_settings.save()
AddonGitHubUserSettings._storage[0].store.update(
{'_id': raw_user_settings['_id']}, {
'$unset': {
'oauth_access_token': True,
'oauth_token_type': True,
'github_user': True,
},
'$set': {
'oauth_settings': oauth_settings.github_user_id,
}
})
AddonGitHubOauthSettings._storage[0].store.update(
{'github_user_id': oauth_settings.github_user_id}, {
'$push': {
'__backrefs.accessed.addongithubusersettings.oauth_settings':
raw_user_settings['_id'],
}
})
print('Finished migrating AddonGithubUserSettings record: {}'.
format(raw_user_settings['_id']))
count += 1
# Old fields have not yet been unset
elif None in set([access_token, token_type, github_user_name]):
if not dry:
AddonGitHubUserSettings._storage[0].store.update(
{'_id': raw_user_settings['_id']}, {
'$unset': {
'oauth_access_token': True,
'oauth_token_type': True,
'github_user': True,
},
})
print('Unset oauth_access_token and oauth_token_type: {0}'.
format(raw_user_settings['_id']))
count += 1
return count, inval_cred_handled
def github_upload_file(auth, node_addon, **kwargs):
node = kwargs['node'] or kwargs['project']
user = auth.user
now = datetime.datetime.utcnow()
path = get_path(kwargs, required=False) or ''
branch = request.args.get('branch')
sha = request.args.get('sha')
if branch is None:
raise HTTPError(http.BAD_REQUEST)
connection = GitHub.from_settings(node_addon.user_settings)
upload = request.files.get('file')
filename = secure_filename(upload.filename)
content = upload.read()
# Check max file size
upload.seek(0, os.SEEK_END)
size = upload.tell()
if size > node_addon.config.max_file_size * 1024 * 1024:
raise HTTPError(http.BAD_REQUEST)
# Get SHA of existing file if present; requires an additional call to the
# GitHub API
try:
tree = connection.tree(
node_addon.user, node_addon.repo, sha=sha or branch
).tree
except EmptyRepoError:
tree = []
except NotFoundError:
raise HTTPError(http.BAD_REQUEST)
existing = [
thing
for thing in tree
if thing.path == os.path.join(path, filename)
]
sha = existing[0].sha if existing else None
author = {
'name': user.fullname,
'email': '{0}@osf.io'.format(user._id),
}
if existing:
data = connection.update_file(
node_addon.user, node_addon.repo, os.path.join(path, filename),
MESSAGES['update'], content, sha=sha, branch=branch, author=author
)
else:
data = connection.create_file(
node_addon.user, node_addon.repo, os.path.join(path, filename),
MESSAGES['add'], content, branch=branch, author=author
)
if data is not None:
ref = ref_to_params(sha=data['commit'].sha)
view_url = os.path.join(
node.url, 'github', 'file', path, filename
) + '/' + ref
download_url = os.path.join(
node.url, 'github', 'file', path, filename, 'download'
) + '/' + ref
node.add_log(
action=(
'github_' + (
models.NodeLog.FILE_UPDATED
if sha
else models.NodeLog.FILE_ADDED
)
),
params={
'project': node.parent_id,
'node': node._primary_key,
'path': os.path.join(path, filename),
'urls': {
'view': view_url,
'download': download_url,
},
'github': {
'user': node_addon.user,
'repo': node_addon.repo,
'sha': data['commit'].sha,
},
},
auth=auth,
log_date=now,
)
# Fail if file size is not provided; this happens when the file was
# too large to upload to GitHub
if data['content'].size is None:
logger.error(
'Could not upload file {0} to GitHub: No size provided'.format(
filename
)
)
raise HTTPError(http.BAD_REQUEST)
info = {
'addon': 'github',
'name': filename,
'size': [
data['content'].size,
rubeus.format_filesize(data['content'].size),
],
'kind': 'file',
'urls': build_github_urls(
data['content'], node.url, node.api_url, branch, sha,
),
'permissions': {
'view': True,
'edit': True,
},
}
return info, 201
raise HTTPError(http.BAD_REQUEST)
def is_private(self):
connection = GitHub.from_settings(self.user_settings)
return connection.repo(user=self.user, repo=self.repo).private
def github_view_file(auth, **kwargs):
node = kwargs['node'] or kwargs['project']
node_settings = kwargs['node_addon']
path = get_path(kwargs)
file_name = os.path.split(path)[1]
# Get branch / commit
branch = request.args.get('branch')
sha = request.args.get('sha', branch)
ref = sha or branch
connection = GitHub.from_settings(node_settings.user_settings)
# Get current file for delete url
current_file = connection.contents(
user=node_settings.user, repo=node_settings.repo, path=path,
ref=sha or branch)
anonymous = has_anonymous_link(node, auth)
try:
# If GUID has already been created, we won't redirect, and can check
# whether the file exists below
guid = GithubGuidFile.find_one(
Q('node', 'eq', node) &
Q('path', 'eq', path)
)
except ModularOdmException:
# If GUID doesn't exist, check whether file exists before creating
commits = connection.history(
node_settings.user, node_settings.repo, path, ref,
)
if not commits:
raise HTTPError(http.NOT_FOUND)
guid = GithubGuidFile(
node=node,
path=path,
)
guid.save()
redirect_url = check_file_guid(guid)
if redirect_url:
return redirect(redirect_url)
# Get default branch if neither SHA nor branch is provided
if ref is None:
repo = connection.repo(node_settings.user, node_settings.repo)
ref = branch = repo.default_branch
# Get file history; use SHA or branch if registered, else branch
start_sha = ref if node.is_registration else branch
commits = connection.history(
node_settings.user, node_settings.repo, path, sha=start_sha
)
# Get current commit
shas = [
commit['sha']
for commit in commits
]
if not shas:
raise HTTPError(http.NOT_FOUND)
current_sha = sha if sha in shas else shas[0]
# Get file URL
download_url = '/' + guid._id + '/download/' + ref_to_params(branch, current_sha)
render_url = os.path.join(
node.api_url, 'github', 'file', path, 'render'
) + '/' + ref_to_params(branch, current_sha)
delete_url = None
if current_file:
delete_url = node.api_url_for('github_delete_file', path=path) + ref_to_params(branch, current_file.sha)
for commit in commits:
commit['download'] = (
'/' + guid._id + '/download/' + ref_to_params(sha=commit['sha'])
)
commit['view'] = (
'/' + guid._id + '/' + ref_to_params(branch, sha=commit['sha'])
)
if anonymous:
commit['name'] = 'A user'
commit['email'] = ''
# Get or create rendered file
cache_file_name = get_cache_file(
path, current_sha,
)
rendered = get_cache_content(node_settings, cache_file_name)
if rendered is None:
try:
_, data, size = connection.file(
node_settings.user, node_settings.repo, path, ref=sha,
)
except TooBigError:
rendered = 'File too large to download.'
if rendered is None:
# Skip if too large to be rendered.
if github_settings.MAX_RENDER_SIZE is not None and size > github_settings.MAX_RENDER_SIZE:
rendered = 'File too large to render; download file to view it.'
else:
rendered = get_cache_content(
node_settings,
cache_file_name,
start_render=True,
remote_path=guid.path,
file_content=data,
download_url=download_url,
)
rv = {
'node': {
'id': node._id,
'title': node.title
},
'file_name': file_name,
'files_page_url': node.web_url_for('collect_file_trees'),
'current_sha': current_sha,
'render_url': render_url,
'rendered': rendered,
'download_url': download_url,
'delete_url': delete_url,
'commits': commits,
}
rv.update(_view_project(node, auth, primary=True))
return rv
def github_view_file(auth, **kwargs):
node = kwargs['node'] or kwargs['project']
node_settings = kwargs['node_addon']
path = get_path(kwargs)
file_name = os.path.split(path)[1]
# Get branch / commit
branch = request.args.get('branch')
sha = request.args.get('sha', branch)
ref = sha or branch
connection = GitHub.from_settings(node_settings.user_settings)
# Get current file for delete url
current_file = connection.contents(user=node_settings.user,
repo=node_settings.repo,
path=path,
ref=sha or branch)
anonymous = has_anonymous_link(node, auth)
try:
# If GUID has already been created, we won't redirect, and can check
# whether the file exists below
guid = GithubGuidFile.find_one(
Q('node', 'eq', node) & Q('path', 'eq', path))
except ModularOdmException:
# If GUID doesn't exist, check whether file exists before creating
commits = connection.history(
node_settings.user,
node_settings.repo,
path,
ref,
)
if not commits:
raise HTTPError(http.NOT_FOUND)
guid = GithubGuidFile(
node=node,
path=path,
)
guid.save()
redirect_url = check_file_guid(guid)
if redirect_url:
return redirect(redirect_url)
# Get default branch if neither SHA nor branch is provided
if ref is None:
repo = connection.repo(node_settings.user, node_settings.repo)
ref = branch = repo.default_branch
# Get file history; use SHA or branch if registered, else branch
start_sha = ref if node.is_registration else branch
commits = connection.history(node_settings.user,
node_settings.repo,
path,
sha=start_sha)
# Get current commit
shas = [commit['sha'] for commit in commits]
if not shas:
raise HTTPError(http.NOT_FOUND)
current_sha = sha if sha in shas else shas[0]
# Get file URL
download_url = '/' + guid._id + '/download/' + ref_to_params(
branch, current_sha)
render_url = os.path.join(node.api_url, 'github', 'file', path,
'render') + '/' + ref_to_params(
branch, current_sha)
delete_url = None
if current_file:
delete_url = node.api_url_for('github_delete_file',
path=path) + ref_to_params(
branch, current_file.sha)
for commit in commits:
commit['download'] = ('/' + guid._id + '/download/' +
ref_to_params(sha=commit['sha']))
commit['view'] = ('/' + guid._id + '/' +
ref_to_params(branch, sha=commit['sha']))
if anonymous:
commit['name'] = 'A user'
commit['email'] = ''
# Get or create rendered file
cache_file_name = get_cache_file(
path,
current_sha,
)
rendered = get_cache_content(node_settings, cache_file_name)
if rendered is None:
try:
_, data, size = connection.file(
node_settings.user,
node_settings.repo,
path,
ref=sha,
)
except TooBigError:
rendered = 'File too large to download.'
if rendered is None:
# Skip if too large to be rendered.
if github_settings.MAX_RENDER_SIZE is not None and size > github_settings.MAX_RENDER_SIZE:
rendered = 'File too large to render; download file to view it.'
else:
rendered = get_cache_content(
node_settings,
cache_file_name,
start_render=True,
remote_path=guid.path,
file_content=data,
download_url=download_url,
)
rv = {
'node': {
'id': node._id,
'title': node.title
},
'file_name': file_name,
'files_page_url': node.web_url_for('collect_file_trees'),
'current_sha': current_sha,
'render_url': render_url,
'rendered': rendered,
'download_url': download_url,
'delete_url': delete_url,
'commits': commits,
}
rv.update(_view_project(node, auth, primary=True))
return rv
def github_hgrid_data(node_settings, auth, **kwargs):
# Quit if no repo linked
if not node_settings.complete:
return
connection = GitHub.from_settings(node_settings.user_settings)
# Initialize repo here in the event that it is set in the privacy check
# below. This potentially saves an API call in _check_permissions, below.
repo = None
# Quit if privacy mismatch and not contributor
node = node_settings.owner
if node.is_public and not node.is_contributor(auth.user):
try:
repo = connection.repo(node_settings.user, node_settings.repo)
except NotFoundError:
# TODO: Test me @jmcarp
# TODO: Add warning message
logger.error("Could not access GitHub repo")
return None
if repo.private:
return None
try:
branch, sha, branches = get_refs(
node_settings, branch=kwargs.get("branch"), sha=kwargs.get("sha"), connection=connection
)
except (NotFoundError, GitHubError):
# TODO: Show an alert or change GitHub configuration?
logger.error("GitHub repo not found")
return
if branch is not None:
ref = ref_to_params(branch, sha)
can_edit = check_permissions(node_settings, auth, connection, branch, sha, repo=repo)
else:
ref = None
can_edit = False
name_tpl = "{user}/{repo}".format(user=node_settings.user, repo=node_settings.repo)
permissions = {"edit": can_edit, "view": True}
urls = {
"upload": node_settings.owner.api_url + "github/file/" + (ref or ""),
"fetch": node_settings.owner.api_url + "github/hgrid/" + (ref or ""),
"branch": node_settings.owner.api_url + "github/hgrid/root/",
"zip": node_settings.owner.api_url + "github/zipball/" + (ref or ""),
"repo": github_repo_url(owner=node_settings.user, repo=node_settings.repo, branch=branch),
}
branch_names = [each.name for each in branches]
if not branch_names:
branch_names = [branch] # if repo un-init-ed then still add default branch to list of branches
return [
rubeus.build_addon_root(
node_settings, name_tpl, urls=urls, permissions=permissions, branches=branch_names, defaultBranch=branch
)
]