def add_hook_to_old_node_settings(document, account):
connect = GitHubClient(external_account=account)
secret = make_hook_secret()
hook = None
try:
hook = connect.add_hook(
document['user'], document['repo'],
'web',
{
'url': urlparse.urljoin(
HOOK_DOMAIN,
os.path.join(
Node.load(document['owner']).api_url, 'github', 'hook/'
)
),
'content_type': github_settings.HOOK_CONTENT_TYPE,
'secret': secret,
},
events=github_settings.HOOK_EVENTS,
)
except ApiError:
pass
if hook:
database['addongithubnodesettings'].find_and_modify(
{'_id': document['_id']},
{
'$set': {
'hook_id': hook.id,
'hook_secret': secret
}
}
)
def add_hook(self, save=True):
if self.user_settings:
connect = GitHubClient(external_account=self.external_account)
secret = utils.make_hook_secret()
hook = connect.add_hook(
self.user,
self.repo,
'web',
{
'url':
urlparse.urljoin(
hook_domain,
os.path.join(self.owner.api_url, 'github', 'hook/')),
'content_type':
github_settings.HOOK_CONTENT_TYPE,
'secret':
secret,
},
events=github_settings.HOOK_EVENTS,
)
if hook:
self.hook_id = hook.id
self.hook_secret = secret
if save:
self.save()
def add_hook_to_old_node_settings(document, account):
connect = GitHubClient(external_account=account)
secret = make_hook_secret()
hook = None
try:
hook = connect.add_hook(
document['user'],
document['repo'],
'web',
{
'url':
urlparse.urljoin(
HOOK_DOMAIN,
os.path.join(
Node.load(document['owner']).api_url, 'github',
'hook/')),
'content_type':
github_settings.HOOK_CONTENT_TYPE,
'secret':
secret,
},
events=github_settings.HOOK_EVENTS,
)
except ApiError:
pass
if hook:
database['addongithubnodesettings'].find_and_modify(
{'_id': document['_id']},
{'$set': {
'hook_id': hook.id,
'hook_secret': secret
}})
def add_hook(self, save=True):
if self.user_settings:
connect = GitHubClient(external_account=self.external_account)
secret = utils.make_hook_secret()
hook = connect.add_hook(
self.user, self.repo,
'web',
{
'url': urlparse.urljoin(
hook_domain,
os.path.join(
self.owner.api_url, 'github', 'hook/'
)
),
'content_type': github_settings.HOOK_CONTENT_TYPE,
'secret': secret,
},
events=github_settings.HOOK_EVENTS,
)
if hook:
self.hook_id = hook.id
self.hook_secret = secret
if save:
self.save()
def to_json(self, user):
ret = super(GitHubNodeSettings, self).to_json(user)
user_settings = user.get_addon('github')
ret.update({
'user_has_auth': user_settings and user_settings.has_auth,
'is_registration': self.owner.is_registration,
})
if self.user_settings and self.user_settings.has_auth:
valid_credentials = False
owner = self.user_settings.owner
connection = GitHubClient(external_account=self.external_account)
# TODO: Fetch repo list client-side
# Since /user/repos excludes organization repos to which the
# current user has push access, we have to make extra requests to
# find them
valid_credentials = True
try:
repos = itertools.chain.from_iterable(
(connection.repos(), connection.my_org_repos()))
repo_names = [
'{0} / {1}'.format(repo.owner.login, repo.name)
for repo in repos
]
except GitHubError:
repo_names = []
valid_credentials = False
if owner == user:
ret.update({'repo_names': repo_names})
ret.update({
'node_has_auth':
True,
'github_user':
self.user or '',
'github_repo':
self.repo or '',
'github_repo_full_name':
'{0} / {1}'.format(self.user, self.repo) if
(self.user and self.repo) else '',
'auth_osf_name':
owner.fullname,
'auth_osf_url':
owner.url,
'auth_osf_id':
owner._id,
'github_user_name':
self.external_account.display_name,
'github_user_url':
self.external_account.profile_url,
'is_owner':
owner == user,
'valid_credentials':
valid_credentials,
'addons_url':
web_url_for('user_addons'),
'files_url':
self.owner.web_url_for('collect_file_trees')
})
return ret
def before_page_load(self, node, user):
"""
:param Node node:
:param User user:
:return str: Alert message
"""
messages = []
# Quit if not contributor
if not node.is_contributor(user):
return messages
# Quit if not configured
if self.user is None or self.repo is None:
return messages
# Quit if no user authorization
if self.user_settings is None:
return messages
connect = GitHubClient(external_account=self.external_account)
try:
repo = connect.repo(self.user, self.repo)
except (ApiError, GitHubError):
return
node_permissions = 'public' if node.is_public else 'private'
repo_permissions = 'private' if repo.private else 'public'
if repo_permissions != node_permissions:
message = (
'Warning: This OSF {category} is {node_perm}, but the GitHub '
'repo {user} / {repo} is {repo_perm}.'.format(
category=markupsafe.escape(node.project_or_component),
node_perm=markupsafe.escape(node_permissions),
repo_perm=markupsafe.escape(repo_permissions),
user=markupsafe.escape(self.user),
repo=markupsafe.escape(self.repo),
)
)
if repo_permissions == 'private':
message += (
' Users can view the contents of this private GitHub '
'repository through this public project.'
)
else:
message += (
' The files in this GitHub repo can be viewed on GitHub '
'<u><a href="https://github.com/{user}/{repo}/">here</a></u>.'
).format(
user=self.user,
repo=self.repo,
)
messages.append(message)
return messages
def before_page_load(self, node, user):
"""
:param Node node:
:param User user:
:return str: Alert message
"""
messages = []
# Quit if not contributor
if not node.is_contributor(user):
return messages
# Quit if not configured
if self.user is None or self.repo is None:
return messages
# Quit if no user authorization
if self.user_settings is None:
return messages
connect = GitHubClient(external_account=self.external_account)
try:
repo = connect.repo(self.user, self.repo)
except (ApiError, GitHubError):
return
node_permissions = 'public' if node.is_public else 'private'
repo_permissions = 'private' if repo.private else 'public'
if repo_permissions != node_permissions:
message = (
'Warning: This OSF {category} is {node_perm}, but the GitHub '
'repo {user} / {repo} is {repo_perm}.'.format(
category=node.project_or_component,
node_perm=node_permissions,
repo_perm=repo_permissions,
user=self.user,
repo=self.repo,
)
)
if repo_permissions == 'private':
message += (
' Users can view the contents of this private GitHub '
'repository through this public project.'
)
else:
message += (
' The files in this GitHub repo can be viewed on GitHub '
'<u><a href="https://github.com/{user}/{repo}/">here</a></u>.'
).format(
user=self.user,
repo=self.repo,
)
messages.append(message)
return messages
def revoke_remote_oauth_access(self, external_account):
"""Overrides default behavior during external_account deactivation.
Tells GitHub to remove the grant for the OSF associated with this account.
"""
connection = GitHubClient(external_account=external_account)
try:
connection.revoke_token()
except GitHubError:
pass
def revoke_remote_oauth_access(self, external_account):
"""Overrides default behavior during external_account deactivation.
Tells GitHub to remove the grant for the OSF associated with this account.
"""
connection = GitHubClient(external_account=external_account)
try:
connection.revoke_token()
except GitHubError:
pass
def handle_callback(self, response):
"""View called when the OAuth flow is completed. Adds a new GitHubUserSettings
record to the user and saves the account info.
"""
client = GitHubClient(access_token=response['access_token'])
user_info = client.user()
return {
'provider_id': str(user_info.id),
'profile_url': user_info.html_url,
'display_name': user_info.login
}
def github_download_starball(node_addon, **kwargs):
archive = kwargs.get('archive', 'tar')
ref = request.args.get('sha', 'master')
connection = GitHubClient(external_account=node_addon.external_account)
headers, data = connection.starball(node_addon.user, node_addon.repo,
archive, ref)
resp = make_response(data)
for key, value in headers.iteritems():
resp.headers[key] = value
return resp
def github_download_starball(node_addon, **kwargs):
archive = kwargs.get('archive', 'tar')
ref = request.args.get('sha', 'master')
connection = GitHubClient(external_account=node_addon.external_account)
headers, data = connection.starball(
node_addon.user, node_addon.repo, archive, ref
)
resp = make_response(data)
for key, value in headers.iteritems():
resp.headers[key] = value
return resp
def handle_callback(self, response):
"""View called when the OAuth flow is completed. Adds a new GitHubUserSettings
record to the user and saves the account info.
"""
client = GitHubClient(
access_token=response['access_token']
)
user_info = client.user()
return {
'provider_id': str(user_info.id),
'profile_url': user_info.html_url,
'display_name': user_info.login
}
def delete_hook(self, save=True):
"""
:return bool: Hook was deleted
"""
if self.user_settings and self.hook_id:
connection = GitHubClient(external_account=self.external_account)
try:
response = connection.delete_hook(self.user, self.repo, self.hook_id)
except (GitHubError, NotFoundError):
return False
if response:
self.hook_id = None
if save:
self.save()
return True
return False
def delete_hook(self, save=True):
"""
:return bool: Hook was deleted
"""
if self.user_settings and self.hook_id:
connection = GitHubClient(external_account=self.external_account)
try:
response = connection.delete_hook(self.user, self.repo, self.hook_id)
except (GitHubError, NotFoundError):
return False
if response:
self.hook_id = None
if save:
self.save()
return True
return False
def credentials_are_valid(self, user_settings, client):
if user_settings:
client = client or GitHubClient(external_account=user_settings.external_accounts[0])
try:
client.user()
except (GitHubError, IndexError):
return False
return True
def github_create_repo(**kwargs):
repo_name = request.json.get('name')
if not repo_name:
raise HTTPError(http.BAD_REQUEST)
node_settings = kwargs['node_addon']
connection = GitHubClient(external_account=node_settings.external_account)
try:
repo = connection.create_repo(repo_name, auto_init=True)
except GitHubError:
# TODO: Check status code
raise HTTPError(http.BAD_REQUEST)
return {
'user': repo.owner.login,
'repo': repo.name,
}
def github_create_repo(**kwargs):
repo_name = request.json.get('name')
if not repo_name:
raise HTTPError(http.BAD_REQUEST)
node_settings = kwargs['node_addon']
connection = GitHubClient(external_account=node_settings.external_account)
try:
repo = connection.create_repo(repo_name, auto_init=True)
except GitHubError:
# TODO: Check status code
raise HTTPError(http.BAD_REQUEST)
return {
'user': repo.owner.login,
'repo': repo.name,
}
def to_json(self, user):
ret = super(GitHubNodeSettings, self).to_json(user)
user_settings = user.get_addon('github')
ret.update({
'user_has_auth': user_settings and user_settings.has_auth,
'is_registration': self.owner.is_registration,
})
if self.user_settings and self.user_settings.has_auth:
valid_credentials = False
owner = self.user_settings.owner
connection = GitHubClient(external_account=self.external_account)
# TODO: Fetch repo list client-side
# Since /user/repos excludes organization repos to which the
# current user has push access, we have to make extra requests to
# find them
valid_credentials = True
try:
repos = itertools.chain.from_iterable((connection.repos(), connection.my_org_repos()))
repo_names = [
'{0} / {1}'.format(repo.owner.login, repo.name)
for repo in repos
]
except GitHubError:
repo_names = []
valid_credentials = False
if owner == user:
ret.update({'repo_names': repo_names})
ret.update({
'node_has_auth': True,
'github_user': self.user or '',
'github_repo': self.repo or '',
'github_repo_full_name': '{0} / {1}'.format(self.user, self.repo) if (self.user and self.repo) else '',
'auth_osf_name': owner.fullname,
'auth_osf_url': owner.url,
'auth_osf_id': owner._id,
'github_user_name': self.external_account.display_name,
'github_user_url': self.external_account.profile_url,
'is_owner': owner == user,
'valid_credentials': valid_credentials,
'addons_url': web_url_for('user_addons'),
'files_url': self.owner.web_url_for('collect_file_trees')
})
return ret
def migrate_to_external_account(user_settings_document,
oauth_settings_document):
if not oauth_settings_document.get('oauth_access_token'):
return (None, None, None)
try:
user_info = GitHubClient(
access_token=oauth_settings_document['oauth_access_token']).user()
except (GitHubError, ApiError):
user_id = oauth_settings_document['github_user_id']
profile_url = None
display_name = oauth_settings_document['github_user_name']
else:
user_id = user_info.id
profile_url = user_info.html_url
display_name = user_info.login
new = False
user = User.load(user_settings_document['owner'])
try:
external_account = ExternalAccount.find(Q('provider_id', 'eq',
user_id))[0]
logger.info(
'Duplicate account use found: User {0} with github_user_id {1}'.
format(user.username, user_id))
except IndexError:
new = True
external_account = ExternalAccount(
provider=PROVIDER,
provider_name=PROVIDER_NAME,
provider_id=user_id,
profile_url=profile_url,
oauth_key=oauth_settings_document['oauth_access_token'],
display_name=display_name,
)
external_account.save()
if not profile_url:
invalid_oauth_creds[external_account._id] = (
user_settings_document['_id'], oauth_settings_document['_id'])
logger.info(
"Created ExternalAccount<_id:{0}> with invalid oauth credentials."
.format(external_account._id))
user.external_accounts.append(external_account)
user.save()
return external_account, user, new
def creds_are_valid(ea_id):
logger.warn('Validating credentials for externalaccount {}'.format(ea_id))
ea = ExternalAccount.load(ea_id)
if ea.provider == 'github':
try:
GitHubClient(external_account=ea).user()
except (GitHubError, IndexError):
logger.info('Invalid creds: {}'.format(ea_id))
return False
elif ea.provider == 'dropbox':
try:
DropboxClient(ea.oauth_key).account_info()
except (ValueError, IndexError, ErrorResponse):
logger.info('Invalid creds: {}'.format(ea_id))
return False
else:
raise Exception('Unexpected provider: {}'.format(ea.provider))
logger.info('Valid creds: {}'.format(ea_id))
return True
def get_refs(addon, branch=None, sha=None, connection=None):
"""Get the appropriate branch name and sha given the addon settings object,
and optionally the branch and sha from the request arguments.
:param str branch: Branch name. If None, return the default branch from the
repo settings.
:param str sha: The SHA.
:param GitHub connection: GitHub API object. If None, one will be created
from the addon's user settings.
"""
connection = connection or GitHubClient(
external_account=addon.external_account)
if sha and not branch:
raise HTTPError(http.BAD_REQUEST)
# Get default branch if not provided
if not branch:
repo = connection.repo(addon.user, addon.repo)
if repo is None:
return None, None, None
branch = repo.default_branch
# Get registered branches if provided
registered_branches = ([
Branch.from_json(b)
for b in addon.registration_data.get('branches', [])
] if addon.owner.is_registration else [])
registered_branch_names = [each.name for each in registered_branches]
# Fail if registered and branch not in registration data
if registered_branches and branch not in registered_branch_names:
raise HTTPError(http.BAD_REQUEST)
# Get data from GitHub API if not registered
branches = registered_branches or connection.branches(
addon.user, addon.repo)
# Use registered SHA if provided
for each in branches:
if branch == each.name:
sha = each.commit.sha
break
return branch, sha, branches
class TestGitHubSerializer(StorageAddonSerializerTestSuiteMixin, OsfTestCase):
addon_short_name = 'github'
Serializer = GitHubSerializer
ExternalAccountFactory = GitHubAccountFactory
client = GitHubClient()
def set_provider_id(self, pid):
self.node_settings.folder_id = pid
## Overrides ##
def setUp(self):
super(TestGitHubSerializer, self).setUp()
self.mock_api_user = mock.patch(
"website.addons.github.api.GitHubClient.user")
self.mock_api_user.return_value = mock.Mock()
self.mock_api_user.start()
def tearDown(self):
self.mock_api_user.stop()
super(TestGitHubSerializer, self).tearDown()
def github_set_config(auth, **kwargs):
node_settings = kwargs.get('node_addon', None)
node = kwargs.get('node', None)
user_settings = kwargs.get('user_addon', None)
try:
if not node:
node = node_settings.owner
if not user_settings:
user_settings = node_settings.user_settings
except AttributeError:
raise HTTPError(http.BAD_REQUEST)
# Parse request
github_user_name = request.json.get('github_user', '')
github_repo_name = request.json.get('github_repo', '')
if not github_user_name or not github_repo_name:
raise HTTPError(http.BAD_REQUEST)
# Verify that repo exists and that user can access
connection = GitHubClient(external_account=node_settings.external_account)
repo = connection.repo(github_user_name, github_repo_name)
if repo is None:
if user_settings:
message = (
'Cannot access repo. Either the repo does not exist '
'or your account does not have permission to view it.'
)
else:
message = (
'Cannot access repo.'
)
return {'message': message}, http.BAD_REQUEST
changed = (
github_user_name != node_settings.user or
github_repo_name != node_settings.repo
)
# Update hooks
if changed:
# Delete existing hook, if any
node_settings.delete_hook()
# Update node settings
node_settings.user = github_user_name
node_settings.repo = github_repo_name
# Log repo select
node.add_log(
action='github_repo_linked',
params={
'project': node.parent_id,
'node': node._id,
'github': {
'user': github_user_name,
'repo': github_repo_name,
}
},
auth=auth,
)
# Add new hook
if node_settings.user and node_settings.repo:
node_settings.add_hook(save=False)
node_settings.save()
return {}
def is_private(self):
connection = GitHubClient(external_account=self.external_account)
return connection.repo(user=self.user, repo=self.repo).private
def github_hgrid_data(node_settings, auth, **kwargs):
# Quit if no repo linked
if not node_settings.complete:
return
connection = GitHubClient(external_account=node_settings.external_account)
# Initialize repo here in the event that it is set in the privacy check
# below. This potentially saves an API call in _check_permissions, below.
repo = None
# Quit if privacy mismatch and not contributor
node = node_settings.owner
if node.is_public and not node.is_contributor(auth.user):
try:
repo = connection.repo(node_settings.user, node_settings.repo)
except NotFoundError:
# TODO: Test me @jmcarp
# TODO: Add warning message
logger.error('Could not access GitHub repo')
return None
if repo.private:
return None
try:
branch, sha, branches = get_refs(
node_settings,
branch=kwargs.get('branch'),
sha=kwargs.get('sha'),
connection=connection,
)
except (NotFoundError, GitHubError):
# TODO: Show an alert or change GitHub configuration?
logger.error('GitHub repo not found')
return
if branch is not None:
ref = ref_to_params(branch, sha)
can_edit = check_permissions(
node_settings, auth, connection, branch, sha, repo=repo,
)
else:
ref = None
can_edit = False
name_tpl = '{user}/{repo}'.format(
user=node_settings.user, repo=node_settings.repo
)
permissions = {
'edit': can_edit,
'view': True,
'private': node_settings.is_private
}
urls = {
'upload': node_settings.owner.api_url + 'github/file/' + (ref or ''),
'fetch': node_settings.owner.api_url + 'github/hgrid/' + (ref or ''),
'branch': node_settings.owner.api_url + 'github/hgrid/root/',
'zip': node_settings.owner.api_url + 'github/zipball/' + (ref or ''),
'repo': "https://github.com/{0}/{1}/tree/{2}".format(node_settings.user, node_settings.repo, branch)
}
branch_names = [each.name for each in branches]
if not branch_names:
branch_names = [branch] # if repo un-init-ed then still add default branch to list of branches
return [rubeus.build_addon_root(
node_settings,
name_tpl,
urls=urls,
permissions=permissions,
branches=branch_names,
defaultBranch=branch,
)]
def github_hgrid_data(node_settings, auth, **kwargs):
# Quit if no repo linked
if not node_settings.complete:
return
connection = GitHubClient(external_account=node_settings.external_account)
# Initialize repo here in the event that it is set in the privacy check
# below. This potentially saves an API call in _check_permissions, below.
repo = None
# Quit if privacy mismatch and not contributor
node = node_settings.owner
if node.is_public and not node.is_contributor(auth.user):
try:
repo = connection.repo(node_settings.user, node_settings.repo)
except NotFoundError:
# TODO: Test me @jmcarp
# TODO: Add warning message
logger.error('Could not access GitHub repo')
return None
if repo.private:
return None
try:
branch, sha, branches = get_refs(
node_settings,
branch=kwargs.get('branch'),
sha=kwargs.get('sha'),
connection=connection,
)
except (NotFoundError, GitHubError):
# TODO: Show an alert or change GitHub configuration?
logger.error('GitHub repo not found')
return
if branch is not None:
ref = ref_to_params(branch, sha)
can_edit = check_permissions(
node_settings,
auth,
connection,
branch,
sha,
repo=repo,
)
else:
ref = None
can_edit = False
name_tpl = '{user}/{repo}'.format(user=node_settings.user,
repo=node_settings.repo)
permissions = {
'edit': can_edit,
'view': True,
'private': node_settings.is_private
}
urls = {
'upload':
node_settings.owner.api_url + 'github/file/' + (ref or ''),
'fetch':
node_settings.owner.api_url + 'github/hgrid/' + (ref or ''),
'branch':
node_settings.owner.api_url + 'github/hgrid/root/',
'zip':
node_settings.owner.api_url + 'github/zipball/' + (ref or ''),
'repo':
'https://github.com/{0}/{1}/tree/{2}'.format(node_settings.user,
node_settings.repo,
branch)
}
branch_names = [each.name for each in branches]
if not branch_names:
branch_names = [
branch
] # if repo un-init-ed then still add default branch to list of branches
return [
rubeus.build_addon_root(
node_settings,
name_tpl,
urls=urls,
permissions=permissions,
branches=branch_names,
defaultBranch=branch,
private_key=kwargs.get('view_only', None),
)
]
def github_set_config(auth, **kwargs):
node_settings = kwargs.get('node_addon', None)
node = kwargs.get('node', None)
user_settings = kwargs.get('user_addon', None)
try:
if not node:
node = node_settings.owner
if not user_settings:
user_settings = node_settings.user_settings
except AttributeError:
raise HTTPError(http.BAD_REQUEST)
# Parse request
github_user_name = request.json.get('github_user', '')
github_repo_name = request.json.get('github_repo', '')
if not github_user_name or not github_repo_name:
raise HTTPError(http.BAD_REQUEST)
# Verify that repo exists and that user can access
connection = GitHubClient(external_account=node_settings.external_account)
repo = connection.repo(github_user_name, github_repo_name)
if repo is None:
if user_settings:
message = ('Cannot access repo. Either the repo does not exist '
'or your account does not have permission to view it.')
else:
message = ('Cannot access repo.')
return {'message': message}, http.BAD_REQUEST
changed = (github_user_name != node_settings.user
or github_repo_name != node_settings.repo)
# Update hooks
if changed:
# Delete existing hook, if any
node_settings.delete_hook()
# Update node settings
node_settings.user = github_user_name
node_settings.repo = github_repo_name
# Log repo select
node.add_log(
action='github_repo_linked',
params={
'project': node.parent_id,
'node': node._id,
'github': {
'user': github_user_name,
'repo': github_repo_name,
}
},
auth=auth,
)
# Add new hook
if node_settings.user and node_settings.repo:
node_settings.add_hook(save=False)
node_settings.save()
return {}
def is_private(self):
connection = GitHubClient(external_account=self.external_account)
return connection.repo(user=self.user, repo=self.repo).private