def after_set_privacy(self, node, permissions): """ :param Node node: :param str permissions: :return str: Alert message """ if not github_settings.SET_PRIVACY: return connect = GitHub.from_settings(self.user_settings) data = connect.set_privacy(self.user, self.repo, permissions == "private") if data is None or "errors" in data: repo = connect.repo(self.user, self.repo) if repo is not None: current_privacy = "private" if repo.private else "public" else: current_privacy = "unknown" return "Could not set privacy for repo {user}::{repo}. " "Current privacy status is {perm}.".format( user=self.user, repo=self.repo, perm=current_privacy ) return "GitHub repo {user}::{repo} made {perm}.".format(user=self.user, repo=self.repo, perm=permissions)
def add_hook(self, save=True): if self.user_settings: connect = GitHub.from_settings(self.user_settings) secret = utils.make_hook_secret() hook = connect.add_hook( self.user, self.repo, 'web', { 'url': urlparse.urljoin( hook_domain, os.path.join( self.owner.api_url, 'github', 'hook/' ) ), 'content_type': github_settings.HOOK_CONTENT_TYPE, 'secret': secret, } ) if hook: self.hook_id = hook.id self.hook_secret = secret if save: self.save()
def after_register(self, node, registration, user, save=True): """ :param Node node: Original node :param Node registration: Registered node :param User user: User creating registration :param bool save: Save settings after callback :return tuple: Tuple of cloned settings and alert message """ clone, message = super(AddonGitHubNodeSettings, self).after_register(node, registration, user, save=False) # Copy foreign fields from current add-on clone.user_settings = self.user_settings # Store current branch data if self.user and self.repo: connect = GitHub.from_settings(self.user_settings) try: branches = [ branch.to_json() for branch in connect.branches(self.user, self.repo) ] clone.registration_data['branches'] = branches except ApiError: pass if save: clone.save() return clone, message
def update_hook(node_settings): """Discard the existing webhook for a GitHub node add-on and create a new one. """ logger.warn( 'Updating GitHub hook on node {0}'.format( node_settings.owner._id ) ) connection = GitHub.from_settings(node_settings.user_settings) repo = connection.repo(node_settings.user, node_settings.repo) hook = repo.hook(node_settings.hook_id) if hook is None: logger.warn('Hook {0} not found'.format(node_settings.hook_id)) return secret = utils.make_hook_secret() config = hook.config config['content_type'] = github_settings.HOOK_CONTENT_TYPE config['secret'] = secret hook.edit(config=config) node_settings.hook_secret = secret node_settings.save()
def update_hook(node_settings): """Discard the existing webhook for a GitHub node add-on and create a new one. """ logger.warn('Updating GitHub hook on node {0}'.format( node_settings.owner._id)) connection = GitHub.from_settings(node_settings.user_settings) repo = connection.repo(node_settings.user, node_settings.repo) hook = repo.hook(node_settings.hook_id) if hook is None: logger.warn('Hook {0} not found'.format(node_settings.hook_id)) return secret = utils.make_hook_secret() config = hook.config config['content_type'] = github_settings.HOOK_CONTENT_TYPE config['secret'] = secret hook.edit(config=config) node_settings.hook_secret = secret node_settings.save()
def after_set_privacy(self, node, permissions): """ :param Node node: :param str permissions: :return str: Alert message """ if not github_settings.SET_PRIVACY: return connect = GitHub.from_settings(self.user_settings) data = connect.set_privacy(self.user, self.repo, permissions == 'private') if data is None or 'errors' in data: repo = connect.repo(self.user, self.repo) if repo is not None: current_privacy = 'private' if repo.private else 'public' else: current_privacy = 'unknown' return ('Could not set privacy for repo {user}::{repo}. ' 'Current privacy status is {perm}.'.format( user=self.user, repo=self.repo, perm=current_privacy, )) return ('GitHub repo {user}::{repo} made {perm}.'.format( user=self.user, repo=self.repo, perm=permissions, ))
def tests_none_trees_raise_empty_repo(self, mock_repo): mock_tree = mock.Mock() mock_tree.tree.return_value = None mock_repo.return_value = mock_tree with assert_raises(EmptyRepoError): GitHub().tree('', '', '')
def after_register(self, node, registration, user, save=True): """ :param Node node: Original node :param Node registration: Registered node :param User user: User creating registration :param bool save: Save settings after callback :return tuple: Tuple of cloned settings and alert message """ clone, message = super(AddonGitHubNodeSettings, self).after_register( node, registration, user, save=False ) # Copy foreign fields from current add-on clone.user_settings = self.user_settings # Store current branch data if self.user and self.repo: connect = GitHub.from_settings(self.user_settings) try: branches = [ branch.to_json() for branch in connect.branches(self.user, self.repo) ] clone.registration_data['branches'] = branches except ApiError: pass if save: clone.save() return clone, message
def to_json(self, user): ret = super(AddonGitHubNodeSettings, self).to_json(user) user_settings = user.get_addon('github') ret.update({ 'user_has_auth': user_settings and user_settings.has_auth, 'is_registration': self.owner.is_registration, }) if self.user_settings and self.user_settings.has_auth: valid_credentials = False owner = self.user_settings.owner if user_settings and user_settings.owner == owner: connection = GitHub.from_settings(user_settings) # TODO: Fetch repo list client-side # Since /user/repos excludes organization repos to which the # current user has push access, we have to make extra requests to # find them valid_credentials = True try: repos = itertools.chain.from_iterable( (connection.repos(), connection.my_org_repos())) repo_names = [ '{0} / {1}'.format(repo.owner.login, repo.name) for repo in repos ] except GitHubError as error: if error.code == http.UNAUTHORIZED: repo_names = [] valid_credentials = False ret.update({'repo_names': repo_names}) ret.update({ 'node_has_auth': True, 'github_user': self.user or '', 'github_repo': self.repo or '', 'github_repo_full_name': '{0} / {1}'.format(self.user, self.repo), 'auth_osf_name': owner.fullname, 'auth_osf_url': owner.url, 'auth_osf_id': owner._id, 'github_user_name': self.user_settings.github_user_name, 'github_user_url': 'https://github.com/{0}'.format( self.user_settings.github_user_name), 'is_owner': owner == user, 'valid_credentials': valid_credentials, 'addons_url': web_url_for('user_addons'), }) return ret
def before_page_load(self, node, user): """ :param Node node: :param User user: :return str: Alert message """ messages = [] # Quit if not contributor if not node.is_contributor(user): return messages # Quit if not configured if self.user is None or self.repo is None: return messages # Quit if no user authorization if self.user_settings is None: return messages connect = GitHub.from_settings(self.user_settings) try: repo = connect.repo(self.user, self.repo) except ApiError: return node_permissions = 'public' if node.is_public else 'private' repo_permissions = 'private' if repo.private else 'public' if repo_permissions != node_permissions: message = ( 'Warnings: This OSF {category} is {node_perm}, but the GitHub ' 'repo {user} / {repo} is {repo_perm}.'.format( category=node.project_or_component, node_perm=node_permissions, repo_perm=repo_permissions, user=self.user, repo=self.repo, ) ) if repo_permissions == 'private': message += ( ' Users can view the contents of this private GitHub ' 'repository through this public project.' ) else: message += ( ' The files in this GitHub repo can be viewed on GitHub ' '<a href="https://github.com/{user}/{repo}/">here</a>.' ).format( user=self.user, repo=self.repo, ) messages.append(message) return messages
def before_page_load(self, node, user): """ :param Node node: :param User user: :return str: Alert message """ messages = [] # Quit if not contributor if not node.is_contributor(user): return messages # Quit if not configured if self.user is None or self.repo is None: return messages # Quit if no user authorization if self.user_settings is None: return messages connect = GitHub.from_settings(self.user_settings) try: repo = connect.repo(self.user, self.repo) except (ApiError, GitHubError): return node_permissions = 'public' if node.is_public else 'private' repo_permissions = 'private' if repo.private else 'public' if repo_permissions != node_permissions: message = ( 'Warnings: This OSF {category} is {node_perm}, but the GitHub ' 'repo {user} / {repo} is {repo_perm}.'.format( category=node.project_or_component, node_perm=node_permissions, repo_perm=repo_permissions, user=self.user, repo=self.repo, ) ) if repo_permissions == 'private': message += ( ' Users can view the contents of this private GitHub ' 'repository through this public project.' ) else: message += ( ' The files in this GitHub repo can be viewed on GitHub ' '<a href="https://github.com/{user}/{repo}/">here</a>.' ).format( user=self.user, repo=self.repo, ) messages.append(message) return messages
def revoke_token(self): connection = GitHub.from_settings(self) try: connection.revoke_token() except GitHubError as error: if error.code == http.UNAUTHORIZED: return ( 'Your GitHub credentials were removed from the OSF, but we ' 'were unable to revoke your access token from GitHub. Your ' 'GitHub credentials may no longer be valid.') raise
def revoke_token(self): connection = GitHub.from_settings(self) try: connection.revoke_token() except GitHubError as error: if error.code == http.UNAUTHORIZED: return ( 'Your GitHub credentials were removed from the OSF, but we ' 'were unable to revoke your access token from GitHub. Your ' 'GitHub credentials may no longer be valid.' ) raise
def github_delete_file(auth, node_addon, **kwargs): node = kwargs['node'] or kwargs['project'] now = datetime.datetime.utcnow() # Must remove trailing slash, else GitHub fails silently on delete path = get_path(kwargs).rstrip('/') sha = request.args.get('sha') if sha is None: raise HTTPError(http.BAD_REQUEST) branch = request.args.get('branch') author = { 'name': auth.user.fullname, 'email': '{0}@osf.io'.format(auth.user._id), } connection = GitHub.from_settings(node_addon.user_settings) data = connection.delete_file( node_addon.user, node_addon.repo, path, MESSAGES['delete'], sha=sha, branch=branch, author=author, ) if data is None: raise HTTPError(http.BAD_REQUEST) node.add_log( action='github_' + models.NodeLog.FILE_REMOVED, params={ 'project': node.parent_id, 'node': node._primary_key, 'path': path, 'github': { 'user': node_addon.user, 'repo': node_addon.repo, }, }, auth=auth, log_date=now, ) return {}
def github_download_starball(node_addon, **kwargs): archive = kwargs.get('archive', 'tar') ref = request.args.get('sha', 'master') connection = GitHub.from_settings(node_addon.user_settings) headers, data = connection.starball(node_addon.user, node_addon.repo, archive, ref) resp = make_response(data) for key, value in headers.iteritems(): resp.headers[key] = value return resp
def github_download_starball(node_addon, **kwargs): archive = kwargs.get('archive', 'tar') ref = request.args.get('sha', 'master') connection = GitHub.from_settings(node_addon.user_settings) headers, data = connection.starball( node_addon.user, node_addon.repo, archive, ref ) resp = make_response(data) for key, value in headers.iteritems(): resp.headers[key] = value return resp
def delete_hook(self, save=True): """ :return bool: Hook was deleted """ if self.user_settings and self.hook_id: connection = GitHub.from_settings(self.user_settings) try: response = connection.delete_hook(self.user, self.repo, self.hook_id) except (GitHubError, NotFoundError): return False if response: self.hook_id = None if save: self.save() return True return False
def get_refs(addon, branch=None, sha=None, connection=None): """Get the appropriate branch name and sha given the addon settings object, and optionally the branch and sha from the request arguments. :param str branch: Branch name. If None, return the default branch from the repo settings. :param str sha: The SHA. :param GitHub connection: GitHub API object. If None, one will be created from the addon's user settings. """ connection = connection or GitHub.from_settings(addon.user_settings) if sha and not branch: raise HTTPError(http.BAD_REQUEST) # Get default branch if not provided if not branch: repo = connection.repo(addon.user, addon.repo) if repo is None: return None, None, None branch = repo.default_branch # Get registered branches if provided registered_branches = ( [Branch.from_json(b) for b in addon.registration_data.get('branches', [])] if addon.owner.is_registration else [] ) registered_branch_names = [ each.name for each in registered_branches ] # Fail if registered and branch not in registration data if registered_branches and branch not in registered_branch_names: raise HTTPError(http.BAD_REQUEST) # Get data from GitHub API if not registered branches = registered_branches or connection.branches(addon.user, addon.repo) # Use registered SHA if provided for each in branches: if branch == each.name: sha = each.commit.sha break return branch, sha, branches
def github_hgrid_data_contents(**kwargs): """Return a repo's file tree as a dict formatted for HGrid. """ auth = kwargs['auth'] node = kwargs['node'] or kwargs['project'] node_addon = kwargs['node_addon'] path = kwargs.get('path', '') connection = GitHub.from_settings(node_addon.user_settings) # The requested branch and sha req_branch, req_sha = request.args.get('branch'), request.args.get('sha') # The actual branch and sha to use, given the addon settings branch, sha, branches = get_refs(node_addon, req_branch, req_sha, connection=connection) # Get file tree try: contents = connection.contents( user=node_addon.user, repo=node_addon.repo, path=path, ref=sha or branch, ) except ApiError: raise HTTPError(http.NOT_FOUND) can_edit = check_permissions(node_addon, auth, connection, branch, sha) if contents: hgrid_tree = to_hgrid( contents, node_url=node.url, node_api_url=node.api_url, branch=branch, sha=sha, can_edit=can_edit, parent=path, max_size=node_addon.config.max_file_size, accepted_files=node_addon.config.accept_extensions) else: hgrid_tree = [] return hgrid_tree
def to_json(self, user): ret = super(AddonGitHubNodeSettings, self).to_json(user) user_settings = user.get_addon('github') ret.update({ 'user_has_auth': user_settings and user_settings.has_auth, 'is_registration': self.owner.is_registration, }) if self.user_settings and self.user_settings.has_auth: valid_credentials = False owner = self.user_settings.owner if user_settings and user_settings.owner == owner: connection = GitHub.from_settings(user_settings) # TODO: Fetch repo list client-side # Since /user/repos excludes organization repos to which the # current user has push access, we have to make extra requests to # find them valid_credentials = True try: repos = itertools.chain.from_iterable((connection.repos(), connection.my_org_repos())) repo_names = [ '{0} / {1}'.format(repo.owner.login, repo.name) for repo in repos ] except GitHubError as error: if error.code == http.UNAUTHORIZED: repo_names = [] valid_credentials = False ret.update({'repo_names': repo_names}) ret.update({ 'node_has_auth': True, 'github_user': self.user or '', 'github_repo': self.repo or '', 'github_repo_full_name': '{0} / {1}'.format(self.user, self.repo), 'auth_osf_name': owner.fullname, 'auth_osf_url': owner.url, 'auth_osf_id': owner._id, 'github_user_name': self.user_settings.github_user_name, 'github_user_url': 'https://github.com/{0}'.format(self.user_settings.github_user_name), 'is_owner': owner == user, 'valid_credentials': valid_credentials, 'addons_url': web_url_for('user_addons'), }) return ret
def get_refs(addon, branch=None, sha=None, connection=None): """Get the appropriate branch name and sha given the addon settings object, and optionally the branch and sha from the request arguments. :param str branch: Branch name. If None, return the default branch from the repo settings. :param str sha: The SHA. :param GitHub connection: GitHub API object. If None, one will be created from the addon's user settings. """ connection = connection or GitHub.from_settings(addon.user_settings) if sha and not branch: raise HTTPError(http.BAD_REQUEST) # Get default branch if not provided if not branch: repo = connection.repo(addon.user, addon.repo) if repo is None: return None, None, None branch = repo.default_branch # Get registered branches if provided registered_branches = ([ Branch.from_json(b) for b in addon.registration_data.get('branches', []) ] if addon.owner.is_registration else []) registered_branch_names = [each.name for each in registered_branches] # Fail if registered and branch not in registration data if registered_branches and branch not in registered_branch_names: raise HTTPError(http.BAD_REQUEST) # Get data from GitHub API if not registered branches = registered_branches or connection.branches( addon.user, addon.repo) # Use registered SHA if provided for each in branches: if branch == each.name: sha = each.commit.sha break return branch, sha, branches
def github_download_file(**kwargs): node_settings = kwargs['node_addon'] path = get_path(kwargs) ref = request.args.get('sha') connection = GitHub.from_settings(node_settings.user_settings) try: name, data, _ = connection.file(node_settings.user, node_settings.repo, path, ref=ref) except TooBigError: raise HTTPError( http.BAD_REQUEST, data={ 'message_short': 'File too large', 'message_long': 'This file is too large to download through ' 'the GitHub API.', }, ) if data is None: raise HTTPError(http.NOT_FOUND) # Build response resp = make_response(data) mimetype = get_mimetype(path, data) # Add binary MIME type if mimetype not found if mimetype is None: resp.headers['Content-Type'] = 'application/octet-stream' else: resp.headers['Content-Type'] = mimetype resp.headers['Content-Disposition'] = 'attachment; filename={0}'.format( name) return resp
def to_json(self, user): ret = super(AddonGitHubNodeSettings, self).to_json(user) user_settings = user.get_addon("github") ret.update( {"user_has_auth": user_settings and user_settings.has_auth, "is_registration": self.owner.is_registration} ) if self.user_settings and self.user_settings.has_auth: valid_credentials = False owner = self.user_settings.owner if user_settings and user_settings.owner == owner: connection = GitHub.from_settings(user_settings) # TODO: Fetch repo list client-side # Since /user/repos excludes organization repos to which the # current user has push access, we have to make extra requests to # find them valid_credentials = True try: repos = itertools.chain.from_iterable((connection.repos(), connection.my_org_repos())) repo_names = ["{0} / {1}".format(repo.owner.login, repo.name) for repo in repos] except GitHubError as error: if error.code == http.UNAUTHORIZED: repo_names = [] valid_credentials = False ret.update({"repo_names": repo_names}) ret.update( { "node_has_auth": True, "github_user": self.user or "", "github_repo": self.repo or "", "github_repo_full_name": "{0} / {1}".format(self.user, self.repo), "auth_osf_name": owner.fullname, "auth_osf_url": owner.url, "auth_osf_id": owner._id, "github_user_name": self.user_settings.github_user_name, "github_user_url": "https://github.com/{0}".format(self.user_settings.github_user_name), "is_owner": owner == user, "valid_credentials": valid_credentials, "addons_url": web_url_for("user_addons"), } ) return ret
def add_hook(self, save=True): if self.user_settings: connect = GitHub.from_settings(self.user_settings) secret = utils.make_hook_secret() hook = connect.add_hook( self.user, self.repo, "web", { "url": urlparse.urljoin(hook_domain, os.path.join(self.owner.api_url, "github", "hook/")), "content_type": github_settings.HOOK_CONTENT_TYPE, "secret": secret, }, ) if hook: self.hook_id = hook.id self.hook_secret = secret if save: self.save()
def after_set_privacy(self, node, permissions): """ :param Node node: :param str permissions: :return str: Alert message """ if not github_settings.SET_PRIVACY: return connect = GitHub.from_settings(self.user_settings) data = connect.set_privacy( self.user, self.repo, permissions == 'private' ) if data is None or 'errors' in data: repo = connect.repo(self.user, self.repo) if repo is not None: current_privacy = 'private' if repo.private else 'public' else: current_privacy = 'unknown' return ( 'Could not set privacy for repo {user}::{repo}. ' 'Current privacy status is {perm}.'.format( user=self.user, repo=self.repo, perm=current_privacy, ) ) return ( 'GitHub repo {user}::{repo} made {perm}.'.format( user=self.user, repo=self.repo, perm=permissions, ) )
def github_download_file(**kwargs): node_settings = kwargs['node_addon'] path = get_path(kwargs) ref = request.args.get('sha') connection = GitHub.from_settings(node_settings.user_settings) try: name, data, _ = connection.file( node_settings.user, node_settings.repo, path, ref=ref ) except TooBigError: raise HTTPError( http.BAD_REQUEST, data={ 'message_short': 'File too large', 'message_long': 'This file is too large to download through ' 'the GitHub API.', }, ) if data is None: raise HTTPError(http.NOT_FOUND) # Build response resp = make_response(data) mimetype = get_mimetype(path, data) # Add binary MIME type if mimetype not found if mimetype is None: resp.headers['Content-Type'] = 'application/octet-stream' else: resp.headers['Content-Type'] = mimetype resp.headers['Content-Disposition'] = 'attachment; filename={0}'.format( name) return resp
def github_hgrid_data_contents(**kwargs): """Return a repo's file tree as a dict formatted for HGrid. """ auth = kwargs['auth'] node = kwargs['node'] or kwargs['project'] node_addon = kwargs['node_addon'] path = kwargs.get('path', '') connection = GitHub.from_settings(node_addon.user_settings) # The requested branch and sha req_branch, req_sha = request.args.get('branch'), request.args.get('sha') # The actual branch and sha to use, given the addon settings branch, sha, branches = get_refs( node_addon, req_branch, req_sha, connection=connection ) # Get file tree try: contents = connection.contents( user=node_addon.user, repo=node_addon.repo, path=path, ref=sha or branch, ) except ApiError: raise HTTPError(http.NOT_FOUND) can_edit = check_permissions(node_addon, auth, connection, branch, sha) if contents: hgrid_tree = to_hgrid( contents, node_url=node.url, node_api_url=node.api_url, branch=branch, sha=sha, can_edit=can_edit, parent=path, max_size=node_addon.config.max_file_size, accepted_files=node_addon.config.accept_extensions ) else: hgrid_tree = [] return hgrid_tree
def is_private(self): connection = GitHub.from_settings(self.user_settings) return connection.repo(user=self.user, repo=self.repo).private
def github_hgrid_data(node_settings, auth, **kwargs): # Quit if no repo linked if not node_settings.complete: return connection = GitHub.from_settings(node_settings.user_settings) # Initialize repo here in the event that it is set in the privacy check # below. This potentially saves an API call in _check_permissions, below. repo = None # Quit if privacy mismatch and not contributor node = node_settings.owner if node.is_public and not node.is_contributor(auth.user): try: repo = connection.repo(node_settings.user, node_settings.repo) except NotFoundError: # TODO: Test me @jmcarp # TODO: Add warning message logger.error('Could not access GitHub repo') return None if repo.private: return None try: branch, sha, branches = get_refs( node_settings, branch=kwargs.get('branch'), sha=kwargs.get('sha'), connection=connection, ) except NotFoundError: # TODO: Show an alert or change GitHub configuration? logger.error('GitHub repo not found') return if branch is not None: ref = ref_to_params(branch, sha) can_edit = check_permissions( node_settings, auth, connection, branch, sha, repo=repo, ) else: ref = None can_edit = False name_tpl = '{user}/{repo}'.format(user=node_settings.user, repo=node_settings.repo) permissions = {'edit': can_edit, 'view': True} urls = { 'upload': node_settings.owner.api_url + 'github/file/' + (ref or ''), 'fetch': node_settings.owner.api_url + 'github/hgrid/' + (ref or ''), 'branch': node_settings.owner.api_url + 'github/hgrid/root/', 'zip': node_settings.owner.api_url + 'github/zipball/' + (ref or ''), 'repo': github_repo_url(owner=node_settings.user, repo=node_settings.repo, branch=branch) } return [ rubeus.build_addon_root( node_settings, name_tpl, urls=urls, permissions=permissions, branches=[each.name for each in branches], defaultBranch=branch, ) ]
def do_migration(records, dry=True): count, inval_cred_handled = 0, 0 for raw_user_settings in records: # False if missing, None if field exists access_token = raw_user_settings.get('oauth_access_token', False) token_type = raw_user_settings.get('oauth_token_type', False) github_user_name = raw_user_settings.get('github_user', False) if access_token and token_type and github_user_name: if not dry: gh = GitHub(access_token, token_type) try: github_user = gh.user() except github3.models.GitHubError: AddonGitHubUserSettings._storage[0].store.update( {'_id': raw_user_settings['_id']}, { '$unset': { "oauth_access_token" : True, "oauth_token_type" : True, "github_user" : True, }, } ) inval_cred_handled += 1 print('invalidated credentials handled record: {}'.format(raw_user_settings['_id'])) continue oauth_settings = AddonGitHubOauthSettings() oauth_settings.github_user_id = str(github_user.id) oauth_settings.save() oauth_settings.oauth_access_token = access_token oauth_settings.oauth_token_type = token_type oauth_settings.github_user_name = github_user_name oauth_settings.save() AddonGitHubUserSettings._storage[0].store.update( {'_id': raw_user_settings['_id']}, { '$unset': { 'oauth_access_token': True, 'oauth_token_type': True, 'github_user': True, }, '$set': { 'oauth_settings': oauth_settings.github_user_id, } } ) AddonGitHubOauthSettings._storage[0].store.update( {'github_user_id': oauth_settings.github_user_id}, { '$push': { '__backrefs.accessed.addongithubusersettings.oauth_settings': raw_user_settings['_id'], } } ) print('Finished migrating AddonGithubUserSettings record: {}'.format(raw_user_settings['_id'])) count += 1 # Old fields have not yet been unset elif None in set([access_token, token_type, github_user_name]): if not dry: AddonGitHubUserSettings._storage[0].store.update( {'_id': raw_user_settings['_id']}, { '$unset': { 'oauth_access_token': True, 'oauth_token_type': True, 'github_user': True, }, } ) print('Unset oauth_access_token and oauth_token_type: {0}'.format(raw_user_settings['_id'])) count += 1 return count, inval_cred_handled
def github_hgrid_data(node_settings, auth, **kwargs): # Quit if no repo linked if not node_settings.complete: return connection = GitHub.from_settings(node_settings.user_settings) # Initialize repo here in the event that it is set in the privacy check # below. This potentially saves an API call in _check_permissions, below. repo = None # Quit if privacy mismatch and not contributor node = node_settings.owner if node.is_public and not node.is_contributor(auth.user): try: repo = connection.repo(node_settings.user, node_settings.repo) except NotFoundError: # TODO: Test me @jmcarp # TODO: Add warning message logger.error('Could not access GitHub repo') return None if repo.private: return None try: branch, sha, branches = get_refs( node_settings, branch=kwargs.get('branch'), sha=kwargs.get('sha'), connection=connection, ) except NotFoundError: # TODO: Show an alert or change GitHub configuration? logger.error('GitHub repo not found') return if branch is not None: ref = ref_to_params(branch, sha) can_edit = check_permissions( node_settings, auth, connection, branch, sha, repo=repo, ) else: ref = None can_edit = False name_tpl = '{user}/{repo}'.format( user=node_settings.user, repo=node_settings.repo ) permissions = { 'edit': can_edit, 'view': True } urls = { 'upload': node_settings.owner.api_url + 'github/file/' + (ref or ''), 'fetch': node_settings.owner.api_url + 'github/hgrid/' + (ref or ''), 'branch': node_settings.owner.api_url + 'github/hgrid/root/', 'zip': node_settings.owner.api_url + 'github/zipball/' + (ref or ''), 'repo': github_repo_url(owner=node_settings.user, repo=node_settings.repo, branch=branch) } return [rubeus.build_addon_root( node_settings, name_tpl, urls=urls, permissions=permissions, branches=[each.name for each in branches], defaultBranch=branch, )]
def github_upload_file(auth, node_addon, **kwargs): node = kwargs['node'] or kwargs['project'] user = auth.user now = datetime.datetime.utcnow() path = get_path(kwargs, required=False) or '' branch = request.args.get('branch') sha = request.args.get('sha') if branch is None: raise HTTPError(http.BAD_REQUEST) connection = GitHub.from_settings(node_addon.user_settings) upload = request.files.get('file') filename = secure_filename(upload.filename) content = upload.read() # Check max file size upload.seek(0, os.SEEK_END) size = upload.tell() if size > node_addon.config.max_file_size * 1024 * 1024: raise HTTPError(http.BAD_REQUEST) # Get SHA of existing file if present; requires an additional call to the # GitHub API try: tree = connection.tree(node_addon.user, node_addon.repo, sha=sha or branch).tree except EmptyRepoError: tree = [] except NotFoundError: raise HTTPError(http.BAD_REQUEST) existing = [ thing for thing in tree if thing.path == os.path.join(path, filename) ] sha = existing[0].sha if existing else None author = { 'name': user.fullname, 'email': '{0}@osf.io'.format(user._id), } if existing: data = connection.update_file(node_addon.user, node_addon.repo, os.path.join(path, filename), MESSAGES['update'], content, sha=sha, branch=branch, author=author) else: data = connection.create_file(node_addon.user, node_addon.repo, os.path.join(path, filename), MESSAGES['add'], content, branch=branch, author=author) if data is not None: ref = ref_to_params(sha=data['commit'].sha) view_url = os.path.join(node.url, 'github', 'file', path, filename) + '/' + ref download_url = os.path.join(node.url, 'github', 'file', path, filename, 'download') + '/' + ref node.add_log( action=('github_' + (models.NodeLog.FILE_UPDATED if sha else models.NodeLog.FILE_ADDED)), params={ 'project': node.parent_id, 'node': node._primary_key, 'path': os.path.join(path, filename), 'urls': { 'view': view_url, 'download': download_url, }, 'github': { 'user': node_addon.user, 'repo': node_addon.repo, 'sha': data['commit'].sha, }, }, auth=auth, log_date=now, ) # Fail if file size is not provided; this happens when the file was # too large to upload to GitHub if data['content'].size is None: logger.error( 'Could not upload file {0} to GitHub: No size provided'.format( filename)) raise HTTPError(http.BAD_REQUEST) info = { 'addon': 'github', 'name': filename, 'size': [ data['content'].size, rubeus.format_filesize(data['content'].size), ], 'kind': 'file', 'urls': build_github_urls( data['content'], node.url, node.api_url, branch, sha, ), 'permissions': { 'view': True, 'edit': True, }, } return info, 201 raise HTTPError(http.BAD_REQUEST)
def do_migration(records, dry=True): count, inval_cred_handled = 0, 0 for raw_user_settings in records: # False if missing, None if field exists access_token = raw_user_settings.get('oauth_access_token', False) token_type = raw_user_settings.get('oauth_token_type', False) github_user_name = raw_user_settings.get('github_user', False) if access_token and token_type and github_user_name: if not dry: gh = GitHub(access_token, token_type) try: github_user = gh.user() except github3.models.GitHubError: AddonGitHubUserSettings._storage[0].store.update( {'_id': raw_user_settings['_id']}, { '$unset': { "oauth_access_token": True, "oauth_token_type": True, "github_user": True, }, }) inval_cred_handled += 1 print('invalidated credentials handled record: {}'.format( raw_user_settings['_id'])) continue oauth_settings = AddonGitHubOauthSettings() oauth_settings.github_user_id = str(github_user.id) oauth_settings.save() oauth_settings.oauth_access_token = access_token oauth_settings.oauth_token_type = token_type oauth_settings.github_user_name = github_user_name oauth_settings.save() AddonGitHubUserSettings._storage[0].store.update( {'_id': raw_user_settings['_id']}, { '$unset': { 'oauth_access_token': True, 'oauth_token_type': True, 'github_user': True, }, '$set': { 'oauth_settings': oauth_settings.github_user_id, } }) AddonGitHubOauthSettings._storage[0].store.update( {'github_user_id': oauth_settings.github_user_id}, { '$push': { '__backrefs.accessed.addongithubusersettings.oauth_settings': raw_user_settings['_id'], } }) print('Finished migrating AddonGithubUserSettings record: {}'. format(raw_user_settings['_id'])) count += 1 # Old fields have not yet been unset elif None in set([access_token, token_type, github_user_name]): if not dry: AddonGitHubUserSettings._storage[0].store.update( {'_id': raw_user_settings['_id']}, { '$unset': { 'oauth_access_token': True, 'oauth_token_type': True, 'github_user': True, }, }) print('Unset oauth_access_token and oauth_token_type: {0}'. format(raw_user_settings['_id'])) count += 1 return count, inval_cred_handled
def github_upload_file(auth, node_addon, **kwargs): node = kwargs['node'] or kwargs['project'] user = auth.user now = datetime.datetime.utcnow() path = get_path(kwargs, required=False) or '' branch = request.args.get('branch') sha = request.args.get('sha') if branch is None: raise HTTPError(http.BAD_REQUEST) connection = GitHub.from_settings(node_addon.user_settings) upload = request.files.get('file') filename = secure_filename(upload.filename) content = upload.read() # Check max file size upload.seek(0, os.SEEK_END) size = upload.tell() if size > node_addon.config.max_file_size * 1024 * 1024: raise HTTPError(http.BAD_REQUEST) # Get SHA of existing file if present; requires an additional call to the # GitHub API try: tree = connection.tree( node_addon.user, node_addon.repo, sha=sha or branch ).tree except EmptyRepoError: tree = [] except NotFoundError: raise HTTPError(http.BAD_REQUEST) existing = [ thing for thing in tree if thing.path == os.path.join(path, filename) ] sha = existing[0].sha if existing else None author = { 'name': user.fullname, 'email': '{0}@osf.io'.format(user._id), } if existing: data = connection.update_file( node_addon.user, node_addon.repo, os.path.join(path, filename), MESSAGES['update'], content, sha=sha, branch=branch, author=author ) else: data = connection.create_file( node_addon.user, node_addon.repo, os.path.join(path, filename), MESSAGES['add'], content, branch=branch, author=author ) if data is not None: ref = ref_to_params(sha=data['commit'].sha) view_url = os.path.join( node.url, 'github', 'file', path, filename ) + '/' + ref download_url = os.path.join( node.url, 'github', 'file', path, filename, 'download' ) + '/' + ref node.add_log( action=( 'github_' + ( models.NodeLog.FILE_UPDATED if sha else models.NodeLog.FILE_ADDED ) ), params={ 'project': node.parent_id, 'node': node._primary_key, 'path': os.path.join(path, filename), 'urls': { 'view': view_url, 'download': download_url, }, 'github': { 'user': node_addon.user, 'repo': node_addon.repo, 'sha': data['commit'].sha, }, }, auth=auth, log_date=now, ) # Fail if file size is not provided; this happens when the file was # too large to upload to GitHub if data['content'].size is None: logger.error( 'Could not upload file {0} to GitHub: No size provided'.format( filename ) ) raise HTTPError(http.BAD_REQUEST) info = { 'addon': 'github', 'name': filename, 'size': [ data['content'].size, rubeus.format_filesize(data['content'].size), ], 'kind': 'file', 'urls': build_github_urls( data['content'], node.url, node.api_url, branch, sha, ), 'permissions': { 'view': True, 'edit': True, }, } return info, 201 raise HTTPError(http.BAD_REQUEST)
def github_view_file(auth, **kwargs): node = kwargs['node'] or kwargs['project'] node_settings = kwargs['node_addon'] path = get_path(kwargs) file_name = os.path.split(path)[1] # Get branch / commit branch = request.args.get('branch') sha = request.args.get('sha', branch) ref = sha or branch connection = GitHub.from_settings(node_settings.user_settings) # Get current file for delete url current_file = connection.contents( user=node_settings.user, repo=node_settings.repo, path=path, ref=sha or branch) anonymous = has_anonymous_link(node, auth) try: # If GUID has already been created, we won't redirect, and can check # whether the file exists below guid = GithubGuidFile.find_one( Q('node', 'eq', node) & Q('path', 'eq', path) ) except ModularOdmException: # If GUID doesn't exist, check whether file exists before creating commits = connection.history( node_settings.user, node_settings.repo, path, ref, ) if not commits: raise HTTPError(http.NOT_FOUND) guid = GithubGuidFile( node=node, path=path, ) guid.save() redirect_url = check_file_guid(guid) if redirect_url: return redirect(redirect_url) # Get default branch if neither SHA nor branch is provided if ref is None: repo = connection.repo(node_settings.user, node_settings.repo) ref = branch = repo.default_branch # Get file history; use SHA or branch if registered, else branch start_sha = ref if node.is_registration else branch commits = connection.history( node_settings.user, node_settings.repo, path, sha=start_sha ) # Get current commit shas = [ commit['sha'] for commit in commits ] if not shas: raise HTTPError(http.NOT_FOUND) current_sha = sha if sha in shas else shas[0] # Get file URL download_url = '/' + guid._id + '/download/' + ref_to_params(branch, current_sha) render_url = os.path.join( node.api_url, 'github', 'file', path, 'render' ) + '/' + ref_to_params(branch, current_sha) delete_url = None if current_file: delete_url = node.api_url_for('github_delete_file', path=path) + ref_to_params(branch, current_file.sha) for commit in commits: commit['download'] = ( '/' + guid._id + '/download/' + ref_to_params(sha=commit['sha']) ) commit['view'] = ( '/' + guid._id + '/' + ref_to_params(branch, sha=commit['sha']) ) if anonymous: commit['name'] = 'A user' commit['email'] = '' # Get or create rendered file cache_file_name = get_cache_file( path, current_sha, ) rendered = get_cache_content(node_settings, cache_file_name) if rendered is None: try: _, data, size = connection.file( node_settings.user, node_settings.repo, path, ref=sha, ) except TooBigError: rendered = 'File too large to download.' if rendered is None: # Skip if too large to be rendered. if github_settings.MAX_RENDER_SIZE is not None and size > github_settings.MAX_RENDER_SIZE: rendered = 'File too large to render; download file to view it.' else: rendered = get_cache_content( node_settings, cache_file_name, start_render=True, remote_path=guid.path, file_content=data, download_url=download_url, ) rv = { 'node': { 'id': node._id, 'title': node.title }, 'file_name': file_name, 'files_page_url': node.web_url_for('collect_file_trees'), 'current_sha': current_sha, 'render_url': render_url, 'rendered': rendered, 'download_url': download_url, 'delete_url': delete_url, 'commits': commits, } rv.update(_view_project(node, auth, primary=True)) return rv
def github_view_file(auth, **kwargs): node = kwargs['node'] or kwargs['project'] node_settings = kwargs['node_addon'] path = get_path(kwargs) file_name = os.path.split(path)[1] # Get branch / commit branch = request.args.get('branch') sha = request.args.get('sha', branch) ref = sha or branch connection = GitHub.from_settings(node_settings.user_settings) # Get current file for delete url current_file = connection.contents(user=node_settings.user, repo=node_settings.repo, path=path, ref=sha or branch) anonymous = has_anonymous_link(node, auth) try: # If GUID has already been created, we won't redirect, and can check # whether the file exists below guid = GithubGuidFile.find_one( Q('node', 'eq', node) & Q('path', 'eq', path)) except ModularOdmException: # If GUID doesn't exist, check whether file exists before creating commits = connection.history( node_settings.user, node_settings.repo, path, ref, ) if not commits: raise HTTPError(http.NOT_FOUND) guid = GithubGuidFile( node=node, path=path, ) guid.save() redirect_url = check_file_guid(guid) if redirect_url: return redirect(redirect_url) # Get default branch if neither SHA nor branch is provided if ref is None: repo = connection.repo(node_settings.user, node_settings.repo) ref = branch = repo.default_branch # Get file history; use SHA or branch if registered, else branch start_sha = ref if node.is_registration else branch commits = connection.history(node_settings.user, node_settings.repo, path, sha=start_sha) # Get current commit shas = [commit['sha'] for commit in commits] if not shas: raise HTTPError(http.NOT_FOUND) current_sha = sha if sha in shas else shas[0] # Get file URL download_url = '/' + guid._id + '/download/' + ref_to_params( branch, current_sha) render_url = os.path.join(node.api_url, 'github', 'file', path, 'render') + '/' + ref_to_params( branch, current_sha) delete_url = None if current_file: delete_url = node.api_url_for('github_delete_file', path=path) + ref_to_params( branch, current_file.sha) for commit in commits: commit['download'] = ('/' + guid._id + '/download/' + ref_to_params(sha=commit['sha'])) commit['view'] = ('/' + guid._id + '/' + ref_to_params(branch, sha=commit['sha'])) if anonymous: commit['name'] = 'A user' commit['email'] = '' # Get or create rendered file cache_file_name = get_cache_file( path, current_sha, ) rendered = get_cache_content(node_settings, cache_file_name) if rendered is None: try: _, data, size = connection.file( node_settings.user, node_settings.repo, path, ref=sha, ) except TooBigError: rendered = 'File too large to download.' if rendered is None: # Skip if too large to be rendered. if github_settings.MAX_RENDER_SIZE is not None and size > github_settings.MAX_RENDER_SIZE: rendered = 'File too large to render; download file to view it.' else: rendered = get_cache_content( node_settings, cache_file_name, start_render=True, remote_path=guid.path, file_content=data, download_url=download_url, ) rv = { 'node': { 'id': node._id, 'title': node.title }, 'file_name': file_name, 'files_page_url': node.web_url_for('collect_file_trees'), 'current_sha': current_sha, 'render_url': render_url, 'rendered': rendered, 'download_url': download_url, 'delete_url': delete_url, 'commits': commits, } rv.update(_view_project(node, auth, primary=True)) return rv
def github_hgrid_data(node_settings, auth, **kwargs): # Quit if no repo linked if not node_settings.complete: return connection = GitHub.from_settings(node_settings.user_settings) # Initialize repo here in the event that it is set in the privacy check # below. This potentially saves an API call in _check_permissions, below. repo = None # Quit if privacy mismatch and not contributor node = node_settings.owner if node.is_public and not node.is_contributor(auth.user): try: repo = connection.repo(node_settings.user, node_settings.repo) except NotFoundError: # TODO: Test me @jmcarp # TODO: Add warning message logger.error("Could not access GitHub repo") return None if repo.private: return None try: branch, sha, branches = get_refs( node_settings, branch=kwargs.get("branch"), sha=kwargs.get("sha"), connection=connection ) except (NotFoundError, GitHubError): # TODO: Show an alert or change GitHub configuration? logger.error("GitHub repo not found") return if branch is not None: ref = ref_to_params(branch, sha) can_edit = check_permissions(node_settings, auth, connection, branch, sha, repo=repo) else: ref = None can_edit = False name_tpl = "{user}/{repo}".format(user=node_settings.user, repo=node_settings.repo) permissions = {"edit": can_edit, "view": True} urls = { "upload": node_settings.owner.api_url + "github/file/" + (ref or ""), "fetch": node_settings.owner.api_url + "github/hgrid/" + (ref or ""), "branch": node_settings.owner.api_url + "github/hgrid/root/", "zip": node_settings.owner.api_url + "github/zipball/" + (ref or ""), "repo": github_repo_url(owner=node_settings.user, repo=node_settings.repo, branch=branch), } branch_names = [each.name for each in branches] if not branch_names: branch_names = [branch] # if repo un-init-ed then still add default branch to list of branches return [ rubeus.build_addon_root( node_settings, name_tpl, urls=urls, permissions=permissions, branches=branch_names, defaultBranch=branch ) ]