def github_hgrid_data(node_settings, auth, **kwargs): # Quit if no repo linked if not node_settings.complete: return connection = GitHubClient(external_account=node_settings.external_account) # Initialize repo here in the event that it is set in the privacy check # below. This potentially saves an API call in _check_permissions, below. repo = None # Quit if privacy mismatch and not contributor node = node_settings.owner if node.is_public and not node.is_contributor(auth.user): try: repo = connection.repo(node_settings.user, node_settings.repo) except NotFoundError: # TODO: Test me @jmcarp # TODO: Add warning message logger.error('Could not access GitHub repo') return None if repo.private: return None try: branch, sha, branches = get_refs( node_settings, branch=kwargs.get('branch'), sha=kwargs.get('sha'), connection=connection, ) except (NotFoundError, GitHubError): # TODO: Show an alert or change GitHub configuration? logger.error('GitHub repo not found') return if branch is not None: ref = ref_to_params(branch, sha) can_edit = check_permissions( node_settings, auth, connection, branch, sha, repo=repo, ) else: ref = None can_edit = False name_tpl = '{user}/{repo}'.format( user=node_settings.user, repo=node_settings.repo ) permissions = { 'edit': can_edit, 'view': True, 'private': node_settings.is_private } urls = { 'upload': node_settings.owner.api_url + 'github/file/' + (ref or ''), 'fetch': node_settings.owner.api_url + 'github/hgrid/' + (ref or ''), 'branch': node_settings.owner.api_url + 'github/hgrid/root/', 'zip': node_settings.owner.api_url + 'github/zipball/' + (ref or ''), 'repo': "https://github.com/{0}/{1}/tree/{2}".format(node_settings.user, node_settings.repo, branch) } branch_names = [each.name for each in branches] if not branch_names: branch_names = [branch] # if repo un-init-ed then still add default branch to list of branches return [rubeus.build_addon_root( node_settings, name_tpl, urls=urls, permissions=permissions, branches=branch_names, defaultBranch=branch, )]
def github_view_file(auth, **kwargs): node = kwargs['node'] or kwargs['project'] node_settings = kwargs['node_addon'] path = get_path(kwargs) file_name = os.path.split(path)[1] # Get branch / commit branch = request.args.get('branch') sha = request.args.get('sha', branch) ref = sha or branch connection = GitHub.from_settings(node_settings.user_settings) # Get current file for delete url current_file = connection.contents(user=node_settings.user, repo=node_settings.repo, path=path, ref=sha or branch) anonymous = has_anonymous_link(node, auth) try: # If GUID has already been created, we won't redirect, and can check # whether the file exists below guid = GithubGuidFile.find_one( Q('node', 'eq', node) & Q('path', 'eq', path)) except ModularOdmException: # If GUID doesn't exist, check whether file exists before creating commits = connection.history( node_settings.user, node_settings.repo, path, ref, ) if not commits: raise HTTPError(http.NOT_FOUND) guid = GithubGuidFile( node=node, path=path, ) guid.save() redirect_url = check_file_guid(guid) if redirect_url: return redirect(redirect_url) # Get default branch if neither SHA nor branch is provided if ref is None: repo = connection.repo(node_settings.user, node_settings.repo) ref = branch = repo.default_branch # Get file history; use SHA or branch if registered, else branch start_sha = ref if node.is_registration else branch commits = connection.history(node_settings.user, node_settings.repo, path, sha=start_sha) # Get current commit shas = [commit['sha'] for commit in commits] if not shas: raise HTTPError(http.NOT_FOUND) current_sha = sha if sha in shas else shas[0] # Get file URL download_url = '/' + guid._id + '/download/' + ref_to_params( branch, current_sha) render_url = os.path.join(node.api_url, 'github', 'file', path, 'render') + '/' + ref_to_params( branch, current_sha) delete_url = None if current_file: delete_url = node.api_url_for('github_delete_file', path=path) + ref_to_params( branch, current_file.sha) for commit in commits: commit['download'] = ('/' + guid._id + '/download/' + ref_to_params(sha=commit['sha'])) commit['view'] = ('/' + guid._id + '/' + ref_to_params(branch, sha=commit['sha'])) if anonymous: commit['name'] = 'A user' commit['email'] = '' # Get or create rendered file cache_file_name = get_cache_file( path, current_sha, ) rendered = get_cache_content(node_settings, cache_file_name) if rendered is None: try: _, data, size = connection.file( node_settings.user, node_settings.repo, path, ref=sha, ) except TooBigError: rendered = 'File too large to download.' if rendered is None: # Skip if too large to be rendered. if github_settings.MAX_RENDER_SIZE is not None and size > github_settings.MAX_RENDER_SIZE: rendered = 'File too large to render; download file to view it.' else: rendered = get_cache_content( node_settings, cache_file_name, start_render=True, remote_path=guid.path, file_content=data, download_url=download_url, ) rv = { 'node': { 'id': node._id, 'title': node.title }, 'file_name': file_name, 'files_page_url': node.web_url_for('collect_file_trees'), 'current_sha': current_sha, 'render_url': render_url, 'rendered': rendered, 'download_url': download_url, 'delete_url': delete_url, 'commits': commits, } rv.update(_view_project(node, auth, primary=True)) return rv
def github_hgrid_data(node_settings, auth, **kwargs): # Quit if no repo linked if not node_settings.complete: return connection = GitHub.from_settings(node_settings.user_settings) # Initialize repo here in the event that it is set in the privacy check # below. This potentially saves an API call in _check_permissions, below. repo = None # Quit if privacy mismatch and not contributor node = node_settings.owner if node.is_public and not node.is_contributor(auth.user): try: repo = connection.repo(node_settings.user, node_settings.repo) except NotFoundError: # TODO: Test me @jmcarp # TODO: Add warning message logger.error('Could not access GitHub repo') return None if repo.private: return None try: branch, sha, branches = get_refs( node_settings, branch=kwargs.get('branch'), sha=kwargs.get('sha'), connection=connection, ) except NotFoundError: # TODO: Show an alert or change GitHub configuration? logger.error('GitHub repo not found') return if branch is not None: ref = ref_to_params(branch, sha) can_edit = check_permissions( node_settings, auth, connection, branch, sha, repo=repo, ) else: ref = None can_edit = False name_tpl = '{user}/{repo}'.format(user=node_settings.user, repo=node_settings.repo) permissions = {'edit': can_edit, 'view': True} urls = { 'upload': node_settings.owner.api_url + 'github/file/' + (ref or ''), 'fetch': node_settings.owner.api_url + 'github/hgrid/' + (ref or ''), 'branch': node_settings.owner.api_url + 'github/hgrid/root/', 'zip': node_settings.owner.api_url + 'github/zipball/' + (ref or ''), 'repo': github_repo_url(owner=node_settings.user, repo=node_settings.repo, branch=branch) } return [ rubeus.build_addon_root( node_settings, name_tpl, urls=urls, permissions=permissions, branches=[each.name for each in branches], defaultBranch=branch, ) ]
def github_upload_file(auth, node_addon, **kwargs): node = kwargs['node'] or kwargs['project'] user = auth.user now = datetime.datetime.utcnow() path = get_path(kwargs, required=False) or '' branch = request.args.get('branch') sha = request.args.get('sha') if branch is None: raise HTTPError(http.BAD_REQUEST) connection = GitHub.from_settings(node_addon.user_settings) upload = request.files.get('file') filename = secure_filename(upload.filename) content = upload.read() # Check max file size upload.seek(0, os.SEEK_END) size = upload.tell() if size > node_addon.config.max_file_size * 1024 * 1024: raise HTTPError(http.BAD_REQUEST) # Get SHA of existing file if present; requires an additional call to the # GitHub API try: tree = connection.tree(node_addon.user, node_addon.repo, sha=sha or branch).tree except EmptyRepoError: tree = [] except NotFoundError: raise HTTPError(http.BAD_REQUEST) existing = [ thing for thing in tree if thing.path == os.path.join(path, filename) ] sha = existing[0].sha if existing else None author = { 'name': user.fullname, 'email': '{0}@osf.io'.format(user._id), } if existing: data = connection.update_file(node_addon.user, node_addon.repo, os.path.join(path, filename), MESSAGES['update'], content, sha=sha, branch=branch, author=author) else: data = connection.create_file(node_addon.user, node_addon.repo, os.path.join(path, filename), MESSAGES['add'], content, branch=branch, author=author) if data is not None: ref = ref_to_params(sha=data['commit'].sha) view_url = os.path.join(node.url, 'github', 'file', path, filename) + '/' + ref download_url = os.path.join(node.url, 'github', 'file', path, filename, 'download') + '/' + ref node.add_log( action=('github_' + (models.NodeLog.FILE_UPDATED if sha else models.NodeLog.FILE_ADDED)), params={ 'project': node.parent_id, 'node': node._primary_key, 'path': os.path.join(path, filename), 'urls': { 'view': view_url, 'download': download_url, }, 'github': { 'user': node_addon.user, 'repo': node_addon.repo, 'sha': data['commit'].sha, }, }, auth=auth, log_date=now, ) # Fail if file size is not provided; this happens when the file was # too large to upload to GitHub if data['content'].size is None: logger.error( 'Could not upload file {0} to GitHub: No size provided'.format( filename)) raise HTTPError(http.BAD_REQUEST) info = { 'addon': 'github', 'name': filename, 'size': [ data['content'].size, rubeus.format_filesize(data['content'].size), ], 'kind': 'file', 'urls': build_github_urls( data['content'], node.url, node.api_url, branch, sha, ), 'permissions': { 'view': True, 'edit': True, }, } return info, 201 raise HTTPError(http.BAD_REQUEST)
def github_hgrid_data(node_settings, auth, **kwargs): # Quit if no repo linked if not node_settings.complete: return connection = GitHub.from_settings(node_settings.user_settings) # Initialize repo here in the event that it is set in the privacy check # below. This potentially saves an API call in _check_permissions, below. repo = None # Quit if privacy mismatch and not contributor node = node_settings.owner if node.is_public and not node.is_contributor(auth.user): try: repo = connection.repo(node_settings.user, node_settings.repo) except NotFoundError: # TODO: Test me @jmcarp # TODO: Add warning message logger.error("Could not access GitHub repo") return None if repo.private: return None try: branch, sha, branches = get_refs( node_settings, branch=kwargs.get("branch"), sha=kwargs.get("sha"), connection=connection ) except (NotFoundError, GitHubError): # TODO: Show an alert or change GitHub configuration? logger.error("GitHub repo not found") return if branch is not None: ref = ref_to_params(branch, sha) can_edit = check_permissions(node_settings, auth, connection, branch, sha, repo=repo) else: ref = None can_edit = False name_tpl = "{user}/{repo}".format(user=node_settings.user, repo=node_settings.repo) permissions = {"edit": can_edit, "view": True} urls = { "upload": node_settings.owner.api_url + "github/file/" + (ref or ""), "fetch": node_settings.owner.api_url + "github/hgrid/" + (ref or ""), "branch": node_settings.owner.api_url + "github/hgrid/root/", "zip": node_settings.owner.api_url + "github/zipball/" + (ref or ""), "repo": github_repo_url(owner=node_settings.user, repo=node_settings.repo, branch=branch), } branch_names = [each.name for each in branches] if not branch_names: branch_names = [branch] # if repo un-init-ed then still add default branch to list of branches return [ rubeus.build_addon_root( node_settings, name_tpl, urls=urls, permissions=permissions, branches=branch_names, defaultBranch=branch ) ]
def github_view_file(auth, **kwargs): node = kwargs['node'] or kwargs['project'] node_settings = kwargs['node_addon'] path = get_path(kwargs) file_name = os.path.split(path)[1] # Get branch / commit branch = request.args.get('branch') sha = request.args.get('sha', branch) ref = sha or branch connection = GitHub.from_settings(node_settings.user_settings) # Get current file for delete url current_file = connection.contents( user=node_settings.user, repo=node_settings.repo, path=path, ref=sha or branch) anonymous = has_anonymous_link(node, auth) try: # If GUID has already been created, we won't redirect, and can check # whether the file exists below guid = GithubGuidFile.find_one( Q('node', 'eq', node) & Q('path', 'eq', path) ) except ModularOdmException: # If GUID doesn't exist, check whether file exists before creating commits = connection.history( node_settings.user, node_settings.repo, path, ref, ) if not commits: raise HTTPError(http.NOT_FOUND) guid = GithubGuidFile( node=node, path=path, ) guid.save() redirect_url = check_file_guid(guid) if redirect_url: return redirect(redirect_url) # Get default branch if neither SHA nor branch is provided if ref is None: repo = connection.repo(node_settings.user, node_settings.repo) ref = branch = repo.default_branch # Get file history; use SHA or branch if registered, else branch start_sha = ref if node.is_registration else branch commits = connection.history( node_settings.user, node_settings.repo, path, sha=start_sha ) # Get current commit shas = [ commit['sha'] for commit in commits ] if not shas: raise HTTPError(http.NOT_FOUND) current_sha = sha if sha in shas else shas[0] # Get file URL download_url = '/' + guid._id + '/download/' + ref_to_params(branch, current_sha) render_url = os.path.join( node.api_url, 'github', 'file', path, 'render' ) + '/' + ref_to_params(branch, current_sha) delete_url = None if current_file: delete_url = node.api_url_for('github_delete_file', path=path) + ref_to_params(branch, current_file.sha) for commit in commits: commit['download'] = ( '/' + guid._id + '/download/' + ref_to_params(sha=commit['sha']) ) commit['view'] = ( '/' + guid._id + '/' + ref_to_params(branch, sha=commit['sha']) ) if anonymous: commit['name'] = 'A user' commit['email'] = '' # Get or create rendered file cache_file_name = get_cache_file( path, current_sha, ) rendered = get_cache_content(node_settings, cache_file_name) if rendered is None: try: _, data, size = connection.file( node_settings.user, node_settings.repo, path, ref=sha, ) except TooBigError: rendered = 'File too large to download.' if rendered is None: # Skip if too large to be rendered. if github_settings.MAX_RENDER_SIZE is not None and size > github_settings.MAX_RENDER_SIZE: rendered = 'File too large to render; download file to view it.' else: rendered = get_cache_content( node_settings, cache_file_name, start_render=True, remote_path=guid.path, file_content=data, download_url=download_url, ) rv = { 'node': { 'id': node._id, 'title': node.title }, 'file_name': file_name, 'files_page_url': node.web_url_for('collect_file_trees'), 'current_sha': current_sha, 'render_url': render_url, 'rendered': rendered, 'download_url': download_url, 'delete_url': delete_url, 'commits': commits, } rv.update(_view_project(node, auth, primary=True)) return rv
def github_upload_file(auth, node_addon, **kwargs): node = kwargs['node'] or kwargs['project'] user = auth.user now = datetime.datetime.utcnow() path = get_path(kwargs, required=False) or '' branch = request.args.get('branch') sha = request.args.get('sha') if branch is None: raise HTTPError(http.BAD_REQUEST) connection = GitHub.from_settings(node_addon.user_settings) upload = request.files.get('file') filename = secure_filename(upload.filename) content = upload.read() # Check max file size upload.seek(0, os.SEEK_END) size = upload.tell() if size > node_addon.config.max_file_size * 1024 * 1024: raise HTTPError(http.BAD_REQUEST) # Get SHA of existing file if present; requires an additional call to the # GitHub API try: tree = connection.tree( node_addon.user, node_addon.repo, sha=sha or branch ).tree except EmptyRepoError: tree = [] except NotFoundError: raise HTTPError(http.BAD_REQUEST) existing = [ thing for thing in tree if thing.path == os.path.join(path, filename) ] sha = existing[0].sha if existing else None author = { 'name': user.fullname, 'email': '{0}@osf.io'.format(user._id), } if existing: data = connection.update_file( node_addon.user, node_addon.repo, os.path.join(path, filename), MESSAGES['update'], content, sha=sha, branch=branch, author=author ) else: data = connection.create_file( node_addon.user, node_addon.repo, os.path.join(path, filename), MESSAGES['add'], content, branch=branch, author=author ) if data is not None: ref = ref_to_params(sha=data['commit'].sha) view_url = os.path.join( node.url, 'github', 'file', path, filename ) + '/' + ref download_url = os.path.join( node.url, 'github', 'file', path, filename, 'download' ) + '/' + ref node.add_log( action=( 'github_' + ( models.NodeLog.FILE_UPDATED if sha else models.NodeLog.FILE_ADDED ) ), params={ 'project': node.parent_id, 'node': node._primary_key, 'path': os.path.join(path, filename), 'urls': { 'view': view_url, 'download': download_url, }, 'github': { 'user': node_addon.user, 'repo': node_addon.repo, 'sha': data['commit'].sha, }, }, auth=auth, log_date=now, ) # Fail if file size is not provided; this happens when the file was # too large to upload to GitHub if data['content'].size is None: logger.error( 'Could not upload file {0} to GitHub: No size provided'.format( filename ) ) raise HTTPError(http.BAD_REQUEST) info = { 'addon': 'github', 'name': filename, 'size': [ data['content'].size, rubeus.format_filesize(data['content'].size), ], 'kind': 'file', 'urls': build_github_urls( data['content'], node.url, node.api_url, branch, sha, ), 'permissions': { 'view': True, 'edit': True, }, } return info, 201 raise HTTPError(http.BAD_REQUEST)
def github_hgrid_data(node_settings, auth, **kwargs): # Quit if no repo linked if not node_settings.complete: return connection = GitHub.from_settings(node_settings.user_settings) # Initialize repo here in the event that it is set in the privacy check # below. This potentially saves an API call in _check_permissions, below. repo = None # Quit if privacy mismatch and not contributor node = node_settings.owner if node.is_public and not node.is_contributor(auth.user): try: repo = connection.repo(node_settings.user, node_settings.repo) except NotFoundError: # TODO: Test me @jmcarp # TODO: Add warning message logger.error('Could not access GitHub repo') return None if repo.private: return None try: branch, sha, branches = get_refs( node_settings, branch=kwargs.get('branch'), sha=kwargs.get('sha'), connection=connection, ) except NotFoundError: # TODO: Show an alert or change GitHub configuration? logger.error('GitHub repo not found') return if branch is not None: ref = ref_to_params(branch, sha) can_edit = check_permissions( node_settings, auth, connection, branch, sha, repo=repo, ) name_append = github_branch_widget(branches, owner=node_settings.user, repo=node_settings.repo, branch=branch, sha=sha) else: ref = None can_edit = False name_append = None name_tpl = '{user}/{repo}'.format( user=node_settings.user, repo=node_settings.repo ) permissions = { 'edit': can_edit, 'view': True } urls = { 'upload': node_settings.owner.api_url + 'github/file/' + (ref or ''), 'fetch': node_settings.owner.api_url + 'github/hgrid/' + (ref or ''), 'branch': node_settings.owner.api_url + 'github/hgrid/root/', 'zip': node_settings.owner.api_url + 'github/zipball/' + (ref or ''), 'repo': github_repo_url(owner=node_settings.user, repo=node_settings.repo, branch=branch) } buttons = [ rubeus.build_addon_button('<i class="icon-download-alt"></i>', 'githubDownloadZip', "Download Zip"), rubeus.build_addon_button('<i class="icon-external-link"></i>', 'githubVisitRepo', "Visit Repository"), ] return [rubeus.build_addon_root( node_settings, name_tpl, urls=urls, permissions=permissions, extra=name_append, buttons=buttons, )]