def render_file(version_idx, file_version, file_record): """ :param int version_idx: One-based version index :param FileVersion file_version: File version to render :param FileRecord file_record: Base file object """ file_obj = model.OsfStorageGuidFile.find_one( Q('node', 'eq', file_record.node) & Q('path', 'eq', file_record.path) ) cache_file_name = get_cache_filename(file_version) node_settings = file_obj.node.get_addon('osfstorage') rendered = get_cache_content(node_settings, cache_file_name) if rendered is None: download_url = get_download_url(version_idx, file_version, file_record) file_response = requests.get(download_url) rendered = get_cache_content( node_settings, cache_file_name, start_render=True, remote_path=file_obj.path, file_content=file_response.content, download_url=file_obj.get_download_path(version_idx), ) return rendered
def dataverse_view_file(node_addon, auth, **kwargs): node = node_addon.owner file_id = kwargs.get('path') fail_if_unauthorized(node_addon, auth, file_id) fail_if_private(file_id) # lazily create a file GUID record file_obj, created = DataverseFile.get_or_create(node=node, path=file_id) redirect_url = check_file_guid(file_obj) if redirect_url: return redirect(redirect_url) # Get or create rendered file cache_file_name = '{0}.html'.format(file_id) rendered = get_cache_content(node_addon, cache_file_name) if rendered is None: filename, content = scrape_dataverse(file_id) _, ext = os.path.splitext(filename) download_url = node.api_url_for( 'dataverse_download_file_proxy', path=file_id ) rendered = get_cache_content( node_addon, cache_file_name, start_render=True, remote_path=file_obj.file_id + ext, # Include extension for MFR file_content=content, download_url=download_url, ) else: filename, _ = scrape_dataverse(file_id, name_only=True) render_url = node.api_url_for( 'dataverse_get_rendered_file', path=file_id, render=True ) ret = { 'file_name': filename, 'rendered': rendered, 'render_url': render_url, 'urls': { 'render': render_url, 'download': node.web_url_for('dataverse_download_file', path=file_id), 'info': node.api_url_for('dataverse_get_file_info', path=file_id), } } ret.update(_view_project(node, auth)) return ret
def osffiles_get_rendered_file(**kwargs): """ """ node_settings = kwargs['node_addon'] cache_file = get_cache_file(kwargs['fid'], kwargs['vid']) return get_cache_content(node_settings, cache_file)
def ping_render(**kwargs): node_settings = kwargs['node_addon'] path = kwargs.get('path') etag = request.args.get('etag') cache_file = get_cache_file_name(path, etag) return get_cache_content(node_settings, cache_file)
def figshare_get_rendered_file(*args, **kwargs): node_settings = kwargs['node_addon'] article_id = kwargs['aid'] file_id = kwargs['fid'] cache_file = get_cache_file(article_id, file_id) return get_cache_content(node_settings, cache_file)
def dataverse_get_rendered_file(**kwargs): """ """ node_settings = kwargs['node_addon'] file_id = kwargs['path'] cache_file = '{0}.html'.format(file_id) return get_cache_content(node_settings, cache_file)
def github_get_rendered_file(**kwargs): """ """ node_settings = kwargs['node_addon'] path = get_path(kwargs) sha = request.args.get('sha') cache_file = get_cache_file(path, sha) return get_cache_content(node_settings, cache_file)
def figshare_get_rendered_file(*args, **kwargs): node_settings = kwargs['node_addon'] article_id = kwargs['aid'] file_id = kwargs['fid'] cache_file = get_cache_file( article_id, file_id ) return get_cache_content(node_settings, cache_file)
def figshare_view_file(*args, **kwargs): auth = kwargs['auth'] node = kwargs['node'] or kwargs['project'] node_settings = kwargs['node_addon'] article_id = kwargs.get('aid') or None file_id = kwargs.get('fid') or None anonymous = has_anonymous_link(node, auth) if not article_id or not file_id: raise HTTPError(http.NOT_FOUND) connect = Figshare.from_settings(node_settings.user_settings) if node_settings.figshare_type == 'project': item = connect.project(node_settings, node_settings.figshare_id) else: item = connect.article(node_settings, node_settings.figshare_id) if article_id not in str(item): raise HTTPError(http.NOT_FOUND) article = connect.article(node_settings, article_id) found = False for f in article['items'][0]['files']: if f['id'] == int(file_id): found = f break if not found: raise HTTPError(http.NOT_FOUND) try: # If GUID has already been created, we won't redirect, and can check # whether the file exists below guid = FigShareGuidFile.find_one( Q('node', 'eq', node) & Q('article_id', 'eq', article_id) & Q('file_id', 'eq', file_id)) except: guid = FigShareGuidFile(node=node, article_id=article_id, file_id=file_id) guid.save() redirect_url = check_file_guid(guid) if redirect_url: return redirect(redirect_url) private = not (article['items'][0]['status'] == 'Public') figshare_url = 'http://figshare.com/' if private: figshare_url += 'preview/_preview/{0}'.format( article['items'][0]['article_id']) else: figshare_url += 'articles/{0}/{1}'.format( article['items'][0]['title'].replace(' ', '_'), article['items'][0]['article_id']) version_url = "http://figshare.com/articles/{filename}/{file_id}".format( filename=article['items'][0]['title'], file_id=article['items'][0]['article_id']) download_url = node.api_url + 'figshare/download/article/{aid}/file/{fid}'.format( aid=article_id, fid=file_id) render_url = node.api_url + \ 'figshare/render/article/{aid}/file/{fid}'.format(aid=article_id, fid=file_id) delete_url = node.api_url + 'figshare/article/{aid}/file/{fid}/'.format( aid=article_id, fid=file_id) filename = found['name'] cache_file_name = get_cache_file(article_id, file_id) rendered = get_cache_content(node_settings, cache_file_name) if private: rendered = messages.FIGSHARE_VIEW_FILE_PRIVATE.format( url='http://figshare.com/') elif rendered is None: filename, size, filedata = connect.get_file(node_settings, found) if figshare_settings.MAX_RENDER_SIZE is not None and size > figshare_settings.MAX_RENDER_SIZE: rendered = messages.FIGSHARE_VIEW_FILE_OVERSIZED.format( url=found.get('download_url')) else: rendered = get_cache_content( node_settings, cache_file_name, start_render=True, remote_path=filename, file_content=filedata, download_url=download_url, ) # categories = connect.categories()['items'] # TODO Cache this # categories = ''.join( # ["<option value='{val}'>{label}</option>".format(val=i['id'], label=i['name']) for i in categories]) rv = { 'node': { 'id': node._id, 'title': node.title }, 'file_name': filename, 'rendered': rendered, 'file_status': article['items'][0]['status'], 'file_version': article['items'][0]['version'], 'doi': 'http://dx.doi.org/10.6084/m9.figshare.{0}'.format( article['items'][0]['article_id']), 'parent_type': 'fileset' if article['items'][0]['defined_type'] == 'fileset' else 'singlefile', 'parent_id': article['items'][0]['article_id'], # 'figshare_categories': categories, 'figshare_title': article['items'][0]['title'], 'figshare_desc': article['items'][0]['description'], 'render_url': render_url, 'urls': { 'render': render_url, 'download': found.get('download_url'), 'version': version_url, 'figshare': privacy_info_handle(figshare_url, anonymous), 'delete': delete_url, 'files': node.web_url_for('collect_file_trees') } } rv.update(_view_project(node, auth, primary=True)) return rv
def s3_view(**kwargs): path = kwargs.get('path') vid = request.args.get('vid') if not path: raise HTTPError(http.NOT_FOUND) if vid == 'Pre-versioning': vid = 'null' node_settings = kwargs['node_addon'] auth = kwargs['auth'] node = kwargs['node'] or kwargs['project'] wrapper = S3Wrapper.from_addon(node_settings) key = wrapper.get_wrapped_key(urllib.unquote(path), vid=vid) if key is None: raise HTTPError(http.NOT_FOUND) try: guid = S3GuidFile.find_one( Q('node', 'eq', node) & Q('path', 'eq', path) ) except: guid = S3GuidFile( node=node, path=path, ) guid.save() redirect_url = check_file_guid(guid) if redirect_url: return redirect(redirect_url) cache_file_name = get_cache_file_name(path, key.etag) urls = build_urls(node, path, etag=key.etag) if key.s3Key.size > MAX_RENDER_SIZE: render = 'File too large to render; download file to view it' else: # Check to see if the file has already been rendered. render = get_cache_content(node_settings, cache_file_name) if render is None: file_contents = key.s3Key.get_contents_as_string() render = get_cache_content( node_settings, cache_file_name, start_render=True, file_content=file_contents, download_url=urls['download'], ) versions = create_version_list(wrapper, urllib.unquote(path), node) rv = { 'file_name': key.name, 'rendered': render, 'download_url': urls['download'], 'render_url': urls['render'], 'versions': versions, 'current': key.version_id, 'info_url': urls['info'], 'delete_url': urls['delete'], 'files_page_url': node.web_url_for('collect_file_trees') } rv.update(_view_project(node, auth, primary=True)) return rv
def github_view_file(auth, **kwargs): node = kwargs['node'] or kwargs['project'] node_settings = kwargs['node_addon'] path = get_path(kwargs) file_name = os.path.split(path)[1] # Get branch / commit branch = request.args.get('branch') sha = request.args.get('sha', branch) ref = sha or branch connection = GitHub.from_settings(node_settings.user_settings) # Get current file for delete url current_file = connection.contents(user=node_settings.user, repo=node_settings.repo, path=path, ref=sha or branch) anonymous = has_anonymous_link(node, auth) try: # If GUID has already been created, we won't redirect, and can check # whether the file exists below guid = GithubGuidFile.find_one( Q('node', 'eq', node) & Q('path', 'eq', path)) except ModularOdmException: # If GUID doesn't exist, check whether file exists before creating commits = connection.history( node_settings.user, node_settings.repo, path, ref, ) if not commits: raise HTTPError(http.NOT_FOUND) guid = GithubGuidFile( node=node, path=path, ) guid.save() redirect_url = check_file_guid(guid) if redirect_url: return redirect(redirect_url) # Get default branch if neither SHA nor branch is provided if ref is None: repo = connection.repo(node_settings.user, node_settings.repo) ref = branch = repo.default_branch # Get file history; use SHA or branch if registered, else branch start_sha = ref if node.is_registration else branch commits = connection.history(node_settings.user, node_settings.repo, path, sha=start_sha) # Get current commit shas = [commit['sha'] for commit in commits] if not shas: raise HTTPError(http.NOT_FOUND) current_sha = sha if sha in shas else shas[0] # Get file URL download_url = '/' + guid._id + '/download/' + ref_to_params( branch, current_sha) render_url = os.path.join(node.api_url, 'github', 'file', path, 'render') + '/' + ref_to_params( branch, current_sha) delete_url = None if current_file: delete_url = node.api_url_for('github_delete_file', path=path) + ref_to_params( branch, current_file.sha) for commit in commits: commit['download'] = ('/' + guid._id + '/download/' + ref_to_params(sha=commit['sha'])) commit['view'] = ('/' + guid._id + '/' + ref_to_params(branch, sha=commit['sha'])) if anonymous: commit['name'] = 'A user' commit['email'] = '' # Get or create rendered file cache_file_name = get_cache_file( path, current_sha, ) rendered = get_cache_content(node_settings, cache_file_name) if rendered is None: try: _, data, size = connection.file( node_settings.user, node_settings.repo, path, ref=sha, ) except TooBigError: rendered = 'File too large to download.' if rendered is None: # Skip if too large to be rendered. if github_settings.MAX_RENDER_SIZE is not None and size > github_settings.MAX_RENDER_SIZE: rendered = 'File too large to render; download file to view it.' else: rendered = get_cache_content( node_settings, cache_file_name, start_render=True, remote_path=guid.path, file_content=data, download_url=download_url, ) rv = { 'node': { 'id': node._id, 'title': node.title }, 'file_name': file_name, 'files_page_url': node.web_url_for('collect_file_trees'), 'current_sha': current_sha, 'render_url': render_url, 'rendered': rendered, 'download_url': download_url, 'delete_url': delete_url, 'commits': commits, } rv.update(_view_project(node, auth, primary=True)) return rv
def github_view_file(auth, **kwargs): node = kwargs['node'] or kwargs['project'] node_settings = kwargs['node_addon'] path = get_path(kwargs) file_name = os.path.split(path)[1] # Get branch / commit branch = request.args.get('branch') sha = request.args.get('sha', branch) ref = sha or branch connection = GitHub.from_settings(node_settings.user_settings) # Get current file for delete url current_file = connection.contents( user=node_settings.user, repo=node_settings.repo, path=path, ref=sha or branch) anonymous = has_anonymous_link(node, auth) try: # If GUID has already been created, we won't redirect, and can check # whether the file exists below guid = GithubGuidFile.find_one( Q('node', 'eq', node) & Q('path', 'eq', path) ) except ModularOdmException: # If GUID doesn't exist, check whether file exists before creating commits = connection.history( node_settings.user, node_settings.repo, path, ref, ) if not commits: raise HTTPError(http.NOT_FOUND) guid = GithubGuidFile( node=node, path=path, ) guid.save() redirect_url = check_file_guid(guid) if redirect_url: return redirect(redirect_url) # Get default branch if neither SHA nor branch is provided if ref is None: repo = connection.repo(node_settings.user, node_settings.repo) ref = branch = repo.default_branch # Get file history; use SHA or branch if registered, else branch start_sha = ref if node.is_registration else branch commits = connection.history( node_settings.user, node_settings.repo, path, sha=start_sha ) # Get current commit shas = [ commit['sha'] for commit in commits ] if not shas: raise HTTPError(http.NOT_FOUND) current_sha = sha if sha in shas else shas[0] # Get file URL download_url = '/' + guid._id + '/download/' + ref_to_params(branch, current_sha) render_url = os.path.join( node.api_url, 'github', 'file', path, 'render' ) + '/' + ref_to_params(branch, current_sha) delete_url = None if current_file: delete_url = node.api_url_for('github_delete_file', path=path) + ref_to_params(branch, current_file.sha) for commit in commits: commit['download'] = ( '/' + guid._id + '/download/' + ref_to_params(sha=commit['sha']) ) commit['view'] = ( '/' + guid._id + '/' + ref_to_params(branch, sha=commit['sha']) ) if anonymous: commit['name'] = 'A user' commit['email'] = '' # Get or create rendered file cache_file_name = get_cache_file( path, current_sha, ) rendered = get_cache_content(node_settings, cache_file_name) if rendered is None: try: _, data, size = connection.file( node_settings.user, node_settings.repo, path, ref=sha, ) except TooBigError: rendered = 'File too large to download.' if rendered is None: # Skip if too large to be rendered. if github_settings.MAX_RENDER_SIZE is not None and size > github_settings.MAX_RENDER_SIZE: rendered = 'File too large to render; download file to view it.' else: rendered = get_cache_content( node_settings, cache_file_name, start_render=True, remote_path=guid.path, file_content=data, download_url=download_url, ) rv = { 'node': { 'id': node._id, 'title': node.title }, 'file_name': file_name, 'files_page_url': node.web_url_for('collect_file_trees'), 'current_sha': current_sha, 'render_url': render_url, 'rendered': rendered, 'download_url': download_url, 'delete_url': delete_url, 'commits': commits, } rv.update(_view_project(node, auth, primary=True)) return rv
def s3_view(**kwargs): path = kwargs.get('path') vid = request.args.get('vid') if not path: raise HTTPError(http.NOT_FOUND) if vid == 'Pre-versioning': vid = 'null' node_settings = kwargs['node_addon'] auth = kwargs['auth'] node = kwargs['node'] or kwargs['project'] wrapper = S3Wrapper.from_addon(node_settings) key = wrapper.get_wrapped_key(urllib.unquote(path), vid=vid) if key is None: raise HTTPError(http.NOT_FOUND) try: guid = S3GuidFile.find_one( Q('node', 'eq', node) & Q('path', 'eq', path)) except: guid = S3GuidFile( node=node, path=path, ) guid.save() redirect_url = check_file_guid(guid) if redirect_url: return redirect(redirect_url) cache_file_name = get_cache_file_name(path, key.etag) urls = build_urls(node, path, etag=key.etag) if key.s3Key.size > MAX_RENDER_SIZE: render = 'File too large to render; download file to view it' else: # Check to see if the file has already been rendered. render = get_cache_content(node_settings, cache_file_name) if render is None: file_contents = key.s3Key.get_contents_as_string() render = get_cache_content( node_settings, cache_file_name, start_render=True, remote_path=path, file_content=file_contents, download_url=urls['download'], ) versions = create_version_list(wrapper, urllib.unquote(path), node) rv = { 'file_name': key.name, 'rendered': render, 'download_url': urls['download'], 'render_url': urls['render'], 'versions': versions, 'current': key.version_id, 'info_url': urls['info'], 'delete_url': urls['delete'], 'files_page_url': node.web_url_for('collect_file_trees') } rv.update(_view_project(node, auth, primary=True)) return rv
def figshare_view_file(*args, **kwargs): auth = kwargs['auth'] node = kwargs['node'] or kwargs['project'] node_settings = kwargs['node_addon'] article_id = kwargs.get('aid') or None file_id = kwargs.get('fid') or None anonymous = has_anonymous_link(node, auth) if not article_id or not file_id: raise HTTPError(http.NOT_FOUND) connect = Figshare.from_settings(node_settings.user_settings) if node_settings.figshare_type == 'project': item = connect.project(node_settings, node_settings.figshare_id) else: item = connect.article(node_settings, node_settings.figshare_id) if article_id not in str(item): raise HTTPError(http.NOT_FOUND) article = connect.article(node_settings, article_id) found = False for f in article['items'][0]['files']: if f['id'] == int(file_id): found = f break if not found: raise HTTPError(http.NOT_FOUND) try: # If GUID has already been created, we won't redirect, and can check # whether the file exists below guid = FigShareGuidFile.find_one( Q('node', 'eq', node) & Q('article_id', 'eq', article_id) & Q('file_id', 'eq', file_id) ) except: guid = FigShareGuidFile(node=node, article_id=article_id, file_id=file_id) guid.save() redirect_url = check_file_guid(guid) if redirect_url: return redirect(redirect_url) private = not(article['items'][0]['status'] == 'Public') figshare_url = 'http://figshare.com/' if private: figshare_url += 'preview/_preview/{0}'.format(article['items'][0]['article_id']) else: figshare_url += 'articles/{0}/{1}'.format(article['items'][0]['title'].replace(' ', '_'), article['items'][0]['article_id']) version_url = "http://figshare.com/articles/{filename}/{file_id}".format( filename=article['items'][0]['title'], file_id=article['items'][0]['article_id']) download_url = node.api_url + 'figshare/download/article/{aid}/file/{fid}'.format(aid=article_id, fid=file_id) render_url = node.api_url + \ 'figshare/render/article/{aid}/file/{fid}'.format(aid=article_id, fid=file_id) delete_url = node.api_url + 'figshare/article/{aid}/file/{fid}/'.format(aid=article_id, fid=file_id) filename = found['name'] cache_file_name = get_cache_file( article_id, file_id ) rendered = get_cache_content(node_settings, cache_file_name) if private: rendered = messages.FIGSHARE_VIEW_FILE_PRIVATE.format(url='http://figshare.com/') elif rendered is None: filename, size, filedata = connect.get_file(node_settings, found) if figshare_settings.MAX_RENDER_SIZE is not None and size > figshare_settings.MAX_RENDER_SIZE: rendered = messages.FIGSHARE_VIEW_FILE_OVERSIZED.format( url=found.get('download_url')) else: rendered = get_cache_content( node_settings, cache_file_name, start_render=True, remote_path=filename, file_content=filedata, download_url=download_url, ) categories = connect.categories()['items'] # TODO Cache this categories = ''.join( ["<option value='{val}'>{label}</option>".format(val=i['id'], label=i['name']) for i in categories]) rv = { 'node': { 'id': node._id, 'title': node.title }, 'file_name': filename, 'rendered': rendered, 'file_status': article['items'][0]['status'], 'file_version': article['items'][0]['version'], 'doi': 'http://dx.doi.org/10.6084/m9.figshare.{0}'.format(article['items'][0]['article_id']), 'parent_type': 'fileset' if article['items'][0]['defined_type'] == 'fileset' else 'singlefile', 'parent_id': article['items'][0]['article_id'], 'figshare_categories': categories, 'figshare_title': article['items'][0]['title'], 'figshare_desc': article['items'][0]['description'], 'urls': { 'render': render_url, 'download': found.get('download_url'), 'version': version_url, 'figshare': privacy_info_handle(figshare_url, anonymous), 'delete': delete_url, 'files': node.web_url_for('collect_file_trees') } } rv.update(_view_project(node, auth, primary=True)) return rv
def view_file(auth, **kwargs): node_settings = kwargs['node_addon'] node = kwargs['node'] or kwargs['project'] file_name = kwargs['fid'] file_name_clean = file_name.replace('.', '_') try: guid = OsfGuidFile.find_one( Q('node', 'eq', node) & Q('name', 'eq', file_name) ) except: guid = OsfGuidFile( node=node, name=file_name, ) guid.save() redirect_url = check_file_guid(guid) if redirect_url: return redirect(redirect_url) # Throw 404 and log error if file not found in files_versions try: file_id = node.files_versions[file_name_clean][-1] except KeyError: logger.error('File {} not found in files_versions of component {}.'.format( file_name_clean, node._id )) raise HTTPError(http.NOT_FOUND) file_object = NodeFile.load(file_id) # Ensure NodeFile is attached to Node; should be fixed by actions or # improved data modeling in future if not file_object.node: file_object.node = node file_object.save() download_url = file_object.download_url(node) render_url = file_object.render_url(node) info_url = file_object.info_url(node) file_path = os.path.join( settings.UPLOADS_PATH, node._primary_key, file_name ) # Throw 404 and log error if file not found on disk if not os.path.isfile(file_path): logger.error('File {} not found on disk.'.format(file_path)) raise HTTPError(http.NOT_FOUND) _, file_ext = os.path.splitext(file_path.lower()) # Get or create rendered file cache_file = get_cache_file( file_object.filename, file_object.latest_version_number(node) ) rendered = get_cache_content( node_settings, cache_file, start_render=True, file_path=file_path, file_content=None, download_path=download_url, ) rv = { 'file_name': file_name, 'render_url': render_url, 'rendered': rendered, 'info_url': info_url, } rv.update(_view_project(node, auth)) return rv
def view_file(auth, **kwargs): node_settings = kwargs['node_addon'] node = kwargs['node'] or kwargs['project'] file_name = kwargs['fid'] file_name_clean = file_name.replace('.', '_') try: guid = OsfGuidFile.find_one( Q('node', 'eq', node) & Q('name', 'eq', file_name) ) except: guid = OsfGuidFile( node=node, name=file_name, ) guid.save() redirect_url = check_file_guid(guid) if redirect_url: return redirect(redirect_url) # Throw 404 and log error if file not found in files_versions try: file_id = node.files_versions[file_name_clean][-1] except KeyError: logger.error('File {} not found in files_versions of component {}.'.format( file_name_clean, node._id )) raise HTTPError(http.NOT_FOUND) file_object = NodeFile.load(file_id) # Ensure NodeFile is attached to Node; should be fixed by actions or # improved data modeling in future if not file_object.node: file_object.node = node file_object.save() download_url = file_object.download_url(node) render_url = file_object.render_url(node) info_url = file_object.info_url(node) file_path = os.path.join( settings.UPLOADS_PATH, node._primary_key, file_name ) # Throw 404 and log error if file not found on disk if not os.path.isfile(file_path): logger.error('File {} not found on disk.'.format(file_path)) raise HTTPError(http.NOT_FOUND) _, file_ext = os.path.splitext(file_path.lower()) # Get or create rendered file cache_file = get_cache_file( file_object.filename, file_object.latest_version_number(node) ) rendered = get_cache_content( node_settings, cache_file, start_render=True, file_path=file_path, file_content=None, download_path=download_url, ) ret = { 'file_name': file_name, 'render_url': render_url, 'rendered': rendered, 'info_url': info_url, } ret.update(_view_project(node, auth)) return ret