def do_view(path, source_name=None): source = matching_source(source_name) match = cache.check_path(source, path) if match is None: lookup = source.lookup(path) if lookup is None: raise CellxgeneException( f"Could not find item for path {path} in source {source.name}", 404, ) key = CacheKey.for_lookup(source, lookup) print( f"view path={path}, source_name={source_name}, dataset={key.file_path}, annotation_file= {key.annotation_file_path}, key={key.descriptor}, source={key.source_name}" ) with entry_lock: match = cache.check_entry(key) if match is None: uascripts = get_extra_scripts() match = cache.create_entry(key, uascripts) match.timestamp = current_time_stamp() if (match.status == CacheEntryStatus.loaded or match.status == CacheEntryStatus.loading): if source.is_authorized(match.key.descriptor): return match.serve_content(path) else: raise CellxgeneException("User not authorized to access this data", 403) elif match.status == CacheEntryStatus.error: raise ProcessException.from_cache_entry(match)
def create_dir(parent_path, dir_name): full_path = os.path.join(parent_path, dir_name) if "/" in dir_name: raise CellxgeneException( "Please have no slashes in the intended directory.", status.HTTP_400_BAD_REQUEST, ) elif not os.path.exists(parent_path): raise CellxgeneException( "The selected User directory does not exist.", status.HTTP_400_BAD_REQUEST, ) elif os.path.exists(full_path): raise CellxgeneException( "The provided subdirectory already exists within Directory.", status.HTTP_400_BAD_REQUEST, ) elif not is_subdir(full_path, parent_path): raise CellxgeneException( "The directory must be a subdirectory of the parent path.", status.HTTP_400_BAD_REQUEST, ) elif not os.path.isdir(parent_path): raise CellxgeneException( "The parent is not a directory.", status.HTTP_400_BAD_REQUEST ) else: os.mkdir(full_path)
def upload_file(): upload_dir = request.form["path"] full_upload_path = os.path.join(env.cellxgene_data, upload_dir) if is_subdir(full_upload_path, env.cellxgene_data) and os.path.isdir(full_upload_path): if request.method == "POST": if "file" in request.files: f = request.files["file"] if f and f.filename.endswith(".h5ad"): f.save( os.path.join(full_upload_path, secure_filename(f.filename))) return redirect("/filecrawl.html", code=302) else: raise CellxgeneException( "Uploaded file must be in anndata (.h5ad) format.", status.HTTP_400_BAD_REQUEST, ) else: raise CellxgeneException( "A file must be chosen to upload.", status.HTTP_400_BAD_REQUEST, ) else: raise CellxgeneException("Invalid directory.", status.HTTP_400_BAD_REQUEST) return redirect(location, code=302)
def validate_is_dir(file_path): validate_exists(file_path) if not os.path.isdir(file_path): raise CellxgeneException( "Path is not dir: " + file_path, status.HTTP_400_BAD_REQUEST ) return
def get_key(path): if path == "/" or path == "": raise CellxgeneException( "No matching dataset found.", status.HTTP_404_NOT_FOUND ) trimmed = path[:-1] if path[-1] == "/" else path try: # valid paths come in three forms: if trimmed.endswith('.h5ad') and data_file_exists(trimmed): # 1) somedir/dataset.h5ad: a dataset return CacheKey(trimmed, trimmed, None) elif trimmed.endswith('.csv'): # 2) somedir/dataset_annotations/saldaal1-T5HMVBNV.csv : an actual annotations file. annotations_dir = os.path.split(trimmed)[0] dataset = make_h5ad(annotations_dir) if data_file_exists(dataset): data_dir_ensure(annotations_dir) return CacheKey(trimmed, dataset, trimmed) elif trimmed.endswith('_annotations') and data_dir_exists(trimmed): # 3) somedir/dataset_annotations: an annotation directory. The corresponding h5ad must exist, but the directory may not. dataset = make_h5ad(trimmed) if data_file_exists(dataset): return CacheKey(trimmed, dataset, '') except CellxgeneException: pass split = os.path.split(trimmed) return get_key(split[0])
def do_filecrawl(path): filecrawl_path = os.path.join(env.cellxgene_data, path) if not os.path.isdir(filecrawl_path): raise CellxgeneException("Path is not directory: " + filecrawl_path, status.HTTP_400_BAD_REQUEST) entries = recurse_dir(filecrawl_path) rendered_html = render_entries(entries) return render_template( "filecrawl.html", extra_scripts=get_extra_scripts(), rendered_html=rendered_html, path=path, )
def check_entry(self, key): contents = self.entry_list matches = [ c for c in contents if c.key.equals(key) and c.status != CacheEntryStatus.terminated ] if len(matches) == 0: return None elif len(matches) == 1: return matches[0] else: raise CellxgeneException( status.HTTP_500_INTERNAL_SERVER_ERROR, "Found " + str(len(matches)) + " for " + key.dataset, )
def check_path(self, source, path): contents = self.entry_list matches = [ c for c in contents if c.key.source.name == source.name and path.startswith( c.key.descriptor) and c.status != CacheEntryStatus.terminated ] if len(matches) == 0: return None elif len(matches) == 1: return matches[0] else: raise CellxgeneException( status.HTTP_500_INTERNAL_SERVER_ERROR, "Found " + str(len(matches)) + " for " + path, )
def validate_exists(file_path): if not os.path.exists(file_path): raise CellxgeneException( "File does not exist: " + file_path, status.HTTP_400_BAD_REQUEST )
def serve_content(self, path): gateway_basepath = self.key.gateway_basepath() subpath = path[len(self.key.descriptor):] # noqa: E203 if len(subpath) == 0: r = make_response(f"Redirect to {gateway_basepath}\n", 302) r.headers["location"] = gateway_basepath + querystring() return r elif self.status == CacheEntryStatus.loading: launch_time = datetime.datetime.fromtimestamp(self.launchtime) return render_template( "loading.html", launchtime=launch_time, all_output=self.all_output, ) headers = {} copy_headers = [ "accept", "accept-encoding", "accept-language", "cache-control", "connection", "content-length", "content-type", "cookie", "host", "origin", "pragma", "referer", "sec-fetch-mode", "sec-fetch-site", "user-agent", ] for h in copy_headers: if h in request.headers: headers[h] = request.headers[h] full_path = self.cellxgene_basepath() + subpath + querystring() if request.method in ["GET", "HEAD", "OPTIONS"]: cellxgene_response = get(full_path, headers=headers) elif request.method == "PUT": cellxgene_response = put( full_path, headers=headers, data=request.data, ) elif request.method == "POST": cellxgene_response = post( full_path, headers=headers, data=request.data, ) else: raise CellxgeneException(f"Unexpected method {request.method}", 400) content_type = cellxgene_response.headers["content-type"] if "text" in content_type: gateway_content = self.rewrite_text_content( cellxgene_response.content.decode()) else: gateway_content = cellxgene_response.content resp_headers = {} for h in copy_headers: if h in cellxgene_response.headers: resp_headers[h] = cellxgene_response.headers[h] gateway_response = make_response( gateway_content, cellxgene_response.status_code, resp_headers, ) return gateway_response
def serve_content(self, path): gateway_basepath = ( f"{env.external_protocol}://{env.external_host}/view/{self.key.pathpart}/" ) subpath = path[len(self.key.pathpart) :] # noqa: E203 if len(subpath) == 0: r = make_response(f"Redirect to {gateway_basepath}\n", 301) r.headers["location"] = gateway_basepath+querystring() return r elif self.status == "loading": launch_time = datetime.datetime.fromtimestamp(self.launchtime) return render_template( "loading.html", launchtime=launch_time, all_output=self.all_output ) port = self.port cellxgene_basepath = f"http://127.0.0.1:{port}" headers = {} copy_headers = [ 'accept', 'accept-encoding', 'accept-language', 'cache-control', 'connection', 'content-length', 'content-type', 'cookie', 'host', 'origin', 'pragma', 'referer', 'sec-fetch-mode', 'sec-fetch-site', 'user-agent' ] for h in copy_headers: if h in request.headers: headers[h] = request.headers[h] full_path = cellxgene_basepath + subpath + querystring() if request.method in ["GET", "HEAD", "OPTIONS"]: cellxgene_response = get( full_path, headers=headers ) elif request.method == "PUT": cellxgene_response = put( full_path, headers=headers, data=request.data, ) elif request.method == "POST": cellxgene_response = post( full_path, headers=headers, data=request.data, ) else: raise CellxgeneException( f"Unexpected method {request.method}", 400 ) content_type = cellxgene_response.headers["content-type"] if "text" in content_type: cellxgene_content = cellxgene_response.content.decode() gateway_content = cellxgene_content.replace( "http://fonts.gstatic.com", "https://fonts.gstatic.com" ).replace(cellxgene_basepath, gateway_basepath) else: gateway_content = cellxgene_response.content resp_headers = {} for h in copy_headers: if h in cellxgene_response.headers: resp_headers[h] = cellxgene_response.headers[h] gateway_response = make_response( gateway_content, cellxgene_response.status_code, resp_headers, ) return gateway_response