def test_is_hidden_win32_pypy(): import ctypes with tempfile.TemporaryDirectory() as root: subdir1 = os.path.join(root, 'subdir') os.makedirs(subdir1) assert not is_hidden(subdir1, root) subprocess.check_call(["attrib", "+h", subdir1]) with warnings.catch_warnings(record=True) as w: # Cause all warnings to always be triggered. warnings.simplefilter("always") # Trigger a warning. assert not is_hidden(subdir1, root) # Verify the warning was triggered assert len(w) == 1 assert issubclass(w[-1].category, UserWarning) assert "hidden files are not detectable on this system" in str(w[-1].message) with warnings.catch_warnings(record=True) as w: # Cause all warnings to always be triggered. warnings.simplefilter("always") # Trigger a warning. assert not is_file_hidden(subdir1) # Verify the warning was triggered assert len(w) == 1 assert issubclass(w[-1].category, UserWarning) assert "hidden files are not detectable on this system" in str(w[-1].message)
def rename_file(self, old_path, new_path): """Rename a file.""" old_path = old_path.strip("/") new_path = new_path.strip("/") if new_path == old_path: return new_os_path = self._get_os_path(new_path) old_os_path = self._get_os_path(old_path) if (is_hidden(old_os_path, self.root_dir) or is_hidden( new_os_path, self.root_dir)) and not self.allow_hidden: raise web.HTTPError( 400, f"Cannot rename file or directory {old_os_path!r}") # Should we proceed with the move? if os.path.exists(new_os_path) and not samefile( old_os_path, new_os_path): raise web.HTTPError(409, "File already exists: %s" % new_path) # Move the file try: with self.perm_to_403(): shutil.move(old_os_path, new_os_path) except web.HTTPError: raise except Exception as e: raise web.HTTPError( 500, f"Unknown error renaming file: {old_path} {e}") from e
def test_is_hidden_win32_cpython(): import ctypes with tempfile.TemporaryDirectory() as root: subdir1 = os.path.join(root, 'subdir') os.makedirs(subdir1) assert not is_hidden(subdir1, root) subprocess.check_call(["attrib", "+h", subdir1]) assert is_hidden(subdir1, root) assert is_file_hidden(subdir1)
def test_is_hidden_win32(): import ctypes with tempfile.TemporaryDirectory() as root: subdir1 = os.path.join(root, 'subdir') os.makedirs(subdir1) assert not is_hidden(subdir1, root) r = ctypes.windll.kernel32.SetFileAttributesW(subdir1, 0x02) print(r) # Helps debugging assert is_hidden(subdir1, root) assert is_file_hidden(subdir1)
async def _dir_model(self, path, content=True): """Build a model for a directory if content is requested, will include a listing of the directory """ os_path = self._get_os_path(path) four_o_four = u'directory does not exist: %r' % path if not os.path.isdir(os_path): raise web.HTTPError(404, four_o_four) elif is_hidden(os_path, self.root_dir) and not self.allow_hidden: self.log.info( "Refusing to serve hidden directory %r, via 404 Error", os_path) raise web.HTTPError(404, four_o_four) model = self._base_model(path) model['type'] = 'directory' model['size'] = None if content: model['content'] = contents = [] os_dir = self._get_os_path(path) dir_contents = await run_sync_in_worker_thread(os.listdir, os_dir) for name in dir_contents: try: os_path = os.path.join(os_dir, name) except UnicodeDecodeError as e: self.log.warning("failed to decode filename '%s': %s", name, e) continue try: st = await run_sync_in_worker_thread(os.lstat, os_path) except OSError as e: # skip over broken symlinks in listing if e.errno == errno.ENOENT: self.log.warning("%s doesn't exist", os_path) else: self.log.warning("Error stat-ing %s: %s", os_path, e) continue if (not stat.S_ISLNK(st.st_mode) and not stat.S_ISREG(st.st_mode) and not stat.S_ISDIR(st.st_mode)): self.log.debug("%s not a regular file", os_path) continue if self.should_list(name): if self.allow_hidden or not is_file_hidden(os_path, stat_res=st): contents.append(await self.get(path='%s/%s' % (path, name), content=False)) model['format'] = 'json' return model
def test_is_hidden(): with TemporaryDirectory() as root: subdir1 = os.path.join(root, 'subdir') os.makedirs(subdir1) assert not is_hidden(subdir1, root) assert not is_file_hidden(subdir1) subdir2 = os.path.join(root, '.subdir2') os.makedirs(subdir2) assert is_hidden(subdir2, root) assert is_file_hidden(subdir2) # root dir is always visible assert not is_hidden(subdir2, subdir2) subdir34 = os.path.join(root, 'subdir3', '.subdir4') os.makedirs(subdir34) assert is_hidden(subdir34, root) assert is_hidden(subdir34) subdir56 = os.path.join(root, '.subdir5', 'subdir6') os.makedirs(subdir56) assert is_hidden(subdir56, root) assert is_hidden(subdir56) assert not is_file_hidden(subdir56) assert not is_file_hidden(subdir56, os.stat(subdir56))
async def _save_directory(self, os_path, model, path=''): """create a directory""" if is_hidden(os_path, self.root_dir) and not self.allow_hidden: raise web.HTTPError(400, u'Cannot create hidden directory %r' % os_path) if not os.path.exists(os_path): with self.perm_to_403(): await run_sync(os.mkdir, os_path) elif not os.path.isdir(os_path): raise web.HTTPError(400, u'Not a directory: %s' % (os_path)) else: self.log.debug("Directory %r already exists", os_path)
def _save_directory(self, os_path, model, path=""): """create a directory""" if is_hidden(os_path, self.root_dir) and not self.allow_hidden: raise web.HTTPError(400, "Cannot create directory %r" % os_path) if not os.path.exists(os_path): with self.perm_to_403(): os.mkdir(os_path) elif not os.path.isdir(os_path): raise web.HTTPError(400, "Not a directory: %s" % (os_path)) else: self.log.debug("Directory %r already exists", os_path)
def get(self, path, content=True, type=None, format=None): """Takes a path for an entity and returns its model Parameters ---------- path : str the API path that describes the relative path for the target content : bool Whether to include the contents in the reply type : str, optional The requested type - 'file', 'notebook', or 'directory'. Will raise HTTPError 400 if the content doesn't match. format : str, optional The requested format for file contents. 'text' or 'base64'. Ignored if this returns a notebook or directory model. Returns ------- model : dict the contents model. If content=True, returns the contents of the file or directory as well. """ path = path.strip("/") os_path = self._get_os_path(path) four_o_four = "file or directory does not exist: %r" % path if not self.exists(path): raise web.HTTPError(404, four_o_four) if is_hidden(os_path, self.root_dir) and not self.allow_hidden: self.log.info( "Refusing to serve hidden file or directory %r, via 404 Error", os_path) raise web.HTTPError(404, four_o_four) if os.path.isdir(os_path): if type not in (None, "directory"): raise web.HTTPError( 400, f"{path} is a directory, not a {type}", reason="bad type", ) model = self._dir_model(path, content=content) elif type == "notebook" or (type is None and path.endswith(".ipynb")): model = self._notebook_model(path, content=content) else: if type == "directory": raise web.HTTPError(400, "%s is not a directory" % path, reason="bad type") model = self._file_model(path, content=content, format=format) return model
def validate_absolute_path(self, root, absolute_path): """Validate and return the absolute path. Requires tornado 3.1 Adding to tornado's own handling, forbids the serving of hidden files. """ abs_path = super(AuthenticatedFileHandler, self).validate_absolute_path(root, absolute_path) abs_root = os.path.abspath(root) if is_hidden(abs_path, abs_root) and not self.contents_manager.allow_hidden: self.log.info("Refusing to serve hidden file, via 404 Error, use flag 'ContentsManager.allow_hidden' to enable") raise web.HTTPError(404) return abs_path
def _base_model(self, path): """Build the common base of a contents model""" os_path = self._get_os_path(path) info = os.lstat(os_path) four_o_four = "file or directory does not exist: %r" % path if is_hidden(os_path, self.root_dir) and not self.allow_hidden: self.log.info( "Refusing to serve hidden file or directory %r, via 404 Error", os_path) raise web.HTTPError(404, four_o_four) try: # size of file size = info.st_size except (ValueError, OSError): self.log.warning("Unable to get size.") size = None try: last_modified = tz.utcfromtimestamp(info.st_mtime) except (ValueError, OSError): # Files can rarely have an invalid timestamp # https://github.com/jupyter/notebook/issues/2539 # https://github.com/jupyter/notebook/issues/2757 # Use the Unix epoch as a fallback so we don't crash. self.log.warning("Invalid mtime %s for %s", info.st_mtime, os_path) last_modified = datetime(1970, 1, 1, 0, 0, tzinfo=tz.UTC) try: created = tz.utcfromtimestamp(info.st_ctime) except (ValueError, OSError): # See above self.log.warning("Invalid ctime %s for %s", info.st_ctime, os_path) created = datetime(1970, 1, 1, 0, 0, tzinfo=tz.UTC) # Create the base model. model = {} model["name"] = path.rsplit("/", 1)[-1] model["path"] = path model["last_modified"] = last_modified model["created"] = created model["content"] = None model["format"] = None model["mimetype"] = None model["size"] = size model["writable"] = self.is_writable(path) return model
def is_hidden(self, path): """Does the API style path correspond to a hidden directory or file? Parameters ---------- path : string The path to check. This is an API path (`/` separated, relative to root_dir). Returns ------- hidden : bool Whether the path exists and is hidden. """ path = path.strip('/') os_path = self._get_os_path(path=path) return is_hidden(os_path, self.root_dir)
def _dir_model(self, path, content=True): """Build a model for a directory if content is requested, will include a listing of the directory """ os_path = self._get_os_path(path) four_o_four = u'directory does not exist: %r' % path if not os.path.isdir(os_path): raise web.HTTPError(404, four_o_four) elif is_hidden(os_path, self.root_dir) and not self.allow_hidden: self.log.info("Refusing to serve hidden directory %r, via 404 Error", os_path ) raise web.HTTPError(404, four_o_four) model = self._base_model(path) model['type'] = 'directory' model['size'] = None if content: model['content'] = contents = [] os_dir = self._get_os_path(path) for name in os.listdir(os_dir): try: os_path = os.path.join(os_dir, name) except UnicodeDecodeError as e: self.log.warning( "failed to decode filename '%s': %s", name, e) continue try: st = os.lstat(os_path) except OSError as e: # skip over broken symlinks in listing if e.errno == errno.ENOENT: self.log.warning("%s doesn't exist", os_path) elif e.errno != errno.EACCES: # Don't provide clues about protected files self.log.warning("Error stat-ing %s: %s", os_path, e) continue if (not stat.S_ISLNK(st.st_mode) and not stat.S_ISREG(st.st_mode) and not stat.S_ISDIR(st.st_mode)): self.log.debug("%s not a regular file", os_path) continue try: if self.should_list(name): if self.allow_hidden or not is_file_hidden(os_path, stat_res=st): contents.append( self.get(path='%s/%s' % (path, name), content=False) ) except OSError as e: # ELOOP: recursive symlink, also don't show failure due to permissions if e.errno not in [errno.ELOOP, errno.EACCES]: self.log.warning( "Unknown error checking if file %r is hidden", os_path, exc_info=True, ) model['format'] = 'json' return model
async def is_hidden(self, path): """Is path a hidden directory or file""" path = path.strip("/") os_path = self._get_os_path(path=path) return is_hidden(os_path, self.root_dir)
def delete_file(self, path): """Delete file at path.""" path = path.strip("/") os_path = self._get_os_path(path) rm = os.unlink four_o_four = "file or directory does not exist: %r" % path if not self.exists(path): raise web.HTTPError(404, four_o_four) if is_hidden(os_path, self.root_dir) and not self.allow_hidden: raise web.HTTPError( 400, f"Cannot delete file or directory {os_path!r}") def _check_trash(os_path): if sys.platform in {"win32", "darwin"}: return True # It's a bit more nuanced than this, but until we can better # distinguish errors from send2trash, assume that we can only trash # files on the same partition as the home directory. file_dev = os.stat(os_path).st_dev home_dev = os.stat(os.path.expanduser("~")).st_dev return file_dev == home_dev def is_non_empty_dir(os_path): if os.path.isdir(os_path): # A directory containing only leftover checkpoints is # considered empty. cp_dir = getattr(self.checkpoints, "checkpoint_dir", None) if set(os.listdir(os_path)) - {cp_dir}: return True return False if self.delete_to_trash: if not self.always_delete_dir and sys.platform == "win32" and is_non_empty_dir( os_path): # send2trash can really delete files on Windows, so disallow # deleting non-empty files. See Github issue 3631. raise web.HTTPError(400, "Directory %s not empty" % os_path) if _check_trash(os_path): # Looking at the code in send2trash, I don't think the errors it # raises let us distinguish permission errors from other errors in # code. So for now, the "look before you leap" approach is used. if not self.is_writable(path): raise web.HTTPError(403, "Permission denied: %s" % path) self.log.debug("Sending %s to trash", os_path) send2trash(os_path) return else: self.log.warning( "Skipping trash for %s, on different device to home directory", os_path, ) if os.path.isdir(os_path): # Don't permanently delete non-empty directories. if not self.always_delete_dir and is_non_empty_dir(os_path): raise web.HTTPError(400, "Directory %s not empty" % os_path) self.log.debug("Removing directory %s", os_path) with self.perm_to_403(): shutil.rmtree(os_path) else: self.log.debug("Unlinking file %s", os_path) with self.perm_to_403(): rm(os_path)
def save(self, model, path=""): """Save the file model and return the model with no content.""" path = path.strip("/") self.run_pre_save_hooks(model=model, path=path) if "type" not in model: raise web.HTTPError(400, "No file type provided") if "content" not in model and model["type"] != "directory": raise web.HTTPError(400, "No file content provided") os_path = self._get_os_path(path) if is_hidden(os_path, self.root_dir) and not self.allow_hidden: raise web.HTTPError( 400, f"Cannot create file or directory {os_path!r}") self.log.debug("Saving %s", os_path) validation_error: dict = {} try: if model["type"] == "notebook": nb = nbformat.from_dict(model["content"]) self.check_and_sign(nb, path) self._save_notebook(os_path, nb, capture_validation_error=validation_error) # One checkpoint should always exist for notebooks. if not self.checkpoints.list_checkpoints(path): self.create_checkpoint(path) elif model["type"] == "file": # Missing format will be handled internally by _save_file. self._save_file(os_path, model["content"], model.get("format")) elif model["type"] == "directory": self._save_directory(os_path, model, path) else: raise web.HTTPError( 400, "Unhandled contents type: %s" % model["type"]) except web.HTTPError: raise except Exception as e: self.log.error("Error while saving file: %s %s", path, e, exc_info=True) raise web.HTTPError( 500, f"Unexpected error while saving file: {path} {e}") from e validation_message = None if model["type"] == "notebook": self.validate_notebook_model(model, validation_error=validation_error) validation_message = model.get("message", None) model = self.get(path, content=False) if validation_message: model["message"] = validation_message self.run_post_save_hooks(model=model, os_path=os_path) return model