def test_is_hidden_win32_pypy():
import ctypes
with tempfile.TemporaryDirectory() as root:
subdir1 = os.path.join(root, 'subdir')
os.makedirs(subdir1)
assert not is_hidden(subdir1, root)
subprocess.check_call(["attrib", "+h", subdir1])
with warnings.catch_warnings(record=True) as w:
# Cause all warnings to always be triggered.
warnings.simplefilter("always")
# Trigger a warning.
assert not is_hidden(subdir1, root)
# Verify the warning was triggered
assert len(w) == 1
assert issubclass(w[-1].category, UserWarning)
assert "hidden files are not detectable on this system" in str(w[-1].message)
with warnings.catch_warnings(record=True) as w:
# Cause all warnings to always be triggered.
warnings.simplefilter("always")
# Trigger a warning.
assert not is_file_hidden(subdir1)
# Verify the warning was triggered
assert len(w) == 1
assert issubclass(w[-1].category, UserWarning)
assert "hidden files are not detectable on this system" in str(w[-1].message)
def rename_file(self, old_path, new_path):
"""Rename a file."""
old_path = old_path.strip("/")
new_path = new_path.strip("/")
if new_path == old_path:
return
new_os_path = self._get_os_path(new_path)
old_os_path = self._get_os_path(old_path)
if (is_hidden(old_os_path, self.root_dir) or is_hidden(
new_os_path, self.root_dir)) and not self.allow_hidden:
raise web.HTTPError(
400, f"Cannot rename file or directory {old_os_path!r}")
# Should we proceed with the move?
if os.path.exists(new_os_path) and not samefile(
old_os_path, new_os_path):
raise web.HTTPError(409, "File already exists: %s" % new_path)
# Move the file
try:
with self.perm_to_403():
shutil.move(old_os_path, new_os_path)
except web.HTTPError:
raise
except Exception as e:
raise web.HTTPError(
500, f"Unknown error renaming file: {old_path} {e}") from e
def test_is_hidden_win32_cpython():
import ctypes
with tempfile.TemporaryDirectory() as root:
subdir1 = os.path.join(root, 'subdir')
os.makedirs(subdir1)
assert not is_hidden(subdir1, root)
subprocess.check_call(["attrib", "+h", subdir1])
assert is_hidden(subdir1, root)
assert is_file_hidden(subdir1)
def test_is_hidden_win32():
import ctypes
with tempfile.TemporaryDirectory() as root:
subdir1 = os.path.join(root, 'subdir')
os.makedirs(subdir1)
assert not is_hidden(subdir1, root)
r = ctypes.windll.kernel32.SetFileAttributesW(subdir1, 0x02)
print(r) # Helps debugging
assert is_hidden(subdir1, root)
assert is_file_hidden(subdir1)
async def _dir_model(self, path, content=True):
"""Build a model for a directory
if content is requested, will include a listing of the directory
"""
os_path = self._get_os_path(path)
four_o_four = u'directory does not exist: %r' % path
if not os.path.isdir(os_path):
raise web.HTTPError(404, four_o_four)
elif is_hidden(os_path, self.root_dir) and not self.allow_hidden:
self.log.info(
"Refusing to serve hidden directory %r, via 404 Error",
os_path)
raise web.HTTPError(404, four_o_four)
model = self._base_model(path)
model['type'] = 'directory'
model['size'] = None
if content:
model['content'] = contents = []
os_dir = self._get_os_path(path)
dir_contents = await run_sync_in_worker_thread(os.listdir, os_dir)
for name in dir_contents:
try:
os_path = os.path.join(os_dir, name)
except UnicodeDecodeError as e:
self.log.warning("failed to decode filename '%s': %s",
name, e)
continue
try:
st = await run_sync_in_worker_thread(os.lstat, os_path)
except OSError as e:
# skip over broken symlinks in listing
if e.errno == errno.ENOENT:
self.log.warning("%s doesn't exist", os_path)
else:
self.log.warning("Error stat-ing %s: %s", os_path, e)
continue
if (not stat.S_ISLNK(st.st_mode)
and not stat.S_ISREG(st.st_mode)
and not stat.S_ISDIR(st.st_mode)):
self.log.debug("%s not a regular file", os_path)
continue
if self.should_list(name):
if self.allow_hidden or not is_file_hidden(os_path,
stat_res=st):
contents.append(await
self.get(path='%s/%s' % (path, name),
content=False))
model['format'] = 'json'
return model
def test_is_hidden():
with TemporaryDirectory() as root:
subdir1 = os.path.join(root, 'subdir')
os.makedirs(subdir1)
assert not is_hidden(subdir1, root)
assert not is_file_hidden(subdir1)
subdir2 = os.path.join(root, '.subdir2')
os.makedirs(subdir2)
assert is_hidden(subdir2, root)
assert is_file_hidden(subdir2)
# root dir is always visible
assert not is_hidden(subdir2, subdir2)
subdir34 = os.path.join(root, 'subdir3', '.subdir4')
os.makedirs(subdir34)
assert is_hidden(subdir34, root)
assert is_hidden(subdir34)
subdir56 = os.path.join(root, '.subdir5', 'subdir6')
os.makedirs(subdir56)
assert is_hidden(subdir56, root)
assert is_hidden(subdir56)
assert not is_file_hidden(subdir56)
assert not is_file_hidden(subdir56, os.stat(subdir56))
async def _save_directory(self, os_path, model, path=''):
"""create a directory"""
if is_hidden(os_path, self.root_dir) and not self.allow_hidden:
raise web.HTTPError(400, u'Cannot create hidden directory %r' % os_path)
if not os.path.exists(os_path):
with self.perm_to_403():
await run_sync(os.mkdir, os_path)
elif not os.path.isdir(os_path):
raise web.HTTPError(400, u'Not a directory: %s' % (os_path))
else:
self.log.debug("Directory %r already exists", os_path)
def _save_directory(self, os_path, model, path=""):
"""create a directory"""
if is_hidden(os_path, self.root_dir) and not self.allow_hidden:
raise web.HTTPError(400, "Cannot create directory %r" % os_path)
if not os.path.exists(os_path):
with self.perm_to_403():
os.mkdir(os_path)
elif not os.path.isdir(os_path):
raise web.HTTPError(400, "Not a directory: %s" % (os_path))
else:
self.log.debug("Directory %r already exists", os_path)
def get(self, path, content=True, type=None, format=None):
"""Takes a path for an entity and returns its model
Parameters
----------
path : str
the API path that describes the relative path for the target
content : bool
Whether to include the contents in the reply
type : str, optional
The requested type - 'file', 'notebook', or 'directory'.
Will raise HTTPError 400 if the content doesn't match.
format : str, optional
The requested format for file contents. 'text' or 'base64'.
Ignored if this returns a notebook or directory model.
Returns
-------
model : dict
the contents model. If content=True, returns the contents
of the file or directory as well.
"""
path = path.strip("/")
os_path = self._get_os_path(path)
four_o_four = "file or directory does not exist: %r" % path
if not self.exists(path):
raise web.HTTPError(404, four_o_four)
if is_hidden(os_path, self.root_dir) and not self.allow_hidden:
self.log.info(
"Refusing to serve hidden file or directory %r, via 404 Error",
os_path)
raise web.HTTPError(404, four_o_four)
if os.path.isdir(os_path):
if type not in (None, "directory"):
raise web.HTTPError(
400,
f"{path} is a directory, not a {type}",
reason="bad type",
)
model = self._dir_model(path, content=content)
elif type == "notebook" or (type is None and path.endswith(".ipynb")):
model = self._notebook_model(path, content=content)
else:
if type == "directory":
raise web.HTTPError(400,
"%s is not a directory" % path,
reason="bad type")
model = self._file_model(path, content=content, format=format)
return model
def validate_absolute_path(self, root, absolute_path):
"""Validate and return the absolute path.
Requires tornado 3.1
Adding to tornado's own handling, forbids the serving of hidden files.
"""
abs_path = super(AuthenticatedFileHandler, self).validate_absolute_path(root, absolute_path)
abs_root = os.path.abspath(root)
if is_hidden(abs_path, abs_root) and not self.contents_manager.allow_hidden:
self.log.info("Refusing to serve hidden file, via 404 Error, use flag 'ContentsManager.allow_hidden' to enable")
raise web.HTTPError(404)
return abs_path
def _base_model(self, path):
"""Build the common base of a contents model"""
os_path = self._get_os_path(path)
info = os.lstat(os_path)
four_o_four = "file or directory does not exist: %r" % path
if is_hidden(os_path, self.root_dir) and not self.allow_hidden:
self.log.info(
"Refusing to serve hidden file or directory %r, via 404 Error",
os_path)
raise web.HTTPError(404, four_o_four)
try:
# size of file
size = info.st_size
except (ValueError, OSError):
self.log.warning("Unable to get size.")
size = None
try:
last_modified = tz.utcfromtimestamp(info.st_mtime)
except (ValueError, OSError):
# Files can rarely have an invalid timestamp
# https://github.com/jupyter/notebook/issues/2539
# https://github.com/jupyter/notebook/issues/2757
# Use the Unix epoch as a fallback so we don't crash.
self.log.warning("Invalid mtime %s for %s", info.st_mtime, os_path)
last_modified = datetime(1970, 1, 1, 0, 0, tzinfo=tz.UTC)
try:
created = tz.utcfromtimestamp(info.st_ctime)
except (ValueError, OSError): # See above
self.log.warning("Invalid ctime %s for %s", info.st_ctime, os_path)
created = datetime(1970, 1, 1, 0, 0, tzinfo=tz.UTC)
# Create the base model.
model = {}
model["name"] = path.rsplit("/", 1)[-1]
model["path"] = path
model["last_modified"] = last_modified
model["created"] = created
model["content"] = None
model["format"] = None
model["mimetype"] = None
model["size"] = size
model["writable"] = self.is_writable(path)
return model
def is_hidden(self, path):
"""Does the API style path correspond to a hidden directory or file?
Parameters
----------
path : string
The path to check. This is an API path (`/` separated,
relative to root_dir).
Returns
-------
hidden : bool
Whether the path exists and is hidden.
"""
path = path.strip('/')
os_path = self._get_os_path(path=path)
return is_hidden(os_path, self.root_dir)
def _dir_model(self, path, content=True):
"""Build a model for a directory
if content is requested, will include a listing of the directory
"""
os_path = self._get_os_path(path)
four_o_four = u'directory does not exist: %r' % path
if not os.path.isdir(os_path):
raise web.HTTPError(404, four_o_four)
elif is_hidden(os_path, self.root_dir) and not self.allow_hidden:
self.log.info("Refusing to serve hidden directory %r, via 404 Error",
os_path
)
raise web.HTTPError(404, four_o_four)
model = self._base_model(path)
model['type'] = 'directory'
model['size'] = None
if content:
model['content'] = contents = []
os_dir = self._get_os_path(path)
for name in os.listdir(os_dir):
try:
os_path = os.path.join(os_dir, name)
except UnicodeDecodeError as e:
self.log.warning(
"failed to decode filename '%s': %s", name, e)
continue
try:
st = os.lstat(os_path)
except OSError as e:
# skip over broken symlinks in listing
if e.errno == errno.ENOENT:
self.log.warning("%s doesn't exist", os_path)
elif e.errno != errno.EACCES: # Don't provide clues about protected files
self.log.warning("Error stat-ing %s: %s", os_path, e)
continue
if (not stat.S_ISLNK(st.st_mode)
and not stat.S_ISREG(st.st_mode)
and not stat.S_ISDIR(st.st_mode)):
self.log.debug("%s not a regular file", os_path)
continue
try:
if self.should_list(name):
if self.allow_hidden or not is_file_hidden(os_path, stat_res=st):
contents.append(
self.get(path='%s/%s' % (path, name), content=False)
)
except OSError as e:
# ELOOP: recursive symlink, also don't show failure due to permissions
if e.errno not in [errno.ELOOP, errno.EACCES]:
self.log.warning(
"Unknown error checking if file %r is hidden",
os_path,
exc_info=True,
)
model['format'] = 'json'
return model
async def is_hidden(self, path):
"""Is path a hidden directory or file"""
path = path.strip("/")
os_path = self._get_os_path(path=path)
return is_hidden(os_path, self.root_dir)
def delete_file(self, path):
"""Delete file at path."""
path = path.strip("/")
os_path = self._get_os_path(path)
rm = os.unlink
four_o_four = "file or directory does not exist: %r" % path
if not self.exists(path):
raise web.HTTPError(404, four_o_four)
if is_hidden(os_path, self.root_dir) and not self.allow_hidden:
raise web.HTTPError(
400, f"Cannot delete file or directory {os_path!r}")
def _check_trash(os_path):
if sys.platform in {"win32", "darwin"}:
return True
# It's a bit more nuanced than this, but until we can better
# distinguish errors from send2trash, assume that we can only trash
# files on the same partition as the home directory.
file_dev = os.stat(os_path).st_dev
home_dev = os.stat(os.path.expanduser("~")).st_dev
return file_dev == home_dev
def is_non_empty_dir(os_path):
if os.path.isdir(os_path):
# A directory containing only leftover checkpoints is
# considered empty.
cp_dir = getattr(self.checkpoints, "checkpoint_dir", None)
if set(os.listdir(os_path)) - {cp_dir}:
return True
return False
if self.delete_to_trash:
if not self.always_delete_dir and sys.platform == "win32" and is_non_empty_dir(
os_path):
# send2trash can really delete files on Windows, so disallow
# deleting non-empty files. See Github issue 3631.
raise web.HTTPError(400, "Directory %s not empty" % os_path)
if _check_trash(os_path):
# Looking at the code in send2trash, I don't think the errors it
# raises let us distinguish permission errors from other errors in
# code. So for now, the "look before you leap" approach is used.
if not self.is_writable(path):
raise web.HTTPError(403, "Permission denied: %s" % path)
self.log.debug("Sending %s to trash", os_path)
send2trash(os_path)
return
else:
self.log.warning(
"Skipping trash for %s, on different device to home directory",
os_path,
)
if os.path.isdir(os_path):
# Don't permanently delete non-empty directories.
if not self.always_delete_dir and is_non_empty_dir(os_path):
raise web.HTTPError(400, "Directory %s not empty" % os_path)
self.log.debug("Removing directory %s", os_path)
with self.perm_to_403():
shutil.rmtree(os_path)
else:
self.log.debug("Unlinking file %s", os_path)
with self.perm_to_403():
rm(os_path)
def save(self, model, path=""):
"""Save the file model and return the model with no content."""
path = path.strip("/")
self.run_pre_save_hooks(model=model, path=path)
if "type" not in model:
raise web.HTTPError(400, "No file type provided")
if "content" not in model and model["type"] != "directory":
raise web.HTTPError(400, "No file content provided")
os_path = self._get_os_path(path)
if is_hidden(os_path, self.root_dir) and not self.allow_hidden:
raise web.HTTPError(
400, f"Cannot create file or directory {os_path!r}")
self.log.debug("Saving %s", os_path)
validation_error: dict = {}
try:
if model["type"] == "notebook":
nb = nbformat.from_dict(model["content"])
self.check_and_sign(nb, path)
self._save_notebook(os_path,
nb,
capture_validation_error=validation_error)
# One checkpoint should always exist for notebooks.
if not self.checkpoints.list_checkpoints(path):
self.create_checkpoint(path)
elif model["type"] == "file":
# Missing format will be handled internally by _save_file.
self._save_file(os_path, model["content"], model.get("format"))
elif model["type"] == "directory":
self._save_directory(os_path, model, path)
else:
raise web.HTTPError(
400, "Unhandled contents type: %s" % model["type"])
except web.HTTPError:
raise
except Exception as e:
self.log.error("Error while saving file: %s %s",
path,
e,
exc_info=True)
raise web.HTTPError(
500, f"Unexpected error while saving file: {path} {e}") from e
validation_message = None
if model["type"] == "notebook":
self.validate_notebook_model(model,
validation_error=validation_error)
validation_message = model.get("message", None)
model = self.get(path, content=False)
if validation_message:
model["message"] = validation_message
self.run_post_save_hooks(model=model, os_path=os_path)
return model
