def move(cls, item, to_collection, to_href):
if not pathutils.is_safe_filesystem_path_component(to_href):
raise pathutils.UnsafePathError(to_href)
os.replace(
pathutils.path_to_filesystem(
item.collection._filesystem_path, item.href),
pathutils.path_to_filesystem(
to_collection._filesystem_path, to_href))
cls._sync_directory(to_collection._filesystem_path)
if item.collection._filesystem_path != to_collection._filesystem_path:
cls._sync_directory(item.collection._filesystem_path)
# Move the item cache entry
cache_folder = os.path.join(item.collection._filesystem_path,
".Radicale.cache", "item")
to_cache_folder = os.path.join(to_collection._filesystem_path,
".Radicale.cache", "item")
cls._makedirs_synced(to_cache_folder)
try:
os.replace(os.path.join(cache_folder, item.href),
os.path.join(to_cache_folder, to_href))
except FileNotFoundError:
pass
else:
cls._makedirs_synced(to_cache_folder)
if cache_folder != to_cache_folder:
cls._makedirs_synced(cache_folder)
# Track the change
to_collection._update_history_etag(to_href, item)
item.collection._update_history_etag(item.href, None)
to_collection._clean_history()
if item.collection._filesystem_path != to_collection._filesystem_path:
item.collection._clean_history()
def move(self, item, to_collection, to_href):
if not pathutils.is_safe_filesystem_path_component(to_href):
raise pathutils.UnsafePathError(to_href)
os.replace(
pathutils.path_to_filesystem(item.collection._filesystem_path,
item.href),
pathutils.path_to_filesystem(to_collection._filesystem_path,
to_href))
self._sync_directory(to_collection._filesystem_path)
if item.collection._filesystem_path != to_collection._filesystem_path:
self._sync_directory(item.collection._filesystem_path)
# Move the item cache entry
cache_folder = os.path.join(item.collection._filesystem_path,
".Radicale.cache", "item")
to_cache_folder = os.path.join(to_collection._filesystem_path,
".Radicale.cache", "item")
self._makedirs_synced(to_cache_folder)
try:
os.replace(os.path.join(cache_folder, item.href),
os.path.join(to_cache_folder, to_href))
except FileNotFoundError:
pass
else:
self._makedirs_synced(to_cache_folder)
if cache_folder != to_cache_folder:
self._makedirs_synced(cache_folder)
# Track the change
to_collection._update_history_etag(to_href, item)
item.collection._update_history_etag(item.href, None)
to_collection._clean_history()
if item.collection._filesystem_path != to_collection._filesystem_path:
item.collection._clean_history()
def discover(cls,
path,
depth="0",
child_context_manager=(
lambda path, href=None: contextlib.ExitStack())):
# Path should already be sanitized
sane_path = pathutils.strip_path(path)
attributes = sane_path.split("/") if sane_path else []
folder = cls._get_collection_root_folder()
# Create the root collection
cls._makedirs_synced(folder)
try:
filesystem_path = pathutils.path_to_filesystem(folder, sane_path)
except ValueError as e:
# Path is unsafe
logger.debug("Unsafe path %r requested from storage: %s",
sane_path,
e,
exc_info=True)
return
# Check if the path exists and if it leads to a collection or an item
if not os.path.isdir(filesystem_path):
if attributes and os.path.isfile(filesystem_path):
href = attributes.pop()
else:
return
else:
href = None
sane_path = "/".join(attributes)
collection = cls(pathutils.unstrip_path(sane_path, True))
if href:
yield collection._get(href)
return
yield collection
if depth == "0":
return
for href in collection._list():
with child_context_manager(sane_path, href):
yield collection._get(href)
for entry in os.scandir(filesystem_path):
if not entry.is_dir():
continue
href = entry.name
if not pathutils.is_safe_filesystem_path_component(href):
if not href.startswith(".Radicale"):
logger.debug("Skipping collection %r in %r", href,
sane_path)
continue
sane_child_path = posixpath.join(sane_path, href)
child_path = pathutils.unstrip_path(sane_child_path, True)
with child_context_manager(sane_child_path):
yield cls(child_path)
def get(self, environ, base_prefix, path, user):
assert path == "/.web" or path.startswith("/.web/")
assert pathutils.sanitize_path(path) == path
try:
filesystem_path = pathutils.path_to_filesystem(
self.folder, path[len("/.web"):].strip("/"))
except ValueError as e:
logger.debug("Web content with unsafe path %r requested: %s",
path, e, exc_info=True)
return httputils.NOT_FOUND
if os.path.isdir(filesystem_path) and not path.endswith("/"):
location = posixpath.basename(path) + "/"
return (client.FOUND,
{"Location": location, "Content-Type": "text/plain"},
"Redirected to %s" % location)
if os.path.isdir(filesystem_path):
filesystem_path = os.path.join(filesystem_path, "index.html")
if not os.path.isfile(filesystem_path):
return httputils.NOT_FOUND
content_type = MIMETYPES.get(
os.path.splitext(filesystem_path)[1].lower(), FALLBACK_MIMETYPE)
with open(filesystem_path, "rb") as f:
answer = f.read()
last_modified = time.strftime(
"%a, %d %b %Y %H:%M:%S GMT",
time.gmtime(os.fstat(f.fileno()).st_mtime))
headers = {
"Content-Type": content_type,
"Last-Modified": last_modified}
return client.OK, headers, answer
def delete(self, href=None):
if href is None:
# Delete the collection
parent_dir = os.path.dirname(self._filesystem_path)
try:
os.rmdir(self._filesystem_path)
except OSError:
with TemporaryDirectory(
prefix=".Radicale.tmp-", dir=parent_dir) as tmp:
os.rename(self._filesystem_path, os.path.join(
tmp, os.path.basename(self._filesystem_path)))
self._sync_directory(parent_dir)
else:
self._sync_directory(parent_dir)
else:
# Delete an item
if not pathutils.is_safe_filesystem_path_component(href):
raise pathutils.UnsafePathError(href)
path = pathutils.path_to_filesystem(self._filesystem_path, href)
if not os.path.isfile(path):
raise storage.ComponentNotFoundError(href)
os.remove(path)
self._sync_directory(os.path.dirname(path))
# Track the change
self._update_history_etag(href, None)
self._clean_history()
def get(self, environ, base_prefix, path, user):
assert path == "/.web" or path.startswith("/.web/")
assert pathutils.sanitize_path(path) == path
try:
filesystem_path = pathutils.path_to_filesystem(
self.folder, path[len("/.web"):].strip("/"))
except ValueError as e:
logger.debug("Web content with unsafe path %r requested: %s",
path,
e,
exc_info=True)
return httputils.NOT_FOUND
if os.path.isdir(filesystem_path) and not path.endswith("/"):
location = posixpath.basename(path) + "/"
return (client.FOUND, {
"Location": location,
"Content-Type": "text/plain"
}, "Redirected to %s" % location)
if os.path.isdir(filesystem_path):
filesystem_path = os.path.join(filesystem_path, "index.html")
if not os.path.isfile(filesystem_path):
return httputils.NOT_FOUND
content_type = MIMETYPES.get(
os.path.splitext(filesystem_path)[1].lower(), FALLBACK_MIMETYPE)
with open(filesystem_path, "rb") as f:
answer = f.read()
last_modified = time.strftime(
"%a, %d %b %Y %H:%M:%S GMT",
time.gmtime(os.fstat(f.fileno()).st_mtime))
headers = {
"Content-Type": content_type,
"Last-Modified": last_modified
}
return client.OK, headers, answer
def _upload_all_nonatomic(self, items, suffix=""):
"""Upload a new set of items.
This takes a list of vobject items and
uploads them nonatomic and without existence checks.
"""
cache_folder = os.path.join(self._filesystem_path, ".Radicale.cache",
"item")
self._storage._makedirs_synced(cache_folder)
hrefs = set()
for item in items:
uid = item.uid
try:
cache_content = self._item_cache_content(item)
except Exception as e:
raise ValueError(
"Failed to store item %r in temporary collection %r: %s" %
(uid, self.path, e)) from e
href_candidate_funtions = []
if os.name in ("nt", "posix"):
href_candidate_funtions.append(lambda: uid if uid.lower(
).endswith(suffix.lower()) else uid + suffix)
href_candidate_funtions.extend(
(lambda: radicale_item.get_etag(uid).strip('"') + suffix,
lambda: radicale_item.find_available_uid(
hrefs.__contains__, suffix)))
href = f = None
while href_candidate_funtions:
href = href_candidate_funtions.pop(0)()
if href in hrefs:
continue
if not pathutils.is_safe_filesystem_path_component(href):
if not href_candidate_funtions:
raise pathutils.UnsafePathError(href)
continue
try:
f = open(pathutils.path_to_filesystem(
self._filesystem_path, href),
"w",
newline="",
encoding=self._encoding)
break
except OSError as e:
if href_candidate_funtions and (
os.name == "posix" and e.errno == 22
or os.name == "nt" and e.errno == 123):
continue
raise
with f:
f.write(item.serialize())
f.flush()
self._storage._fsync(f)
hrefs.add(href)
with open(os.path.join(cache_folder, href), "wb") as f:
pickle.dump(cache_content, f)
f.flush()
self._storage._fsync(f)
self._storage._sync_directory(cache_folder)
self._storage._sync_directory(self._filesystem_path)
def delete(self, href=None):
if href is None:
# Delete the collection
parent_dir = os.path.dirname(self._filesystem_path)
try:
os.rmdir(self._filesystem_path)
except OSError:
with TemporaryDirectory(
prefix=".Radicale.tmp-", dir=parent_dir) as tmp:
os.rename(self._filesystem_path, os.path.join(
tmp, os.path.basename(self._filesystem_path)))
self._sync_directory(parent_dir)
else:
self._sync_directory(parent_dir)
else:
# Delete an item
if not pathutils.is_safe_filesystem_path_component(href):
raise pathutils.UnsafePathError(href)
path = pathutils.path_to_filesystem(self._filesystem_path, href)
if not os.path.isfile(path):
raise storage.ComponentNotFoundError(href)
os.remove(path)
self._sync_directory(os.path.dirname(path))
# Track the change
self._update_history_etag(href, None)
self._clean_history()
def __init__(self, path, filesystem_path=None):
folder = self._get_collection_root_folder()
# Path should already be sanitized
self.path = pathutils.strip_path(path)
self._encoding = self.configuration.get("encoding", "stock")
if filesystem_path is None:
filesystem_path = pathutils.path_to_filesystem(folder, self.path)
self._filesystem_path = filesystem_path
self._etag_cache = None
super().__init__()
def __init__(self, path, filesystem_path=None):
folder = self._get_collection_root_folder()
# Path should already be sanitized
self.path = pathutils.strip_path(path)
self._encoding = self.configuration.get("encoding", "stock")
if filesystem_path is None:
filesystem_path = pathutils.path_to_filesystem(folder, self.path)
self._filesystem_path = filesystem_path
self._etag_cache = None
super().__init__()
def discover(cls, path, depth="0", child_context_manager=(
lambda path, href=None: contextlib.ExitStack())):
# Path should already be sanitized
sane_path = pathutils.strip_path(path)
attributes = sane_path.split("/") if sane_path else []
folder = cls._get_collection_root_folder()
# Create the root collection
cls._makedirs_synced(folder)
try:
filesystem_path = pathutils.path_to_filesystem(folder, sane_path)
except ValueError as e:
# Path is unsafe
logger.debug("Unsafe path %r requested from storage: %s",
sane_path, e, exc_info=True)
return
# Check if the path exists and if it leads to a collection or an item
if not os.path.isdir(filesystem_path):
if attributes and os.path.isfile(filesystem_path):
href = attributes.pop()
else:
return
else:
href = None
sane_path = "/".join(attributes)
collection = cls(pathutils.unstrip_path(sane_path, True))
if href:
yield collection._get(href)
return
yield collection
if depth == "0":
return
for href in collection._list():
with child_context_manager(sane_path, href):
yield collection._get(href)
for entry in os.scandir(filesystem_path):
if not entry.is_dir():
continue
href = entry.name
if not pathutils.is_safe_filesystem_path_component(href):
if not href.startswith(".Radicale"):
logger.debug("Skipping collection %r in %r",
href, sane_path)
continue
sane_child_path = posixpath.join(sane_path, href)
child_path = pathutils.unstrip_path(sane_child_path, True)
with child_context_manager(sane_child_path):
yield cls(child_path)
def __init__(self, path, filesystem_path=None):
folder = self._get_collection_root_folder()
# Path should already be sanitized
self.path = pathutils.strip_path(path)
self._encoding = self.configuration.get("encoding", "stock")
if filesystem_path is None:
filesystem_path = pathutils.path_to_filesystem(folder, self.path)
self._filesystem_path = filesystem_path
self._props_path = os.path.join(self._filesystem_path,
".Radicale.props")
self._meta_cache = None
self._etag_cache = None
self._item_cache_cleaned = False
def upload(self, href, item):
if not pathutils.is_safe_filesystem_path_component(href):
raise pathutils.UnsafePathError(href)
try:
self._store_item_cache(href, item)
except Exception as e:
raise ValueError("Failed to store item %r in collection %r: %s" %
(href, self.path, e)) from e
path = pathutils.path_to_filesystem(self._filesystem_path, href)
with self._atomic_write(path, newline="") as fd:
fd.write(item.serialize())
# Clean the cache after the actual item is stored, or the cache entry
# will be removed again.
self._clean_item_cache()
# Track the change
self._update_history_etag(href, item)
self._clean_history()
return self._get(href, verify_href=False)
def upload(self, href, item):
if not pathutils.is_safe_filesystem_path_component(href):
raise pathutils.UnsafePathError(href)
try:
self._store_item_cache(href, item)
except Exception as e:
raise ValueError("Failed to store item %r in collection %r: %s" %
(href, self.path, e)) from e
path = pathutils.path_to_filesystem(self._filesystem_path, href)
with self._atomic_write(path, newline="") as fd:
fd.write(item.serialize())
# Clean the cache after the actual item is stored, or the cache entry
# will be removed again.
self._clean_item_cache()
# Track the change
self._update_history_etag(href, item)
self._clean_history()
return self._get(href, verify_href=False)
def replace_fn(source, target):
nonlocal href
while href_candidates:
href = href_candidates.pop(0)()
if href in hrefs:
continue
if not pathutils.is_safe_filesystem_path_component(href):
if not href_candidates:
raise pathutils.UnsafePathError(href)
continue
try:
return os.replace(source, pathutils.path_to_filesystem(
self._filesystem_path, href))
except OSError as e:
if href_candidates and (
os.name == "posix" and e.errno == 22 or
os.name == "nt" and e.errno == 123):
continue
raise
def replace_fn(source, target):
nonlocal href
while href_candidates:
href = href_candidates.pop(0)()
if href in hrefs:
continue
if not pathutils.is_safe_filesystem_path_component(href):
if not href_candidates:
raise pathutils.UnsafePathError(href)
continue
try:
return os.replace(source, pathutils.path_to_filesystem(
self._filesystem_path, href))
except OSError as e:
if href_candidates and (
os.name == "posix" and e.errno == 22 or
os.name == "nt" and e.errno == 123):
continue
raise
def create_collection(self, href, items=None, props=None):
folder = self._get_collection_root_folder()
# Path should already be sanitized
sane_path = pathutils.strip_path(href)
filesystem_path = pathutils.path_to_filesystem(folder, sane_path)
if not props:
self._makedirs_synced(filesystem_path)
return self._collection_class(
self, pathutils.unstrip_path(sane_path, True))
parent_dir = os.path.dirname(filesystem_path)
self._makedirs_synced(parent_dir)
# Create a temporary directory with an unsafe name
with TemporaryDirectory(prefix=".Radicale.tmp-",
dir=parent_dir) as tmp_dir:
# The temporary directory itself can't be renamed
tmp_filesystem_path = os.path.join(tmp_dir, "collection")
os.makedirs(tmp_filesystem_path)
col = self._collection_class(self,
pathutils.unstrip_path(
sane_path, True),
filesystem_path=tmp_filesystem_path)
col.set_meta(props)
if items is not None:
if props.get("tag") == "VCALENDAR":
col._upload_all_nonatomic(items, suffix=".ics")
elif props.get("tag") == "VADDRESSBOOK":
col._upload_all_nonatomic(items, suffix=".vcf")
# This operation is not atomic on the filesystem level but it's
# very unlikely that one rename operations succeeds while the
# other fails or that only one gets written to disk.
if os.path.exists(filesystem_path):
os.rename(filesystem_path, os.path.join(tmp_dir, "delete"))
os.rename(tmp_filesystem_path, filesystem_path)
self._sync_directory(parent_dir)
return self._collection_class(self,
pathutils.unstrip_path(sane_path, True))
def create_collection(self, href, items=None, props=None):
folder = self._get_collection_root_folder()
# Path should already be sanitized
sane_path = pathutils.strip_path(href)
filesystem_path = pathutils.path_to_filesystem(folder, sane_path)
if not props:
self._makedirs_synced(filesystem_path)
return self._collection_class(
self, pathutils.unstrip_path(sane_path, True))
parent_dir = os.path.dirname(filesystem_path)
self._makedirs_synced(parent_dir)
# Create a temporary directory with an unsafe name
with TemporaryDirectory(
prefix=".Radicale.tmp-", dir=parent_dir) as tmp_dir:
# The temporary directory itself can't be renamed
tmp_filesystem_path = os.path.join(tmp_dir, "collection")
os.makedirs(tmp_filesystem_path)
col = self._collection_class(
self, pathutils.unstrip_path(sane_path, True),
filesystem_path=tmp_filesystem_path)
col.set_meta(props)
if items is not None:
if props.get("tag") == "VCALENDAR":
col._upload_all_nonatomic(items, suffix=".ics")
elif props.get("tag") == "VADDRESSBOOK":
col._upload_all_nonatomic(items, suffix=".vcf")
if os.path.lexists(filesystem_path):
pathutils.rename_exchange(tmp_filesystem_path, filesystem_path)
else:
os.rename(tmp_filesystem_path, filesystem_path)
self._sync_directory(parent_dir)
return self._collection_class(
self, pathutils.unstrip_path(sane_path, True))
def _get(self, href, verify_href=True):
if verify_href:
try:
if not pathutils.is_safe_filesystem_path_component(href):
raise pathutils.UnsafePathError(href)
path = pathutils.path_to_filesystem(self._filesystem_path,
href)
except ValueError as e:
logger.debug(
"Can't translate name %r safely to filesystem in %r: %s",
href,
self.path,
e,
exc_info=True)
return None
else:
path = os.path.join(self._filesystem_path, href)
try:
with open(path, "rb") as f:
raw_text = f.read()
except (FileNotFoundError, IsADirectoryError):
return None
except PermissionError:
# Windows raises ``PermissionError`` when ``path`` is a directory
if (os.name == "nt" and os.path.isdir(path)
and os.access(path, os.R_OK)):
return None
raise
# The hash of the component in the file system. This is used to check,
# if the entry in the cache is still valid.
input_hash = self._item_cache_hash(raw_text)
cache_hash, uid, etag, text, name, tag, start, end = \
self._load_item_cache(href, input_hash)
if input_hash != cache_hash:
with self._acquire_cache_lock("item"):
# Lock the item cache to prevent multpile processes from
# generating the same data in parallel.
# This improves the performance for multiple requests.
if self._lock.locked == "r":
# Check if another process created the file in the meantime
cache_hash, uid, etag, text, name, tag, start, end = \
self._load_item_cache(href, input_hash)
if input_hash != cache_hash:
try:
vobject_items = tuple(
vobject.readComponents(
raw_text.decode(self._encoding)))
radicale_item.check_and_sanitize_items(
vobject_items, tag=self.get_meta("tag"))
vobject_item, = vobject_items
temp_item = radicale_item.Item(
collection=self, vobject_item=vobject_item)
cache_hash, uid, etag, text, name, tag, start, end = \
self._store_item_cache(
href, temp_item, input_hash)
except Exception as e:
raise RuntimeError("Failed to load item %r in %r: %s" %
(href, self.path, e)) from e
# Clean cache entries once after the data in the file
# system was edited externally.
if not self._item_cache_cleaned:
self._item_cache_cleaned = True
self._clean_item_cache()
last_modified = time.strftime("%a, %d %b %Y %H:%M:%S GMT",
time.gmtime(os.path.getmtime(path)))
# Don't keep reference to ``vobject_item``, because it requires a lot
# of memory.
return radicale_item.Item(collection=self,
href=href,
last_modified=last_modified,
etag=etag,
text=text,
uid=uid,
name=name,
component_name=tag,
time_range=(start, end))
def _get(self, href, verify_href=True):
if verify_href:
try:
if not pathutils.is_safe_filesystem_path_component(href):
raise pathutils.UnsafePathError(href)
path = pathutils.path_to_filesystem(
self._filesystem_path, href)
except ValueError as e:
logger.debug(
"Can't translate name %r safely to filesystem in %r: %s",
href, self.path, e, exc_info=True)
return None
else:
path = os.path.join(self._filesystem_path, href)
try:
with open(path, "rb") as f:
raw_text = f.read()
except (FileNotFoundError, IsADirectoryError):
return None
except PermissionError:
# Windows raises ``PermissionError`` when ``path`` is a directory
if (os.name == "nt" and
os.path.isdir(path) and os.access(path, os.R_OK)):
return None
raise
# The hash of the component in the file system. This is used to check,
# if the entry in the cache is still valid.
input_hash = self._item_cache_hash(raw_text)
cache_hash, uid, etag, text, name, tag, start, end = \
self._load_item_cache(href, input_hash)
if input_hash != cache_hash:
with self._acquire_cache_lock("item"):
# Lock the item cache to prevent multpile processes from
# generating the same data in parallel.
# This improves the performance for multiple requests.
if self._lock.locked == "r":
# Check if another process created the file in the meantime
cache_hash, uid, etag, text, name, tag, start, end = \
self._load_item_cache(href, input_hash)
if input_hash != cache_hash:
try:
vobject_items = tuple(vobject.readComponents(
raw_text.decode(self._encoding)))
radicale_item.check_and_sanitize_items(
vobject_items, tag=self.get_meta("tag"))
vobject_item, = vobject_items
temp_item = radicale_item.Item(
collection=self, vobject_item=vobject_item)
cache_hash, uid, etag, text, name, tag, start, end = \
self._store_item_cache(
href, temp_item, input_hash)
except Exception as e:
raise RuntimeError("Failed to load item %r in %r: %s" %
(href, self.path, e)) from e
# Clean cache entries once after the data in the file
# system was edited externally.
if not self._item_cache_cleaned:
self._item_cache_cleaned = True
self._clean_item_cache()
last_modified = time.strftime(
"%a, %d %b %Y %H:%M:%S GMT",
time.gmtime(os.path.getmtime(path)))
# Don't keep reference to ``vobject_item``, because it requires a lot
# of memory.
return radicale_item.Item(
collection=self, href=href, last_modified=last_modified, etag=etag,
text=text, uid=uid, name=name, component_name=tag,
time_range=(start, end))
