def move(cls, item, to_collection, to_href): if not pathutils.is_safe_filesystem_path_component(to_href): raise pathutils.UnsafePathError(to_href) os.replace( pathutils.path_to_filesystem( item.collection._filesystem_path, item.href), pathutils.path_to_filesystem( to_collection._filesystem_path, to_href)) cls._sync_directory(to_collection._filesystem_path) if item.collection._filesystem_path != to_collection._filesystem_path: cls._sync_directory(item.collection._filesystem_path) # Move the item cache entry cache_folder = os.path.join(item.collection._filesystem_path, ".Radicale.cache", "item") to_cache_folder = os.path.join(to_collection._filesystem_path, ".Radicale.cache", "item") cls._makedirs_synced(to_cache_folder) try: os.replace(os.path.join(cache_folder, item.href), os.path.join(to_cache_folder, to_href)) except FileNotFoundError: pass else: cls._makedirs_synced(to_cache_folder) if cache_folder != to_cache_folder: cls._makedirs_synced(cache_folder) # Track the change to_collection._update_history_etag(to_href, item) item.collection._update_history_etag(item.href, None) to_collection._clean_history() if item.collection._filesystem_path != to_collection._filesystem_path: item.collection._clean_history()
def move(self, item, to_collection, to_href): if not pathutils.is_safe_filesystem_path_component(to_href): raise pathutils.UnsafePathError(to_href) os.replace( pathutils.path_to_filesystem(item.collection._filesystem_path, item.href), pathutils.path_to_filesystem(to_collection._filesystem_path, to_href)) self._sync_directory(to_collection._filesystem_path) if item.collection._filesystem_path != to_collection._filesystem_path: self._sync_directory(item.collection._filesystem_path) # Move the item cache entry cache_folder = os.path.join(item.collection._filesystem_path, ".Radicale.cache", "item") to_cache_folder = os.path.join(to_collection._filesystem_path, ".Radicale.cache", "item") self._makedirs_synced(to_cache_folder) try: os.replace(os.path.join(cache_folder, item.href), os.path.join(to_cache_folder, to_href)) except FileNotFoundError: pass else: self._makedirs_synced(to_cache_folder) if cache_folder != to_cache_folder: self._makedirs_synced(cache_folder) # Track the change to_collection._update_history_etag(to_href, item) item.collection._update_history_etag(item.href, None) to_collection._clean_history() if item.collection._filesystem_path != to_collection._filesystem_path: item.collection._clean_history()
def discover(cls, path, depth="0", child_context_manager=( lambda path, href=None: contextlib.ExitStack())): # Path should already be sanitized sane_path = pathutils.strip_path(path) attributes = sane_path.split("/") if sane_path else [] folder = cls._get_collection_root_folder() # Create the root collection cls._makedirs_synced(folder) try: filesystem_path = pathutils.path_to_filesystem(folder, sane_path) except ValueError as e: # Path is unsafe logger.debug("Unsafe path %r requested from storage: %s", sane_path, e, exc_info=True) return # Check if the path exists and if it leads to a collection or an item if not os.path.isdir(filesystem_path): if attributes and os.path.isfile(filesystem_path): href = attributes.pop() else: return else: href = None sane_path = "/".join(attributes) collection = cls(pathutils.unstrip_path(sane_path, True)) if href: yield collection._get(href) return yield collection if depth == "0": return for href in collection._list(): with child_context_manager(sane_path, href): yield collection._get(href) for entry in os.scandir(filesystem_path): if not entry.is_dir(): continue href = entry.name if not pathutils.is_safe_filesystem_path_component(href): if not href.startswith(".Radicale"): logger.debug("Skipping collection %r in %r", href, sane_path) continue sane_child_path = posixpath.join(sane_path, href) child_path = pathutils.unstrip_path(sane_child_path, True) with child_context_manager(sane_child_path): yield cls(child_path)
def get(self, environ, base_prefix, path, user): assert path == "/.web" or path.startswith("/.web/") assert pathutils.sanitize_path(path) == path try: filesystem_path = pathutils.path_to_filesystem( self.folder, path[len("/.web"):].strip("/")) except ValueError as e: logger.debug("Web content with unsafe path %r requested: %s", path, e, exc_info=True) return httputils.NOT_FOUND if os.path.isdir(filesystem_path) and not path.endswith("/"): location = posixpath.basename(path) + "/" return (client.FOUND, {"Location": location, "Content-Type": "text/plain"}, "Redirected to %s" % location) if os.path.isdir(filesystem_path): filesystem_path = os.path.join(filesystem_path, "index.html") if not os.path.isfile(filesystem_path): return httputils.NOT_FOUND content_type = MIMETYPES.get( os.path.splitext(filesystem_path)[1].lower(), FALLBACK_MIMETYPE) with open(filesystem_path, "rb") as f: answer = f.read() last_modified = time.strftime( "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(os.fstat(f.fileno()).st_mtime)) headers = { "Content-Type": content_type, "Last-Modified": last_modified} return client.OK, headers, answer
def delete(self, href=None): if href is None: # Delete the collection parent_dir = os.path.dirname(self._filesystem_path) try: os.rmdir(self._filesystem_path) except OSError: with TemporaryDirectory( prefix=".Radicale.tmp-", dir=parent_dir) as tmp: os.rename(self._filesystem_path, os.path.join( tmp, os.path.basename(self._filesystem_path))) self._sync_directory(parent_dir) else: self._sync_directory(parent_dir) else: # Delete an item if not pathutils.is_safe_filesystem_path_component(href): raise pathutils.UnsafePathError(href) path = pathutils.path_to_filesystem(self._filesystem_path, href) if not os.path.isfile(path): raise storage.ComponentNotFoundError(href) os.remove(path) self._sync_directory(os.path.dirname(path)) # Track the change self._update_history_etag(href, None) self._clean_history()
def get(self, environ, base_prefix, path, user): assert path == "/.web" or path.startswith("/.web/") assert pathutils.sanitize_path(path) == path try: filesystem_path = pathutils.path_to_filesystem( self.folder, path[len("/.web"):].strip("/")) except ValueError as e: logger.debug("Web content with unsafe path %r requested: %s", path, e, exc_info=True) return httputils.NOT_FOUND if os.path.isdir(filesystem_path) and not path.endswith("/"): location = posixpath.basename(path) + "/" return (client.FOUND, { "Location": location, "Content-Type": "text/plain" }, "Redirected to %s" % location) if os.path.isdir(filesystem_path): filesystem_path = os.path.join(filesystem_path, "index.html") if not os.path.isfile(filesystem_path): return httputils.NOT_FOUND content_type = MIMETYPES.get( os.path.splitext(filesystem_path)[1].lower(), FALLBACK_MIMETYPE) with open(filesystem_path, "rb") as f: answer = f.read() last_modified = time.strftime( "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(os.fstat(f.fileno()).st_mtime)) headers = { "Content-Type": content_type, "Last-Modified": last_modified } return client.OK, headers, answer
def _upload_all_nonatomic(self, items, suffix=""): """Upload a new set of items. This takes a list of vobject items and uploads them nonatomic and without existence checks. """ cache_folder = os.path.join(self._filesystem_path, ".Radicale.cache", "item") self._storage._makedirs_synced(cache_folder) hrefs = set() for item in items: uid = item.uid try: cache_content = self._item_cache_content(item) except Exception as e: raise ValueError( "Failed to store item %r in temporary collection %r: %s" % (uid, self.path, e)) from e href_candidate_funtions = [] if os.name in ("nt", "posix"): href_candidate_funtions.append(lambda: uid if uid.lower( ).endswith(suffix.lower()) else uid + suffix) href_candidate_funtions.extend( (lambda: radicale_item.get_etag(uid).strip('"') + suffix, lambda: radicale_item.find_available_uid( hrefs.__contains__, suffix))) href = f = None while href_candidate_funtions: href = href_candidate_funtions.pop(0)() if href in hrefs: continue if not pathutils.is_safe_filesystem_path_component(href): if not href_candidate_funtions: raise pathutils.UnsafePathError(href) continue try: f = open(pathutils.path_to_filesystem( self._filesystem_path, href), "w", newline="", encoding=self._encoding) break except OSError as e: if href_candidate_funtions and ( os.name == "posix" and e.errno == 22 or os.name == "nt" and e.errno == 123): continue raise with f: f.write(item.serialize()) f.flush() self._storage._fsync(f) hrefs.add(href) with open(os.path.join(cache_folder, href), "wb") as f: pickle.dump(cache_content, f) f.flush() self._storage._fsync(f) self._storage._sync_directory(cache_folder) self._storage._sync_directory(self._filesystem_path)
def __init__(self, path, filesystem_path=None): folder = self._get_collection_root_folder() # Path should already be sanitized self.path = pathutils.strip_path(path) self._encoding = self.configuration.get("encoding", "stock") if filesystem_path is None: filesystem_path = pathutils.path_to_filesystem(folder, self.path) self._filesystem_path = filesystem_path self._etag_cache = None super().__init__()
def __init__(self, path, filesystem_path=None): folder = self._get_collection_root_folder() # Path should already be sanitized self.path = pathutils.strip_path(path) self._encoding = self.configuration.get("encoding", "stock") if filesystem_path is None: filesystem_path = pathutils.path_to_filesystem(folder, self.path) self._filesystem_path = filesystem_path self._props_path = os.path.join(self._filesystem_path, ".Radicale.props") self._meta_cache = None self._etag_cache = None self._item_cache_cleaned = False
def upload(self, href, item): if not pathutils.is_safe_filesystem_path_component(href): raise pathutils.UnsafePathError(href) try: self._store_item_cache(href, item) except Exception as e: raise ValueError("Failed to store item %r in collection %r: %s" % (href, self.path, e)) from e path = pathutils.path_to_filesystem(self._filesystem_path, href) with self._atomic_write(path, newline="") as fd: fd.write(item.serialize()) # Clean the cache after the actual item is stored, or the cache entry # will be removed again. self._clean_item_cache() # Track the change self._update_history_etag(href, item) self._clean_history() return self._get(href, verify_href=False)
def replace_fn(source, target): nonlocal href while href_candidates: href = href_candidates.pop(0)() if href in hrefs: continue if not pathutils.is_safe_filesystem_path_component(href): if not href_candidates: raise pathutils.UnsafePathError(href) continue try: return os.replace(source, pathutils.path_to_filesystem( self._filesystem_path, href)) except OSError as e: if href_candidates and ( os.name == "posix" and e.errno == 22 or os.name == "nt" and e.errno == 123): continue raise
def create_collection(self, href, items=None, props=None): folder = self._get_collection_root_folder() # Path should already be sanitized sane_path = pathutils.strip_path(href) filesystem_path = pathutils.path_to_filesystem(folder, sane_path) if not props: self._makedirs_synced(filesystem_path) return self._collection_class( self, pathutils.unstrip_path(sane_path, True)) parent_dir = os.path.dirname(filesystem_path) self._makedirs_synced(parent_dir) # Create a temporary directory with an unsafe name with TemporaryDirectory(prefix=".Radicale.tmp-", dir=parent_dir) as tmp_dir: # The temporary directory itself can't be renamed tmp_filesystem_path = os.path.join(tmp_dir, "collection") os.makedirs(tmp_filesystem_path) col = self._collection_class(self, pathutils.unstrip_path( sane_path, True), filesystem_path=tmp_filesystem_path) col.set_meta(props) if items is not None: if props.get("tag") == "VCALENDAR": col._upload_all_nonatomic(items, suffix=".ics") elif props.get("tag") == "VADDRESSBOOK": col._upload_all_nonatomic(items, suffix=".vcf") # This operation is not atomic on the filesystem level but it's # very unlikely that one rename operations succeeds while the # other fails or that only one gets written to disk. if os.path.exists(filesystem_path): os.rename(filesystem_path, os.path.join(tmp_dir, "delete")) os.rename(tmp_filesystem_path, filesystem_path) self._sync_directory(parent_dir) return self._collection_class(self, pathutils.unstrip_path(sane_path, True))
def create_collection(self, href, items=None, props=None): folder = self._get_collection_root_folder() # Path should already be sanitized sane_path = pathutils.strip_path(href) filesystem_path = pathutils.path_to_filesystem(folder, sane_path) if not props: self._makedirs_synced(filesystem_path) return self._collection_class( self, pathutils.unstrip_path(sane_path, True)) parent_dir = os.path.dirname(filesystem_path) self._makedirs_synced(parent_dir) # Create a temporary directory with an unsafe name with TemporaryDirectory( prefix=".Radicale.tmp-", dir=parent_dir) as tmp_dir: # The temporary directory itself can't be renamed tmp_filesystem_path = os.path.join(tmp_dir, "collection") os.makedirs(tmp_filesystem_path) col = self._collection_class( self, pathutils.unstrip_path(sane_path, True), filesystem_path=tmp_filesystem_path) col.set_meta(props) if items is not None: if props.get("tag") == "VCALENDAR": col._upload_all_nonatomic(items, suffix=".ics") elif props.get("tag") == "VADDRESSBOOK": col._upload_all_nonatomic(items, suffix=".vcf") if os.path.lexists(filesystem_path): pathutils.rename_exchange(tmp_filesystem_path, filesystem_path) else: os.rename(tmp_filesystem_path, filesystem_path) self._sync_directory(parent_dir) return self._collection_class( self, pathutils.unstrip_path(sane_path, True))
def _get(self, href, verify_href=True): if verify_href: try: if not pathutils.is_safe_filesystem_path_component(href): raise pathutils.UnsafePathError(href) path = pathutils.path_to_filesystem(self._filesystem_path, href) except ValueError as e: logger.debug( "Can't translate name %r safely to filesystem in %r: %s", href, self.path, e, exc_info=True) return None else: path = os.path.join(self._filesystem_path, href) try: with open(path, "rb") as f: raw_text = f.read() except (FileNotFoundError, IsADirectoryError): return None except PermissionError: # Windows raises ``PermissionError`` when ``path`` is a directory if (os.name == "nt" and os.path.isdir(path) and os.access(path, os.R_OK)): return None raise # The hash of the component in the file system. This is used to check, # if the entry in the cache is still valid. input_hash = self._item_cache_hash(raw_text) cache_hash, uid, etag, text, name, tag, start, end = \ self._load_item_cache(href, input_hash) if input_hash != cache_hash: with self._acquire_cache_lock("item"): # Lock the item cache to prevent multpile processes from # generating the same data in parallel. # This improves the performance for multiple requests. if self._lock.locked == "r": # Check if another process created the file in the meantime cache_hash, uid, etag, text, name, tag, start, end = \ self._load_item_cache(href, input_hash) if input_hash != cache_hash: try: vobject_items = tuple( vobject.readComponents( raw_text.decode(self._encoding))) radicale_item.check_and_sanitize_items( vobject_items, tag=self.get_meta("tag")) vobject_item, = vobject_items temp_item = radicale_item.Item( collection=self, vobject_item=vobject_item) cache_hash, uid, etag, text, name, tag, start, end = \ self._store_item_cache( href, temp_item, input_hash) except Exception as e: raise RuntimeError("Failed to load item %r in %r: %s" % (href, self.path, e)) from e # Clean cache entries once after the data in the file # system was edited externally. if not self._item_cache_cleaned: self._item_cache_cleaned = True self._clean_item_cache() last_modified = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(os.path.getmtime(path))) # Don't keep reference to ``vobject_item``, because it requires a lot # of memory. return radicale_item.Item(collection=self, href=href, last_modified=last_modified, etag=etag, text=text, uid=uid, name=name, component_name=tag, time_range=(start, end))
def _get(self, href, verify_href=True): if verify_href: try: if not pathutils.is_safe_filesystem_path_component(href): raise pathutils.UnsafePathError(href) path = pathutils.path_to_filesystem( self._filesystem_path, href) except ValueError as e: logger.debug( "Can't translate name %r safely to filesystem in %r: %s", href, self.path, e, exc_info=True) return None else: path = os.path.join(self._filesystem_path, href) try: with open(path, "rb") as f: raw_text = f.read() except (FileNotFoundError, IsADirectoryError): return None except PermissionError: # Windows raises ``PermissionError`` when ``path`` is a directory if (os.name == "nt" and os.path.isdir(path) and os.access(path, os.R_OK)): return None raise # The hash of the component in the file system. This is used to check, # if the entry in the cache is still valid. input_hash = self._item_cache_hash(raw_text) cache_hash, uid, etag, text, name, tag, start, end = \ self._load_item_cache(href, input_hash) if input_hash != cache_hash: with self._acquire_cache_lock("item"): # Lock the item cache to prevent multpile processes from # generating the same data in parallel. # This improves the performance for multiple requests. if self._lock.locked == "r": # Check if another process created the file in the meantime cache_hash, uid, etag, text, name, tag, start, end = \ self._load_item_cache(href, input_hash) if input_hash != cache_hash: try: vobject_items = tuple(vobject.readComponents( raw_text.decode(self._encoding))) radicale_item.check_and_sanitize_items( vobject_items, tag=self.get_meta("tag")) vobject_item, = vobject_items temp_item = radicale_item.Item( collection=self, vobject_item=vobject_item) cache_hash, uid, etag, text, name, tag, start, end = \ self._store_item_cache( href, temp_item, input_hash) except Exception as e: raise RuntimeError("Failed to load item %r in %r: %s" % (href, self.path, e)) from e # Clean cache entries once after the data in the file # system was edited externally. if not self._item_cache_cleaned: self._item_cache_cleaned = True self._clean_item_cache() last_modified = time.strftime( "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(os.path.getmtime(path))) # Don't keep reference to ``vobject_item``, because it requires a lot # of memory. return radicale_item.Item( collection=self, href=href, last_modified=last_modified, etag=etag, text=text, uid=uid, name=name, component_name=tag, time_range=(start, end))