def discover(cls, path, depth="0", child_context_manager=( lambda path, href=None: contextlib.ExitStack())): # Path should already be sanitized sane_path = pathutils.strip_path(path) attributes = sane_path.split("/") if sane_path else [] folder = cls._get_collection_root_folder() # Create the root collection cls._makedirs_synced(folder) try: filesystem_path = pathutils.path_to_filesystem(folder, sane_path) except ValueError as e: # Path is unsafe logger.debug("Unsafe path %r requested from storage: %s", sane_path, e, exc_info=True) return # Check if the path exists and if it leads to a collection or an item if not os.path.isdir(filesystem_path): if attributes and os.path.isfile(filesystem_path): href = attributes.pop() else: return else: href = None sane_path = "/".join(attributes) collection = cls(pathutils.unstrip_path(sane_path, True)) if href: yield collection._get(href) return yield collection if depth == "0": return for href in collection._list(): with child_context_manager(sane_path, href): yield collection._get(href) for entry in os.scandir(filesystem_path): if not entry.is_dir(): continue href = entry.name if not pathutils.is_safe_filesystem_path_component(href): if not href.startswith(".Radicale"): logger.debug("Skipping collection %r in %r", href, sane_path) continue sane_child_path = posixpath.join(sane_path, href) child_path = pathutils.unstrip_path(sane_child_path, True) with child_context_manager(sane_child_path): yield cls(child_path)
def move(self, item, to_collection, to_href): if not pathutils.is_safe_filesystem_path_component(to_href): raise pathutils.UnsafePathError(to_href) os.replace( pathutils.path_to_filesystem(item.collection._filesystem_path, item.href), pathutils.path_to_filesystem(to_collection._filesystem_path, to_href)) self._sync_directory(to_collection._filesystem_path) if item.collection._filesystem_path != to_collection._filesystem_path: self._sync_directory(item.collection._filesystem_path) # Move the item cache entry cache_folder = os.path.join(item.collection._filesystem_path, ".Radicale.cache", "item") to_cache_folder = os.path.join(to_collection._filesystem_path, ".Radicale.cache", "item") self._makedirs_synced(to_cache_folder) try: os.replace(os.path.join(cache_folder, item.href), os.path.join(to_cache_folder, to_href)) except FileNotFoundError: pass else: self._makedirs_synced(to_cache_folder) if cache_folder != to_cache_folder: self._makedirs_synced(cache_folder) # Track the change to_collection._update_history_etag(to_href, item) item.collection._update_history_etag(item.href, None) to_collection._clean_history() if item.collection._filesystem_path != to_collection._filesystem_path: item.collection._clean_history()
def delete(self, href=None): if href is None: # Delete the collection parent_dir = os.path.dirname(self._filesystem_path) try: os.rmdir(self._filesystem_path) except OSError: with TemporaryDirectory( prefix=".Radicale.tmp-", dir=parent_dir) as tmp: os.rename(self._filesystem_path, os.path.join( tmp, os.path.basename(self._filesystem_path))) self._sync_directory(parent_dir) else: self._sync_directory(parent_dir) else: # Delete an item if not pathutils.is_safe_filesystem_path_component(href): raise pathutils.UnsafePathError(href) path = pathutils.path_to_filesystem(self._filesystem_path, href) if not os.path.isfile(path): raise storage.ComponentNotFoundError(href) os.remove(path) self._sync_directory(os.path.dirname(path)) # Track the change self._update_history_etag(href, None) self._clean_history()
def _clean_cache(cls, folder, names, max_age=None): """Delete all ``names`` in ``folder`` that are older than ``max_age``. """ age_limit = time.time() - max_age if max_age is not None else None modified = False for name in names: if not pathutils.is_safe_filesystem_path_component(name): continue if age_limit is not None: try: # Race: Another process might have deleted the file. mtime = os.path.getmtime(os.path.join(folder, name)) except FileNotFoundError: continue if mtime > age_limit: continue logger.debug("Found expired item in cache: %r", name) # Race: Another process might have deleted or locked the # file. try: os.remove(os.path.join(folder, name)) except (FileNotFoundError, PermissionError): continue modified = True if modified: cls._sync_directory(folder)
def _upload_all_nonatomic(self, items, suffix=""): """Upload a new set of items. This takes a list of vobject items and uploads them nonatomic and without existence checks. """ cache_folder = os.path.join(self._filesystem_path, ".Radicale.cache", "item") self._storage._makedirs_synced(cache_folder) hrefs = set() for item in items: uid = item.uid try: cache_content = self._item_cache_content(item) except Exception as e: raise ValueError( "Failed to store item %r in temporary collection %r: %s" % (uid, self.path, e)) from e href_candidate_funtions = [] if os.name in ("nt", "posix"): href_candidate_funtions.append(lambda: uid if uid.lower( ).endswith(suffix.lower()) else uid + suffix) href_candidate_funtions.extend( (lambda: radicale_item.get_etag(uid).strip('"') + suffix, lambda: radicale_item.find_available_uid( hrefs.__contains__, suffix))) href = f = None while href_candidate_funtions: href = href_candidate_funtions.pop(0)() if href in hrefs: continue if not pathutils.is_safe_filesystem_path_component(href): if not href_candidate_funtions: raise pathutils.UnsafePathError(href) continue try: f = open(pathutils.path_to_filesystem( self._filesystem_path, href), "w", newline="", encoding=self._encoding) break except OSError as e: if href_candidate_funtions and ( os.name == "posix" and e.errno == 22 or os.name == "nt" and e.errno == 123): continue raise with f: f.write(item.serialize()) f.flush() self._storage._fsync(f) hrefs.add(href) with open(os.path.join(cache_folder, href), "wb") as f: pickle.dump(cache_content, f) f.flush() self._storage._fsync(f) self._storage._sync_directory(cache_folder) self._storage._sync_directory(self._filesystem_path)
def move(cls, item, to_collection, to_href): if not pathutils.is_safe_filesystem_path_component(to_href): raise pathutils.UnsafePathError(to_href) os.replace( pathutils.path_to_filesystem( item.collection._filesystem_path, item.href), pathutils.path_to_filesystem( to_collection._filesystem_path, to_href)) cls._sync_directory(to_collection._filesystem_path) if item.collection._filesystem_path != to_collection._filesystem_path: cls._sync_directory(item.collection._filesystem_path) # Move the item cache entry cache_folder = os.path.join(item.collection._filesystem_path, ".Radicale.cache", "item") to_cache_folder = os.path.join(to_collection._filesystem_path, ".Radicale.cache", "item") cls._makedirs_synced(to_cache_folder) try: os.replace(os.path.join(cache_folder, item.href), os.path.join(to_cache_folder, to_href)) except FileNotFoundError: pass else: cls._makedirs_synced(to_cache_folder) if cache_folder != to_cache_folder: cls._makedirs_synced(cache_folder) # Track the change to_collection._update_history_etag(to_href, item) item.collection._update_history_etag(item.href, None) to_collection._clean_history() if item.collection._filesystem_path != to_collection._filesystem_path: item.collection._clean_history()
def _list(self): for entry in os.scandir(self._filesystem_path): if not entry.is_file(): continue href = entry.name if not pathutils.is_safe_filesystem_path_component(href): if not href.startswith(".Radicale"): logger.debug("Skipping item %r in %r", href, self.path) continue yield href
def _get_deleted_history_hrefs(self): """Returns the hrefs of all deleted items that are still in the history cache.""" history_folder = os.path.join(self._filesystem_path, ".Radicale.cache", "history") try: for entry in os.scandir(history_folder): href = entry.name if not pathutils.is_safe_filesystem_path_component(href): continue if os.path.isfile(os.path.join(self._filesystem_path, href)): continue yield href except FileNotFoundError: pass
def get_multi(self, hrefs): # It's faster to check for file name collissions here, because # we only need to call os.listdir once. files = None for href in hrefs: if files is None: # List dir after hrefs returned one item, the iterator may be # empty and the for-loop is never executed. files = os.listdir(self._filesystem_path) path = os.path.join(self._filesystem_path, href) if (not pathutils.is_safe_filesystem_path_component(href) or href not in files and os.path.lexists(path)): logger.debug( "Can't translate name safely to filesystem: %r", href) yield (href, None) else: yield (href, self._get(href, verify_href=False))
def get_multi(self, hrefs): # It's faster to check for file name collissions here, because # we only need to call os.listdir once. files = None for href in hrefs: if files is None: # List dir after hrefs returned one item, the iterator may be # empty and the for-loop is never executed. files = os.listdir(self._filesystem_path) path = os.path.join(self._filesystem_path, href) if (not pathutils.is_safe_filesystem_path_component(href) or href not in files and os.path.lexists(path)): logger.debug("Can't translate name safely to filesystem: %r", href) yield (href, None) else: yield (href, self._get(href, verify_href=False))
def upload(self, href, item): if not pathutils.is_safe_filesystem_path_component(href): raise pathutils.UnsafePathError(href) try: self._store_item_cache(href, item) except Exception as e: raise ValueError("Failed to store item %r in collection %r: %s" % (href, self.path, e)) from e path = pathutils.path_to_filesystem(self._filesystem_path, href) with self._atomic_write(path, newline="") as fd: fd.write(item.serialize()) # Clean the cache after the actual item is stored, or the cache entry # will be removed again. self._clean_item_cache() # Track the change self._update_history_etag(href, item) self._clean_history() return self._get(href, verify_href=False)
def replace_fn(source, target): nonlocal href while href_candidates: href = href_candidates.pop(0)() if href in hrefs: continue if not pathutils.is_safe_filesystem_path_component(href): if not href_candidates: raise pathutils.UnsafePathError(href) continue try: return os.replace(source, pathutils.path_to_filesystem( self._filesystem_path, href)) except OSError as e: if href_candidates and ( os.name == "posix" and e.errno == 22 or os.name == "nt" and e.errno == 123): continue raise
def _get(self, href, verify_href=True): if verify_href: try: if not pathutils.is_safe_filesystem_path_component(href): raise pathutils.UnsafePathError(href) path = pathutils.path_to_filesystem(self._filesystem_path, href) except ValueError as e: logger.debug( "Can't translate name %r safely to filesystem in %r: %s", href, self.path, e, exc_info=True) return None else: path = os.path.join(self._filesystem_path, href) try: with open(path, "rb") as f: raw_text = f.read() except (FileNotFoundError, IsADirectoryError): return None except PermissionError: # Windows raises ``PermissionError`` when ``path`` is a directory if (os.name == "nt" and os.path.isdir(path) and os.access(path, os.R_OK)): return None raise # The hash of the component in the file system. This is used to check, # if the entry in the cache is still valid. input_hash = self._item_cache_hash(raw_text) cache_hash, uid, etag, text, name, tag, start, end = \ self._load_item_cache(href, input_hash) if input_hash != cache_hash: with self._acquire_cache_lock("item"): # Lock the item cache to prevent multpile processes from # generating the same data in parallel. # This improves the performance for multiple requests. if self._lock.locked == "r": # Check if another process created the file in the meantime cache_hash, uid, etag, text, name, tag, start, end = \ self._load_item_cache(href, input_hash) if input_hash != cache_hash: try: vobject_items = tuple( vobject.readComponents( raw_text.decode(self._encoding))) radicale_item.check_and_sanitize_items( vobject_items, tag=self.get_meta("tag")) vobject_item, = vobject_items temp_item = radicale_item.Item( collection=self, vobject_item=vobject_item) cache_hash, uid, etag, text, name, tag, start, end = \ self._store_item_cache( href, temp_item, input_hash) except Exception as e: raise RuntimeError("Failed to load item %r in %r: %s" % (href, self.path, e)) from e # Clean cache entries once after the data in the file # system was edited externally. if not self._item_cache_cleaned: self._item_cache_cleaned = True self._clean_item_cache() last_modified = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(os.path.getmtime(path))) # Don't keep reference to ``vobject_item``, because it requires a lot # of memory. return radicale_item.Item(collection=self, href=href, last_modified=last_modified, etag=etag, text=text, uid=uid, name=name, component_name=tag, time_range=(start, end))
def _get(self, href, verify_href=True): if verify_href: try: if not pathutils.is_safe_filesystem_path_component(href): raise pathutils.UnsafePathError(href) path = pathutils.path_to_filesystem( self._filesystem_path, href) except ValueError as e: logger.debug( "Can't translate name %r safely to filesystem in %r: %s", href, self.path, e, exc_info=True) return None else: path = os.path.join(self._filesystem_path, href) try: with open(path, "rb") as f: raw_text = f.read() except (FileNotFoundError, IsADirectoryError): return None except PermissionError: # Windows raises ``PermissionError`` when ``path`` is a directory if (os.name == "nt" and os.path.isdir(path) and os.access(path, os.R_OK)): return None raise # The hash of the component in the file system. This is used to check, # if the entry in the cache is still valid. input_hash = self._item_cache_hash(raw_text) cache_hash, uid, etag, text, name, tag, start, end = \ self._load_item_cache(href, input_hash) if input_hash != cache_hash: with self._acquire_cache_lock("item"): # Lock the item cache to prevent multpile processes from # generating the same data in parallel. # This improves the performance for multiple requests. if self._lock.locked == "r": # Check if another process created the file in the meantime cache_hash, uid, etag, text, name, tag, start, end = \ self._load_item_cache(href, input_hash) if input_hash != cache_hash: try: vobject_items = tuple(vobject.readComponents( raw_text.decode(self._encoding))) radicale_item.check_and_sanitize_items( vobject_items, tag=self.get_meta("tag")) vobject_item, = vobject_items temp_item = radicale_item.Item( collection=self, vobject_item=vobject_item) cache_hash, uid, etag, text, name, tag, start, end = \ self._store_item_cache( href, temp_item, input_hash) except Exception as e: raise RuntimeError("Failed to load item %r in %r: %s" % (href, self.path, e)) from e # Clean cache entries once after the data in the file # system was edited externally. if not self._item_cache_cleaned: self._item_cache_cleaned = True self._clean_item_cache() last_modified = time.strftime( "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(os.path.getmtime(path))) # Don't keep reference to ``vobject_item``, because it requires a lot # of memory. return radicale_item.Item( collection=self, href=href, last_modified=last_modified, etag=etag, text=text, uid=uid, name=name, component_name=tag, time_range=(start, end))