def test_cachew_dir_none() -> None: from cachew import settings settings.ENABLE = True # by default it's off in tests (see conftest.py) from my.core.cachew import cache_dir from my.core.common import mcachew from my.core.core_config import _reset_config as reset with reset() as cc: cc.cache_dir = None called = 0 @mcachew(cache_path=cache_dir() / 'ctest') def cf() -> List[int]: nonlocal called called += 1 return [called, called, called] assert list(cf()) == [1, 1, 1] assert list(cf()) == [2, 2, 2]
class Query: def __init__(self, files: Sequence[Path]) -> None: self.files = files # TODO yield errors? @mcachew( cache_path=lambda _, f: cache_dir() / 'orgmode' / _sanitize(f), force_file=True, depends_on=lambda _, f: (f, f.stat().st_mtime), ) def _iterate(self, f: Path) -> Iterable[OrgNote]: o = orgparse.load(f) for x in o: yield to_note(x) def all(self) -> Iterable[OrgNote]: # TODO build a virtual hierarchy from it? for f in self.files: yield from self._iterate(f) def collect_all(self, collector) -> Iterable[orgparse.OrgNode]: for f in self.files: o = orgparse.load(f) yield from collect(o, collector)
def _cached_commits_path(p: Path) -> str: p = cache_dir() / 'my.coding.commits:_cached_commits' / str( p.absolute()).strip("/") p.mkdir(parents=True, exist_ok=True) return str(p)
# fixme: later, rely on the timezone provider # NOTE: the timezone should be set with respect to the export date!!! import pytz # type: ignore tz = pytz.timezone('Europe/London') # TODO when I change tz, check the diff def is_bad_table(name: str) -> bool: # todo hmm would be nice to have a hook that can patch any module up to delegate = getattr(config, 'is_bad_table', None) return False if delegate is None else delegate(name) from my.core.cachew import cache_dir from my.core.common import mcachew @mcachew(depends_on=lambda: inputs(), cache_path=cache_dir('bluemaestro')) def measurements() -> Iterable[Res[Measurement]]: # todo ideally this would be via arguments... but needs to be lazy dbs = inputs() last: Optional[datetime] = None # tables are immutable, so can save on processing.. processed_tables: Set[str] = set() for f in dbs: logger.debug('processing %s', f) tot = 0 new = 0 # todo assert increasing timestamp? with sqlite_connect_immutable(f) as db: db_dt: Optional[datetime] = None
REQUIRES = ["cssselect", "lxml"] from itertools import chain from typing import Set from my.core.common import Stats, LazyLogger, mcachew, warn_if_empty from my.core.cachew import cache_dir from .paths import takeout_input_directories from .takeout_parser import Results, parse_takeout logger = LazyLogger(__name__, level="warning") @mcachew( cache_path=lambda: str(cache_dir() / "_merged_google_events"), depends_on=lambda: list(sorted(takeout_input_directories())), force_file=True, logger=logger, ) def events() -> Results: yield from merge_events(*map(parse_takeout, takeout_input_directories())) @warn_if_empty def merge_events(*sources: Results) -> Results: emitted: Set[int] = set() for event in chain(*sources): if isinstance(event, Exception): yield event continue
yield LikedVideo( title=jlike["snippet"]["title"], desc=jlike["snippet"]["description"], link="https://youtube.com/watch?v={}".format( jlike["contentDetails"]["videoId"] ), dt=_parse_json_date(jlike["snippet"]["publishedAt"]), ) def _parse_html_chat_li(p: Path) -> Iterator[Res[HtmlComment]]: yield from read_html_li(p) @mcachew( cache_path=lambda p: str(cache_dir() / "_parse_html_activity" / simplify_path(p)), force_file=True, logger=logger, ) def _parse_html_activity(p: Path) -> Iterator[Res[HtmlEvent]]: yield from read_html_activity(p) @mcachew( cache_path=lambda p: str( cache_dir() / "_parse_json_youtube_history" / simplify_path(p) ), force_file=True, logger=logger, ) def _parse_json_youtube_history(p: Path) -> Iterator[HtmlEvent]:
def _cached_commits_path(p: Path) -> str: # compute a reduced simple filepath using the absolute path of the repo simple_path = ''.join(filter(lambda c: c in _allowed_letters, str(p.absolute()))) return str(cache_dir() / 'commits' / simple_path / '_cached_commits')