def test_cachew_dir_none() -> None:
from cachew import settings
settings.ENABLE = True # by default it's off in tests (see conftest.py)
from my.core.cachew import cache_dir
from my.core.common import mcachew
from my.core.core_config import _reset_config as reset
with reset() as cc:
cc.cache_dir = None
called = 0
@mcachew(cache_path=cache_dir() / 'ctest')
def cf() -> List[int]:
nonlocal called
called += 1
return [called, called, called]
assert list(cf()) == [1, 1, 1]
assert list(cf()) == [2, 2, 2]
class Query:
def __init__(self, files: Sequence[Path]) -> None:
self.files = files
# TODO yield errors?
@mcachew(
cache_path=lambda _, f: cache_dir() / 'orgmode' / _sanitize(f),
force_file=True,
depends_on=lambda _, f: (f, f.stat().st_mtime),
)
def _iterate(self, f: Path) -> Iterable[OrgNote]:
o = orgparse.load(f)
for x in o:
yield to_note(x)
def all(self) -> Iterable[OrgNote]:
# TODO build a virtual hierarchy from it?
for f in self.files:
yield from self._iterate(f)
def collect_all(self, collector) -> Iterable[orgparse.OrgNode]:
for f in self.files:
o = orgparse.load(f)
yield from collect(o, collector)
def _cached_commits_path(p: Path) -> str:
p = cache_dir() / 'my.coding.commits:_cached_commits' / str(
p.absolute()).strip("/")
p.mkdir(parents=True, exist_ok=True)
return str(p)
# fixme: later, rely on the timezone provider
# NOTE: the timezone should be set with respect to the export date!!!
import pytz # type: ignore
tz = pytz.timezone('Europe/London')
# TODO when I change tz, check the diff
def is_bad_table(name: str) -> bool:
# todo hmm would be nice to have a hook that can patch any module up to
delegate = getattr(config, 'is_bad_table', None)
return False if delegate is None else delegate(name)
from my.core.cachew import cache_dir
from my.core.common import mcachew
@mcachew(depends_on=lambda: inputs(), cache_path=cache_dir('bluemaestro'))
def measurements() -> Iterable[Res[Measurement]]:
# todo ideally this would be via arguments... but needs to be lazy
dbs = inputs()
last: Optional[datetime] = None
# tables are immutable, so can save on processing..
processed_tables: Set[str] = set()
for f in dbs:
logger.debug('processing %s', f)
tot = 0
new = 0
# todo assert increasing timestamp?
with sqlite_connect_immutable(f) as db:
db_dt: Optional[datetime] = None
REQUIRES = ["cssselect", "lxml"]
from itertools import chain
from typing import Set
from my.core.common import Stats, LazyLogger, mcachew, warn_if_empty
from my.core.cachew import cache_dir
from .paths import takeout_input_directories
from .takeout_parser import Results, parse_takeout
logger = LazyLogger(__name__, level="warning")
@mcachew(
cache_path=lambda: str(cache_dir() / "_merged_google_events"),
depends_on=lambda: list(sorted(takeout_input_directories())),
force_file=True,
logger=logger,
)
def events() -> Results:
yield from merge_events(*map(parse_takeout, takeout_input_directories()))
@warn_if_empty
def merge_events(*sources: Results) -> Results:
emitted: Set[int] = set()
for event in chain(*sources):
if isinstance(event, Exception):
yield event
continue
yield LikedVideo(
title=jlike["snippet"]["title"],
desc=jlike["snippet"]["description"],
link="https://youtube.com/watch?v={}".format(
jlike["contentDetails"]["videoId"]
),
dt=_parse_json_date(jlike["snippet"]["publishedAt"]),
)
def _parse_html_chat_li(p: Path) -> Iterator[Res[HtmlComment]]:
yield from read_html_li(p)
@mcachew(
cache_path=lambda p: str(cache_dir() / "_parse_html_activity" / simplify_path(p)),
force_file=True,
logger=logger,
)
def _parse_html_activity(p: Path) -> Iterator[Res[HtmlEvent]]:
yield from read_html_activity(p)
@mcachew(
cache_path=lambda p: str(
cache_dir() / "_parse_json_youtube_history" / simplify_path(p)
),
force_file=True,
logger=logger,
)
def _parse_json_youtube_history(p: Path) -> Iterator[HtmlEvent]:
def _cached_commits_path(p: Path) -> str:
# compute a reduced simple filepath using the absolute path of the repo
simple_path = ''.join(filter(lambda c: c in _allowed_letters, str(p.absolute())))
return str(cache_dir() / 'commits' / simple_path / '_cached_commits')