def _realize(b: Build) -> None: c = build_cattrs(b) o = build_outpath(b) try: assert path_ is not None partpath = join(o, fname) + '.tmp' fullpath = join(o, fname) copyfile(path_, partpath) assert isfile(partpath), f"Can't copy '{path_}' to '{partpath}'" with open(partpath, "rb") as f: realhash = sha256sum(f.read()).hexdigest() assert realhash == c.sha256, ( f"Expected sha256 checksum '{c.sha256}', " f"but got '{realhash}'") rename(partpath, fullpath) if 'unpack' in c.mode: _unpack_inplace(o, fullpath, 'remove' in c.mode) except Exception as e: error(f"Copying failed: {e}") error(f"Keeping temporary directory {o}") raise
def _make(b: Build) -> None: c = build_cattrs(b) o = build_outpath(b) download_dir = o if force_download else tmpfetchdir partpath = join(download_dir, filename_ + '.tmp') try: p = Popen( [CURL(), "--continue-at", "-", "--output", partpath, url], cwd=download_dir) p.wait() assert p.returncode == 0, f"Download failed, errcode '{p.returncode}'" assert isfile(partpath), f"Can't find output file '{partpath}'" with open(partpath, "rb") as f: if sha256 is not None: realhash = sha256sum(f.read()).hexdigest() assert realhash == c.sha256, ( f"Expected sha256 checksum '{c.sha256}', " f"but got '{realhash}'") if sha1 is not None: realhash = sha1sum(f.read()).hexdigest() assert realhash == c.sha1, ( f"Expected sha1 checksum '{c.sha1}', " f"but got '{realhash}'") fullpath = join(o, filename_) rename(partpath, fullpath) except Exception as e: error(f"Download failed: {e}") error(f"Keeping temporary directory {o}") raise
def _mark_status(outpaths: List[Dict[Tag, Path]], status: str, exception: Optional[str] = None) -> None: for rg in outpaths: o = rg[tag_out()] writestr(join(o, 'status_either.txt'), status) if exception is not None: writestr(join(o, 'exception.txt'), exception)
def dirrw(o: Path) -> None: for root, dirs, files in walk(o): for d in dirs: mode = stat(join(root, d))[ST_MODE] chmod(join(root, d), mode | (S_IWRITE)) for f in files: if isfile(f): filerw(Path(join(root, f))) if islink(f): warning( f"Pylightnix doesn't guarantee the consistency of symlink '{f}'" ) chmod(o, stat(o)[ST_MODE] | (S_IWRITE | S_IWGRP | S_IWOTH))
def gc_exceptions(keep_paths:List[Path])->Tuple[List[DRef],List[RRef]]: """ Scans `keep_paths` list for references to Pylightnix storage. Ignores unrelated filesystem objects. """ keep_drefs:List[DRef]=[] keep_rrefs:List[RRef]=[] def _check(f:str): nonlocal keep_drefs, keep_rrefs if islink(a): rref=path2rref(a) if rref is not None: keep_rrefs.append(rref) else: dref=path2dref(a) if dref is not None: keep_drefs.append(dref) for path in keep_paths: if islink(path): _check(path) elif isdir(path): for root, dirs, filenames in walk(path, topdown=True): for dirname in sorted(dirs): a=Path(abspath(join(root, dirname))) _check(a) else: pass return keep_drefs,keep_rrefs
def alldrefs(S=None) -> Iterable[DRef]: """ Iterates over all derivations of the storage located at `S` (PYLIGHTNIX_STORE env is used by default) """ store_path_ = storage(S) for dirname in listdir(store_path_): if dirname[-4:] != '.tmp' and isdir(join(store_path_, dirname)): yield mkdref(HashPart(dirname[:32]), Name(dirname[32 + 1:]))
def dirsize(o: Path) -> int: """ Return size in bytes """ total_size = 0 for dirpath, dirnames, filenames in walk(o): for f in filenames: fp = join(dirpath, f) if not islink(fp): total_size += getsize(fp) return total_size
def _key(gr: RRefGroup, S=None) -> Optional[Union[int, float, str]]: try: with open(join(store_rref2path(grouprref(gr), S), filename), 'r') as f: return float(f.readline()) except OSError: return float('-inf') except ValueError: return float('-inf')
def _key(gr: RRefGroup, S=None) -> Optional[Union[int, float, str]]: try: with open( join(store_rref2path(grouprref(gr), S), '__buildtime__.txt'), 'r') as f: t = parsetime(f.read()) return float(0 if t is None else t) except OSError: return float(0)
def _iter() -> Iterable[Tuple[str, bytes]]: for root, dirs, filenames in walk(abspath(path), topdown=True): for filename in sorted(filenames): if len(filename) > 0 and filename[0] != '_': localpath = abspath(join(root, filename)) if islink(localpath): yield (f'link:{localpath}', encode(readlink(localpath))) with open(localpath, 'rb') as f: yield (localpath, f.read())
def store_buildtime(rref: RRef, S=None) -> Optional[str]: """ Return the buildtime of the current RRef in a format specified by the [PYLIGHTNIX_TIME](#pylightnix.utils.PYLIGHTNIX_TIME) constant. [parsetime](#pylightnix.utils.parsetime) may be used to parse stings into UNIX-Epoch seconds. Buildtime is the time when the realization process has started. Some realizations may not provide this information. """ return tryread(Path(join(store_rref2path(rref, S), '__buildtime__.txt')))
def drefrrefs(dref: DRef, S=None) -> List[RRef]: """ Iterate over all realizations of a derivation `dref`. The sort order is unspecified. Matching is not taken into account. """ (dhash, nm) = undref(dref) drefpath = store_dref2path(dref, S) rrefs: List[RRef] = [] for f in listdir(drefpath): if f[-4:] != '.tmp' and isdir(join(drefpath, f)): rrefs.append(mkrref(HashPart(f), dhash, nm)) return rrefs
def linkdref(dref: DRef, destdir: Optional[Path] = None, name: Optional[str] = None, S=None) -> Path: destdir_ = '.' if destdir is None else destdir name_: str = name if name is not None else ( '_result-' + config_name(store_config(dref, S)) if destdir is None else 'result-' + config_name(store_config(dref, S))) symlink = Path(join(destdir_, name_)) forcelink(Path(relpath(store_dref2path(dref, S), destdir_)), symlink) return symlink
def _realize(b: Build) -> None: c = build_cattrs(b) o = build_outpath(b) download_dir = o if force_download else tmpfetchdir try: partpath = join(download_dir, fname + '.tmp') p = Popen( [WGET(), "--continue", '--output-document', partpath, c.url], cwd=download_dir) p.wait() assert p.returncode == 0, f"Download failed, errcode '{p.returncode}'" assert isfile(partpath), f"Can't find output file '{partpath}'" with open(partpath, "rb") as f: if sha256 is not None: realhash = sha256sum(f.read()).hexdigest() assert realhash == c.sha256, ( f"Expected sha256 checksum '{c.sha256}', " f"but got '{realhash}'") elif sha1 is not None: realhash = sha1sum(f.read()).hexdigest() assert realhash == c.sha1, ( f"Expected sha1 checksum '{c.sha1}', " f"but got '{realhash}'") else: assert False, 'Either sha256 or sha1 arguments should be set' fullpath = join(o, fname) rename(partpath, fullpath) if 'unpack' in c.mode: _unpack_inplace(o, fullpath, 'remove' in c.mode) except Exception as e: error(f"Download failed: {e}") error(f"Keeping temporary directory {o}") raise
def val2path(v:Any, ctx:LensContext)->Path: """ Resolve the current value of Lens into system path. Assert if it is not possible or if the result is associated with multiple paths.""" S:SPath=ctx.storage if isdref(v): dref=DRef(v) context=ctx.context if context is not None: if dref in context: rgs=context_deref(context, dref) assert len(rgs)==1, "Lens doesn't support multirealization dependencies" return Path(store_rref2path(rgs[0][tag_out()])) return store_dref2path(dref) elif isrref(v): return store_rref2path(RRef(v),S) elif isrefpath(v): refpath=list(v) # RefPath is list bpath=ctx.build_path context=ctx.context if context is not None: if refpath[0] in context: rgs=context_deref(context,refpath[0],S) assert len(rgs)==1, "Lens doesn't support multirealization dependencies" return Path(join(store_rref2path(rgs[0][Tag('out')],S), *refpath[1:])) else: if bpath is not None: # FIXME: should we assert on refpath[0]==build.dref ? return Path(join(bpath, *refpath[1:])) else: assert False, f"Can't dereference refpath {refpath}" else: assert False, f"Lens couldn't resolve '{refpath}' without a context" elif isinstance(v, Closure): return val2path(v.dref, ctx) elif isinstance(v, Build): assert ctx.build_path is not None, f"Lens can't access build path of '{v}'" return ctx.build_path else: assert False, f"Lens doesn't know how to resolve '{v}'"
def _either(S: SPath, dref: DRef, ctx: Context, ra: RealizeArg) -> List[Dict[Tag, Path]]: # Write the specified build status to every output def _mark_status(outpaths: List[Dict[Tag, Path]], status: str, exception: Optional[str] = None) -> None: for rg in outpaths: o = rg[tag_out()] writestr(join(o, 'status_either.txt'), status) if exception is not None: writestr(join(o, 'exception.txt'), exception) # Scan all immediate dependecnies of this build, propagate the 'LEFT' status # if any of them has it. for dref_dep in store_deps([dref], S): for rg in context_deref(ctx, dref_dep, S): rref = rg[tag_out()] status = tryreadstr_def( join(store_rref2path(rref, S), 'status_either.txt'), 'RIGHT') if status == 'RIGHT': continue elif status == 'LEFT': outpaths = [{ tag_out(): Path(mkdtemp(prefix="either_tmp", dir=tmp)) }] _mark_status(outpaths, 'LEFT') return outpaths else: assert False, f"Invalid either status {status}" # Execute the original build try: outpaths = f(S, dref, ctx, ra) _mark_status(outpaths, 'RIGHT') except KeyboardInterrupt: raise except BuildError as be: outpaths = be.outgroups _mark_status(outpaths, 'LEFT', format_exc()) except Exception: outpaths = [{ tag_out(): Path(mkdtemp(prefix="either_tmp", dir=tmp)) }] _mark_status(outpaths, 'LEFT', format_exc()) # Return either valid artifacts or a LEFT-substitute return outpaths
def mkdrv_(c: Config, S: SPath) -> DRef: """ Create new derivation in storage `S`. We attempt to do it atomically by creating temp directory first and then renaming it right into it's place in the storage. FIXME: Assert or handle possible (but improbable) hash collision [*] """ assert_store_initialized(S) # c=cp.config assert_valid_config(c) assert_rref_deps(c) refname = config_name(c) dhash = config_hash(c) dref = mkdref(trimhash(dhash), refname) o = Path(mkdtemp(prefix=refname, dir=tempdir())) with open(join(o, 'config.json'), 'w') as f: f.write(config_serialize(c)) filero(Path(join(o, 'config.json'))) drefpath = store_dref2path(dref, S) dreftmp = Path(drefpath + '.tmp') replace(o, dreftmp) try: replace(dreftmp, drefpath) except OSError as err: if err.errno == ENOTEMPTY: # Existing folder means that it has a matched content [*] dirrm(dreftmp, ignore_not_found=False) else: raise return dref
def mklogdir(tag: str, logrootdir: Path, subdirs: list = [], symlinks: bool = True, timetag: Optional[str] = None) -> Path: """ Create `<logrootdir>/<tag>_<time>` folder and set `<logrootdir>/_<tag>_latest` symlink to point to it. """ logpath = logdir(tag, logrootdir=logrootdir, timetag=timetag) makedirs(logpath, exist_ok=True) if symlinks: linkname = join( logrootdir, (('_' + str(tag) + '_latest') if len(tag) > 0 else '_latest')) try: symlink(basename(logpath), linkname) except OSError as e: if e.errno == EEXIST: remove(linkname) symlink(basename(logpath), linkname) else: raise e for sd in subdirs: makedirs(join(logpath, sd), exist_ok=True) return Path(logpath)
def du() -> Dict[DRef, Tuple[int, Dict[RRef, int]]]: """ Calculates the disk usage, in bytes. For every derivation, return it's total disk usage and disk usages per realizations. Note, that total disk usage of a derivation is slightly bigger than sum of it's realization's usages.""" res = {} for dref in alldrefs(): rref_res = {} dref_total = 0 for gr in store_rrefs_(dref): for rref in gr.values(): usage = dirsize(store_rref2path(rref)) rref_res[rref] = usage dref_total += usage dref_total += getsize(join(store_dref2path(dref), 'config.json')) res[dref] = (dref_total, rref_res) return res
def linkrref(rref: RRef, destdir: Optional[Path] = None, name: Optional[str] = None, withtime: bool = False, S=None) -> Path: """ Helper function that creates a symbolic link to a particular realization reference. The link is created under the current directory by default or under the `destdir` directory. Create a symlink pointing to realization `rref`. Other arguments define symlink name and location. Informally, `{tgtpath}/{timeprefix}{name} --> $PYLIGHTNIX_STORE/{dref}/{rref}`. Overwrite existing symlinks. Folder named `tgtpath` should exist. """ destdir_ = '.' if destdir is None else destdir name_: str = name if name is not None else ( '_result-' + config_name(store_config(rref, S)) if destdir is None else 'result-' + config_name(store_config(rref, S))) ts: Optional[str] = store_buildtime(rref, S) if withtime else None timetag_ = f'{ts}_' if ts is not None else '' symlink = Path(join(destdir_, f"{timetag_}{name_}")) forcelink(Path(relpath(store_rref2path(rref, S), destdir_)), symlink) return symlink
def _realize(b:Build)->None: o=build_outpath(b) for an,av in artifacts.items(): with open(join(o,an),'wb') as f: f.write(av)
def rrefdata(rref: RRef, S=None) -> Iterable[Path]: """ Return the paths of top-level artifacts """ root = store_rref2path(rref, S) for fd in scandir(root): if not (fd.is_file() and fd.name in PYLIGHTNIX_RESERVED): yield Path(join(root, fd.name))
def either_status(rref: RRef, S=None) -> str: return readstr(join(store_rref2path(rref, S), 'status_either.txt'))
def assert_promise_fulfilled(k: str, p: PromisePath, o: Path) -> None: ppath = join(o, *p[1:]) assert isfile(ppath) or isdir(ppath) or islink(ppath), ( f"Promise '{k}' of {p[0]} is not fulfilled. " f"{ppath} is expected to be a file or a directory.")
def mkrealization(dref: DRef, l: Context, o: Path, leader: Optional[Tuple[Tag, RRef]] = None, S=None) -> RRef: """ Create the [Realization](#pylightnix.types.RRef) object in the storage `S`. Return new Realization reference. Parameters: - `dref:DRef`: Derivation reference to create the realization of. - `l:Context`: Context which stores dependency information. - `o:Path`: Path to temporal (build) folder which contains artifacts, prepared by the [Realizer](#pylightnix.types.Realizer). - `leader`: Tag name and Group identifier of the Group leader. By default, we use name `out` and derivation's own rref. FIXME: Assert or handle possible but improbable hash collision[*] FIXME: Consider(not sure) writing group.json for all realizations[**] """ c = store_config(dref, S) assert_valid_config(c) (dhash, nm) = undref(dref) assert isdir(o), ( f"While realizing {dref}: Outpath is expected to be a path to existing " f"directory, but got {o}") for fn in PYLIGHTNIX_RESERVED: assert not isfile(join(o, fn)), ( f"While realizing {dref}: output folder '{o}' contains file '{fn}'. " f"This name is reserved, please use another name. List of reserved " f"names: {PYLIGHTNIX_RESERVED}") with open(reserved(o, 'context.json'), 'w') as f: f.write(context_serialize(l)) if leader is not None: # [**] tag, group_rref = leader with open(reserved(o, 'group.json'), 'w') as f: json_dump({'tag': tag, 'group': group_rref}, f) rhash = dirhash(o) rref = mkrref(trimhash(rhash), dhash, nm) rrefpath = store_rref2path(rref, S) rreftmp = Path(rrefpath + '.tmp') replace(o, rreftmp) dirchmod(rreftmp, 'ro') try: replace(rreftmp, rrefpath) except OSError as err: if err.errno == ENOTEMPTY: # Folder name contain the hash of the content, so getting here # probably[*] means that we already have this object in storage so we # just remove temp folder. dirrm(rreftmp, ignore_not_found=False) else: # Attempt to roll-back dirchmod(rreftmp, 'rw') replace(rreftmp, o) raise return rref
def lsdref_(r: DRef) -> Iterable[str]: p = store_dref2path(r) for d in listdir(p): p2 = join(d, p) if isdir(p2): yield d
def logdir(tag: str, logrootdir: Path, timetag: Optional[str] = None): timetag = timestring() if timetag is None else timetag return join(logrootdir, ((str(tag) + '_') if len(tag) > 0 else '') + timetag)
def lsrref_(r: RRef, fn: List[str] = []) -> Iterable[str]: p = join(store_rref2path(r), *fn) for d in listdir(p): yield d
def catrref_(r: RRef, fn: List[str]) -> Iterable[str]: with open(join(store_rref2path(r), *fn), 'r') as f: for l in f.readlines(): yield l
def fetchurl(m: Manager, url: str, sha256: Optional[str] = None, sha1: Optional[str] = None, mode: str = 'unpack,remove', name: Optional[str] = None, filename: Optional[str] = None, force_download: bool = False, check_promises: bool = True, **kwargs) -> DRef: """ Download and unpack an URL addess. Downloading is done by calling `wget` application. Optional unpacking is performed with the `aunpack` script from `atool` package. `sha256` defines the expected SHA-256 hashsum of the stored data. `mode` allows to tweak the stage's behavior: adding word 'unpack' instructs fetchurl to unpack the package, adding 'remove' instructs it to remove the archive after unpacking. If 'unpack' is not expected, then the promise named 'out_path' is created. Agruments: - `m:Manager` the dependency resolution [Manager](#pylightnix.types.Manager). - `url:str` URL to download from. Should point to a single file. - `sha256:str` SHA-256 hash sum of the file. - `model:str='unpack,remove'` Additional options. Format: `[unpack[,remove]]`. - `name:Optional[str]`: Name of the Derivation. The stage will attempt to deduce the name if not specified. - `filename:Optional[str]=None` Name of the filename on disk after downloading. Stage will attempt to deduced it if not specified. - `force_download:bool=False` If False, resume the last download if possible. - `check_promises:bool=True` Passed to `mkdrv` as-is. Example: ```python def hello_src(m:Manager)->DRef: hello_version = '2.10' return fetchurl( m, name='hello-src', url=f'http://ftp.gnu.org/gnu/hello/hello-{hello_version}.tar.gz', sha256='31e066137a962676e89f69d1b65382de95a7ef7d914b8cb956f41ea72e0f516b') rref:RRef=realize(instantiate(hello_src)) print(store_rref2path(rref)) ``` """ import pylightnix.core tmpfetchdir = join(pylightnix.core.PYLIGHTNIX_TMP, 'fetchurl') fname = filename or basename(urlparse(url).path) assert len(fname) > 0, ("Downloadable filename shouldn't be empty. " "Try specifying a valid `filename` argument") def _instantiate() -> Config: assert WGET() is not None if 'unpack' in mode: assert AUNPACK() is not None assert (sha256 is None) or (sha1 is None) makedirs(tmpfetchdir, exist_ok=True) if sha256 is not None: kwargs.update({ 'name': name or 'fetchurl', 'url': url, 'sha256': sha256, 'mode': mode }) elif sha1 is not None: kwargs.update({ 'name': name or 'fetchurl', 'url': url, 'sha1': sha1, 'mode': mode }) else: assert False, 'Either sha256 or sha1 arguments should be set' if 'unpack' not in mode: kwargs.update({'out_path': [promise, fname]}) return mkconfig(kwargs) def _realize(b: Build) -> None: c = build_cattrs(b) o = build_outpath(b) download_dir = o if force_download else tmpfetchdir try: partpath = join(download_dir, fname + '.tmp') p = Popen( [WGET(), "--continue", '--output-document', partpath, c.url], cwd=download_dir) p.wait() assert p.returncode == 0, f"Download failed, errcode '{p.returncode}'" assert isfile(partpath), f"Can't find output file '{partpath}'" with open(partpath, "rb") as f: if sha256 is not None: realhash = sha256sum(f.read()).hexdigest() assert realhash == c.sha256, ( f"Expected sha256 checksum '{c.sha256}', " f"but got '{realhash}'") elif sha1 is not None: realhash = sha1sum(f.read()).hexdigest() assert realhash == c.sha1, ( f"Expected sha1 checksum '{c.sha1}', " f"but got '{realhash}'") else: assert False, 'Either sha256 or sha1 arguments should be set' fullpath = join(o, fname) rename(partpath, fullpath) if 'unpack' in c.mode: _unpack_inplace(o, fullpath, 'remove' in c.mode) except Exception as e: error(f"Download failed: {e}") error(f"Keeping temporary directory {o}") raise return mkdrv(m, _instantiate(), match_only(), build_wrapper(_realize), check_promises=check_promises)