def linkdref(dref: DRef,
destdir: Optional[Path] = None,
name: Optional[str] = None,
S=None) -> Path:
destdir_ = '.' if destdir is None else destdir
name_: str = name if name is not None else (
'_result-' +
config_name(store_config(dref, S)) if destdir is None else 'result-' +
config_name(store_config(dref, S)))
symlink = Path(join(destdir_, name_))
forcelink(Path(relpath(store_dref2path(dref, S), destdir_)), symlink)
return symlink
def _either(S: SPath, dref: DRef, ctx: Context,
ra: RealizeArg) -> List[Dict[Tag, Path]]:
# Write the specified build status to every output
def _mark_status(outpaths: List[Dict[Tag, Path]],
status: str,
exception: Optional[str] = None) -> None:
for rg in outpaths:
o = rg[tag_out()]
writestr(join(o, 'status_either.txt'), status)
if exception is not None:
writestr(join(o, 'exception.txt'), exception)
# Scan all immediate dependecnies of this build, propagate the 'LEFT' status
# if any of them has it.
for dref_dep in store_deps([dref], S):
for rg in context_deref(ctx, dref_dep, S):
rref = rg[tag_out()]
status = tryreadstr_def(
join(store_rref2path(rref, S), 'status_either.txt'),
'RIGHT')
if status == 'RIGHT':
continue
elif status == 'LEFT':
outpaths = [{
tag_out():
Path(mkdtemp(prefix="either_tmp", dir=tmp))
}]
_mark_status(outpaths, 'LEFT')
return outpaths
else:
assert False, f"Invalid either status {status}"
# Execute the original build
try:
outpaths = f(S, dref, ctx, ra)
_mark_status(outpaths, 'RIGHT')
except KeyboardInterrupt:
raise
except BuildError as be:
outpaths = be.outgroups
_mark_status(outpaths, 'LEFT', format_exc())
except Exception:
outpaths = [{
tag_out(): Path(mkdtemp(prefix="either_tmp", dir=tmp))
}]
_mark_status(outpaths, 'LEFT', format_exc())
# Return either valid artifacts or a LEFT-substitute
return outpaths
def gc_exceptions(keep_paths:List[Path])->Tuple[List[DRef],List[RRef]]:
""" Scans `keep_paths` list for references to Pylightnix storage. Ignores
unrelated filesystem objects. """
keep_drefs:List[DRef]=[]
keep_rrefs:List[RRef]=[]
def _check(f:str):
nonlocal keep_drefs, keep_rrefs
if islink(a):
rref=path2rref(a)
if rref is not None:
keep_rrefs.append(rref)
else:
dref=path2dref(a)
if dref is not None:
keep_drefs.append(dref)
for path in keep_paths:
if islink(path):
_check(path)
elif isdir(path):
for root, dirs, filenames in walk(path, topdown=True):
for dirname in sorted(dirs):
a=Path(abspath(join(root, dirname)))
_check(a)
else:
pass
return keep_drefs,keep_rrefs
def tempdir(tmp: Optional[Path] = None) -> Path:
if tmp is None:
assert isinstance(PYLIGHTNIX_TMP, str), \
f"Default temp folder location is not a string: {PYLIGHTNIX_TMP}"
return Path(PYLIGHTNIX_TMP)
else:
return tmp
def dircp(src: Path, dst: Path, make_rw: bool = False) -> None:
""" Powerful folder copyier. """
assert isdir(src)
assert not isdir(dst)
tmppath = Path(dst + '.tmp')
copytree(src, tmppath)
if make_rw:
dirrw(tmppath)
rename(tmppath, dst)
def store_buildtime(rref: RRef, S=None) -> Optional[str]:
""" Return the buildtime of the current RRef in a format specified by the
[PYLIGHTNIX_TIME](#pylightnix.utils.PYLIGHTNIX_TIME) constant.
[parsetime](#pylightnix.utils.parsetime) may be used to parse stings into
UNIX-Epoch seconds.
Buildtime is the time when the realization process has started. Some
realizations may not provide this information. """
return tryread(Path(join(store_rref2path(rref, S), '__buildtime__.txt')))
def dirrm(path: Path, ignore_not_found: bool = True) -> None:
""" Powerful folder remover. Firts rename it to the temporary name. Deal with
possible write-protection. """
try:
tmppath = Path(path + '.tmp')
rename(path, tmppath)
dirrw(tmppath)
rmtree(tmppath)
except FileNotFoundError:
if not ignore_not_found:
raise
def path2dref(p: Path) -> Optional[DRef]:
""" Takes either a system path of some realization in the Pylightnix storage
or a symlink pointing to such path. Return a `DRef` which corresponds to this
path.
Note: `path2dref` operates on `p` symbolically. It doesn't actually check the
presence of such an object in storage """
if islink(p):
p = Path(readlink(p))
_, dref_part = split(p)
dref = DRef('dref:' + dref_part)
return dref if isdref(dref) else None
def dirrw(o: Path) -> None:
for root, dirs, files in walk(o):
for d in dirs:
mode = stat(join(root, d))[ST_MODE]
chmod(join(root, d), mode | (S_IWRITE))
for f in files:
if isfile(f):
filerw(Path(join(root, f)))
if islink(f):
warning(
f"Pylightnix doesn't guarantee the consistency of symlink '{f}'"
)
chmod(o, stat(o)[ST_MODE] | (S_IWRITE | S_IWGRP | S_IWOTH))
def val2path(v:Any, ctx:LensContext)->Path:
""" Resolve the current value of Lens into system path. Assert if it is not
possible or if the result is associated with multiple paths."""
S:SPath=ctx.storage
if isdref(v):
dref=DRef(v)
context=ctx.context
if context is not None:
if dref in context:
rgs=context_deref(context, dref)
assert len(rgs)==1, "Lens doesn't support multirealization dependencies"
return Path(store_rref2path(rgs[0][tag_out()]))
return store_dref2path(dref)
elif isrref(v):
return store_rref2path(RRef(v),S)
elif isrefpath(v):
refpath=list(v) # RefPath is list
bpath=ctx.build_path
context=ctx.context
if context is not None:
if refpath[0] in context:
rgs=context_deref(context,refpath[0],S)
assert len(rgs)==1, "Lens doesn't support multirealization dependencies"
return Path(join(store_rref2path(rgs[0][Tag('out')],S), *refpath[1:]))
else:
if bpath is not None:
# FIXME: should we assert on refpath[0]==build.dref ?
return Path(join(bpath, *refpath[1:]))
else:
assert False, f"Can't dereference refpath {refpath}"
else:
assert False, f"Lens couldn't resolve '{refpath}' without a context"
elif isinstance(v, Closure):
return val2path(v.dref, ctx)
elif isinstance(v, Build):
assert ctx.build_path is not None, f"Lens can't access build path of '{v}'"
return ctx.build_path
else:
assert False, f"Lens doesn't know how to resolve '{v}'"
def linkrref(rref: RRef,
destdir: Optional[Path] = None,
name: Optional[str] = None,
withtime: bool = False,
S=None) -> Path:
""" Helper function that creates a symbolic link to a particular realization
reference. The link is created under the current directory by default or under
the `destdir` directory.
Create a symlink pointing to realization `rref`. Other arguments define
symlink name and location. Informally,
`{tgtpath}/{timeprefix}{name} --> $PYLIGHTNIX_STORE/{dref}/{rref}`.
Overwrite existing symlinks. Folder named `tgtpath` should exist.
"""
destdir_ = '.' if destdir is None else destdir
name_: str = name if name is not None else (
'_result-' +
config_name(store_config(rref, S)) if destdir is None else 'result-' +
config_name(store_config(rref, S)))
ts: Optional[str] = store_buildtime(rref, S) if withtime else None
timetag_ = f'{ts}_' if ts is not None else ''
symlink = Path(join(destdir_, f"{timetag_}{name_}"))
forcelink(Path(relpath(store_rref2path(rref, S), destdir_)), symlink)
return symlink
def mkdrv_(c: Config, S: SPath) -> DRef:
""" Create new derivation in storage `S`.
We attempt to do it atomically by creating temp directory first and then
renaming it right into it's place in the storage.
FIXME: Assert or handle possible (but improbable) hash collision [*]
"""
assert_store_initialized(S)
# c=cp.config
assert_valid_config(c)
assert_rref_deps(c)
refname = config_name(c)
dhash = config_hash(c)
dref = mkdref(trimhash(dhash), refname)
o = Path(mkdtemp(prefix=refname, dir=tempdir()))
with open(join(o, 'config.json'), 'w') as f:
f.write(config_serialize(c))
filero(Path(join(o, 'config.json')))
drefpath = store_dref2path(dref, S)
dreftmp = Path(drefpath + '.tmp')
replace(o, dreftmp)
try:
replace(dreftmp, drefpath)
except OSError as err:
if err.errno == ENOTEMPTY:
# Existing folder means that it has a matched content [*]
dirrm(dreftmp, ignore_not_found=False)
else:
raise
return dref
def path2rref(p: Path) -> Optional[RRef]:
""" Takes either a system path of some realization in the Pylightnix storage
or a symlink pointing to such path. Return `RRef` which corresponds to this
path.
Note: `path2rref` operates on `p` symbolically. It doesn't actually check the
presence of such an object in storage """
if islink(p):
p = Path(readlink(p))
head, h1 = split(p)
_, dref_part = split(head)
dref = DRef('dref:' + dref_part)
if not isdref(dref):
return None
h2, nm = undref(dref)
return mkrref(HashPart(h1), HashPart(h2), mkname(nm))
def mklogdir(tag: str,
logrootdir: Path,
subdirs: list = [],
symlinks: bool = True,
timetag: Optional[str] = None) -> Path:
""" Create `<logrootdir>/<tag>_<time>` folder and set
`<logrootdir>/_<tag>_latest` symlink to point to it. """
logpath = logdir(tag, logrootdir=logrootdir, timetag=timetag)
makedirs(logpath, exist_ok=True)
if symlinks:
linkname = join(
logrootdir,
(('_' + str(tag) + '_latest') if len(tag) > 0 else '_latest'))
try:
symlink(basename(logpath), linkname)
except OSError as e:
if e.errno == EEXIST:
remove(linkname)
symlink(basename(logpath), linkname)
else:
raise e
for sd in subdirs:
makedirs(join(logpath, sd), exist_ok=True)
return Path(logpath)
def reserved(folder: Path, name: str) -> Path:
assert name in PYLIGHTNIX_RESERVED, \
f"File name '{name}' expected to be reserved"
return Path(join(folder, name))
def store_dref2path(r: DRef, S=None) -> Path:
(dhash, nm) = undref(r)
return Path(join(storage(S), dhash + '-' + nm))
def store_rref2path(r: RRef, S=None) -> Path:
(rhash, dhash, nm) = unrref(r)
return Path(join(storage(S), dhash + '-' + nm, rhash))
def store_cfgpath(r: DRef, S=None) -> Path:
return Path(join(store_dref2path(r, S), 'config.json'))
def rrefdata(rref: RRef, S=None) -> Iterable[Path]:
""" Return the paths of top-level artifacts """
root = store_rref2path(rref, S)
for fd in scandir(root):
if not (fd.is_file() and fd.name in PYLIGHTNIX_RESERVED):
yield Path(join(root, fd.name))
def fetchurl2(m: Manager,
url: str,
sha256: Optional[str] = None,
sha1: Optional[str] = None,
name: Optional[str] = None,
filename: Optional[str] = None,
force_download: bool = False,
**kwargs) -> DRef:
""" Download file given it's URL addess.
Downloading is done by calling `wget` application. Optional unpacking is
performed with the `aunpack` script from `atool` package. `sha256` defines the
expected SHA-256 hashsum of the stored data. `mode` allows to tweak the
stage's behavior: adding word 'unpack' instructs fetchurl to unpack the
package, adding 'remove' instructs it to remove the archive after unpacking.
If 'unpack' is not expected, then the promise named 'out_path' is created.
Agruments:
- `m:Manager` the dependency resolution [Manager](#pylightnix.types.Manager).
- `url:str` URL to download from. Should point to a single file.
- `sha256:str` SHA-256 hash sum of the file.
- `model:str='unpack,remove'` Additional options. Format: `[unpack[,remove]]`.
- `name:Optional[str]`: Name of the Derivation. The stage will attempt to
deduce the name if not specified.
- `filename:Optional[str]=None` Name of the filename on disk after downloading.
Stage will attempt to deduced it if not specified.
- `force_download:bool=False` If False, resume the last download if
possible.
- `check_promises:bool=True` Passed to `mkdrv` as-is.
Example:
```python
def hello_src(m:Manager)->DRef:
hello_version = '2.10'
return fetchurl2(
m,
name='hello-src',
url=f'http://ftp.gnu.org/gnu/hello/hello-{hello_version}.tar.gz',
sha256='31e066137a962676e89f69d1b65382de95a7ef7d914b8cb956f41ea72e0f516b')
rref:RRef=realize(instantiate(hello_src))
print(store_rref2path(rref))
```
"""
import pylightnix.core
tmpfetchdir = join(pylightnix.core.PYLIGHTNIX_TMP, 'fetchurl2')
assert isabs(tmpfetchdir), (f"Expect absolute PYLIGHTNIX_TMP path, "
f"got {tmpfetchdir}")
filename_ = filename or basename(urlparse(url).path)
assert len(filename_) > 0, ("Downloadable filename shouldn't be empty. "
"Try specifying a valid `filename` argument")
assert CURL() is not None
makedirs(tmpfetchdir, exist_ok=True)
if name is None:
name = 'fetchurl2'
if sha256 is None and sha1 is None:
if isfile(url):
sha256 = filehash(Path(url))
url = f'file://{url}'
else:
assert False, (
"Either `sha256` or `sha1` arguments should be specified "
"for URLs")
def _config() -> dict:
args: dict = {'name': name}
if sha1 is not None:
args.update({'sha1': sha1})
if sha256 is not None:
args.update({'sha256': sha256})
args.update({'out': [promise, filename_]})
args.update(**kwargs)
return args
def _make(b: Build) -> None:
c = build_cattrs(b)
o = build_outpath(b)
download_dir = o if force_download else tmpfetchdir
partpath = join(download_dir, filename_ + '.tmp')
try:
p = Popen(
[CURL(), "--continue-at", "-", "--output", partpath, url],
cwd=download_dir)
p.wait()
assert p.returncode == 0, f"Download failed, errcode '{p.returncode}'"
assert isfile(partpath), f"Can't find output file '{partpath}'"
with open(partpath, "rb") as f:
if sha256 is not None:
realhash = sha256sum(f.read()).hexdigest()
assert realhash == c.sha256, (
f"Expected sha256 checksum '{c.sha256}', "
f"but got '{realhash}'")
if sha1 is not None:
realhash = sha1sum(f.read()).hexdigest()
assert realhash == c.sha1, (
f"Expected sha1 checksum '{c.sha1}', "
f"but got '{realhash}'")
fullpath = join(o, filename_)
rename(partpath, fullpath)
except Exception as e:
error(f"Download failed: {e}")
error(f"Keeping temporary directory {o}")
raise
return mkdrv(m, mkconfig(_config()), match_only(), build_wrapper(_make))
def mkrealization(dref: DRef,
l: Context,
o: Path,
leader: Optional[Tuple[Tag, RRef]] = None,
S=None) -> RRef:
""" Create the [Realization](#pylightnix.types.RRef) object in the storage
`S`. Return new Realization reference.
Parameters:
- `dref:DRef`: Derivation reference to create the realization of.
- `l:Context`: Context which stores dependency information.
- `o:Path`: Path to temporal (build) folder which contains artifacts,
prepared by the [Realizer](#pylightnix.types.Realizer).
- `leader`: Tag name and Group identifier of the Group leader. By default,
we use name `out` and derivation's own rref.
FIXME: Assert or handle possible but improbable hash collision[*]
FIXME: Consider(not sure) writing group.json for all realizations[**]
"""
c = store_config(dref, S)
assert_valid_config(c)
(dhash, nm) = undref(dref)
assert isdir(o), (
f"While realizing {dref}: Outpath is expected to be a path to existing "
f"directory, but got {o}")
for fn in PYLIGHTNIX_RESERVED:
assert not isfile(join(o, fn)), (
f"While realizing {dref}: output folder '{o}' contains file '{fn}'. "
f"This name is reserved, please use another name. List of reserved "
f"names: {PYLIGHTNIX_RESERVED}")
with open(reserved(o, 'context.json'), 'w') as f:
f.write(context_serialize(l))
if leader is not None: # [**]
tag, group_rref = leader
with open(reserved(o, 'group.json'), 'w') as f:
json_dump({'tag': tag, 'group': group_rref}, f)
rhash = dirhash(o)
rref = mkrref(trimhash(rhash), dhash, nm)
rrefpath = store_rref2path(rref, S)
rreftmp = Path(rrefpath + '.tmp')
replace(o, rreftmp)
dirchmod(rreftmp, 'ro')
try:
replace(rreftmp, rrefpath)
except OSError as err:
if err.errno == ENOTEMPTY:
# Folder name contain the hash of the content, so getting here
# probably[*] means that we already have this object in storage so we
# just remove temp folder.
dirrm(rreftmp, ignore_not_found=False)
else:
# Attempt to roll-back
dirchmod(rreftmp, 'rw')
replace(rreftmp, o)
raise
return rref
