def _realize(b: Build) -> None:
c = build_cattrs(b)
o = build_outpath(b)
try:
assert path_ is not None
partpath = join(o, fname) + '.tmp'
fullpath = join(o, fname)
copyfile(path_, partpath)
assert isfile(partpath), f"Can't copy '{path_}' to '{partpath}'"
with open(partpath, "rb") as f:
realhash = sha256sum(f.read()).hexdigest()
assert realhash == c.sha256, (
f"Expected sha256 checksum '{c.sha256}', "
f"but got '{realhash}'")
rename(partpath, fullpath)
if 'unpack' in c.mode:
_unpack_inplace(o, fullpath, 'remove' in c.mode)
except Exception as e:
error(f"Copying failed: {e}")
error(f"Keeping temporary directory {o}")
raise
def _make(b: Build) -> None:
c = build_cattrs(b)
o = build_outpath(b)
download_dir = o if force_download else tmpfetchdir
partpath = join(download_dir, filename_ + '.tmp')
try:
p = Popen(
[CURL(), "--continue-at", "-", "--output", partpath, url],
cwd=download_dir)
p.wait()
assert p.returncode == 0, f"Download failed, errcode '{p.returncode}'"
assert isfile(partpath), f"Can't find output file '{partpath}'"
with open(partpath, "rb") as f:
if sha256 is not None:
realhash = sha256sum(f.read()).hexdigest()
assert realhash == c.sha256, (
f"Expected sha256 checksum '{c.sha256}', "
f"but got '{realhash}'")
if sha1 is not None:
realhash = sha1sum(f.read()).hexdigest()
assert realhash == c.sha1, (
f"Expected sha1 checksum '{c.sha1}', "
f"but got '{realhash}'")
fullpath = join(o, filename_)
rename(partpath, fullpath)
except Exception as e:
error(f"Download failed: {e}")
error(f"Keeping temporary directory {o}")
raise
def _mark_status(outpaths: List[Dict[Tag, Path]],
status: str,
exception: Optional[str] = None) -> None:
for rg in outpaths:
o = rg[tag_out()]
writestr(join(o, 'status_either.txt'), status)
if exception is not None:
writestr(join(o, 'exception.txt'), exception)
def dirrw(o: Path) -> None:
for root, dirs, files in walk(o):
for d in dirs:
mode = stat(join(root, d))[ST_MODE]
chmod(join(root, d), mode | (S_IWRITE))
for f in files:
if isfile(f):
filerw(Path(join(root, f)))
if islink(f):
warning(
f"Pylightnix doesn't guarantee the consistency of symlink '{f}'"
)
chmod(o, stat(o)[ST_MODE] | (S_IWRITE | S_IWGRP | S_IWOTH))
def gc_exceptions(keep_paths:List[Path])->Tuple[List[DRef],List[RRef]]:
""" Scans `keep_paths` list for references to Pylightnix storage. Ignores
unrelated filesystem objects. """
keep_drefs:List[DRef]=[]
keep_rrefs:List[RRef]=[]
def _check(f:str):
nonlocal keep_drefs, keep_rrefs
if islink(a):
rref=path2rref(a)
if rref is not None:
keep_rrefs.append(rref)
else:
dref=path2dref(a)
if dref is not None:
keep_drefs.append(dref)
for path in keep_paths:
if islink(path):
_check(path)
elif isdir(path):
for root, dirs, filenames in walk(path, topdown=True):
for dirname in sorted(dirs):
a=Path(abspath(join(root, dirname)))
_check(a)
else:
pass
return keep_drefs,keep_rrefs
def alldrefs(S=None) -> Iterable[DRef]:
""" Iterates over all derivations of the storage located at `S`
(PYLIGHTNIX_STORE env is used by default) """
store_path_ = storage(S)
for dirname in listdir(store_path_):
if dirname[-4:] != '.tmp' and isdir(join(store_path_, dirname)):
yield mkdref(HashPart(dirname[:32]), Name(dirname[32 + 1:]))
def dirsize(o: Path) -> int:
""" Return size in bytes """
total_size = 0
for dirpath, dirnames, filenames in walk(o):
for f in filenames:
fp = join(dirpath, f)
if not islink(fp):
total_size += getsize(fp)
return total_size
def _key(gr: RRefGroup, S=None) -> Optional[Union[int, float, str]]:
try:
with open(join(store_rref2path(grouprref(gr), S), filename),
'r') as f:
return float(f.readline())
except OSError:
return float('-inf')
except ValueError:
return float('-inf')
def _key(gr: RRefGroup, S=None) -> Optional[Union[int, float, str]]:
try:
with open(
join(store_rref2path(grouprref(gr), S),
'__buildtime__.txt'), 'r') as f:
t = parsetime(f.read())
return float(0 if t is None else t)
except OSError:
return float(0)
def _iter() -> Iterable[Tuple[str, bytes]]:
for root, dirs, filenames in walk(abspath(path), topdown=True):
for filename in sorted(filenames):
if len(filename) > 0 and filename[0] != '_':
localpath = abspath(join(root, filename))
if islink(localpath):
yield (f'link:{localpath}',
encode(readlink(localpath)))
with open(localpath, 'rb') as f:
yield (localpath, f.read())
def store_buildtime(rref: RRef, S=None) -> Optional[str]:
""" Return the buildtime of the current RRef in a format specified by the
[PYLIGHTNIX_TIME](#pylightnix.utils.PYLIGHTNIX_TIME) constant.
[parsetime](#pylightnix.utils.parsetime) may be used to parse stings into
UNIX-Epoch seconds.
Buildtime is the time when the realization process has started. Some
realizations may not provide this information. """
return tryread(Path(join(store_rref2path(rref, S), '__buildtime__.txt')))
def drefrrefs(dref: DRef, S=None) -> List[RRef]:
""" Iterate over all realizations of a derivation `dref`. The sort order is
unspecified. Matching is not taken into account. """
(dhash, nm) = undref(dref)
drefpath = store_dref2path(dref, S)
rrefs: List[RRef] = []
for f in listdir(drefpath):
if f[-4:] != '.tmp' and isdir(join(drefpath, f)):
rrefs.append(mkrref(HashPart(f), dhash, nm))
return rrefs
def linkdref(dref: DRef,
destdir: Optional[Path] = None,
name: Optional[str] = None,
S=None) -> Path:
destdir_ = '.' if destdir is None else destdir
name_: str = name if name is not None else (
'_result-' +
config_name(store_config(dref, S)) if destdir is None else 'result-' +
config_name(store_config(dref, S)))
symlink = Path(join(destdir_, name_))
forcelink(Path(relpath(store_dref2path(dref, S), destdir_)), symlink)
return symlink
def _realize(b: Build) -> None:
c = build_cattrs(b)
o = build_outpath(b)
download_dir = o if force_download else tmpfetchdir
try:
partpath = join(download_dir, fname + '.tmp')
p = Popen(
[WGET(), "--continue", '--output-document', partpath, c.url],
cwd=download_dir)
p.wait()
assert p.returncode == 0, f"Download failed, errcode '{p.returncode}'"
assert isfile(partpath), f"Can't find output file '{partpath}'"
with open(partpath, "rb") as f:
if sha256 is not None:
realhash = sha256sum(f.read()).hexdigest()
assert realhash == c.sha256, (
f"Expected sha256 checksum '{c.sha256}', "
f"but got '{realhash}'")
elif sha1 is not None:
realhash = sha1sum(f.read()).hexdigest()
assert realhash == c.sha1, (
f"Expected sha1 checksum '{c.sha1}', "
f"but got '{realhash}'")
else:
assert False, 'Either sha256 or sha1 arguments should be set'
fullpath = join(o, fname)
rename(partpath, fullpath)
if 'unpack' in c.mode:
_unpack_inplace(o, fullpath, 'remove' in c.mode)
except Exception as e:
error(f"Download failed: {e}")
error(f"Keeping temporary directory {o}")
raise
def val2path(v:Any, ctx:LensContext)->Path:
""" Resolve the current value of Lens into system path. Assert if it is not
possible or if the result is associated with multiple paths."""
S:SPath=ctx.storage
if isdref(v):
dref=DRef(v)
context=ctx.context
if context is not None:
if dref in context:
rgs=context_deref(context, dref)
assert len(rgs)==1, "Lens doesn't support multirealization dependencies"
return Path(store_rref2path(rgs[0][tag_out()]))
return store_dref2path(dref)
elif isrref(v):
return store_rref2path(RRef(v),S)
elif isrefpath(v):
refpath=list(v) # RefPath is list
bpath=ctx.build_path
context=ctx.context
if context is not None:
if refpath[0] in context:
rgs=context_deref(context,refpath[0],S)
assert len(rgs)==1, "Lens doesn't support multirealization dependencies"
return Path(join(store_rref2path(rgs[0][Tag('out')],S), *refpath[1:]))
else:
if bpath is not None:
# FIXME: should we assert on refpath[0]==build.dref ?
return Path(join(bpath, *refpath[1:]))
else:
assert False, f"Can't dereference refpath {refpath}"
else:
assert False, f"Lens couldn't resolve '{refpath}' without a context"
elif isinstance(v, Closure):
return val2path(v.dref, ctx)
elif isinstance(v, Build):
assert ctx.build_path is not None, f"Lens can't access build path of '{v}'"
return ctx.build_path
else:
assert False, f"Lens doesn't know how to resolve '{v}'"
def _either(S: SPath, dref: DRef, ctx: Context,
ra: RealizeArg) -> List[Dict[Tag, Path]]:
# Write the specified build status to every output
def _mark_status(outpaths: List[Dict[Tag, Path]],
status: str,
exception: Optional[str] = None) -> None:
for rg in outpaths:
o = rg[tag_out()]
writestr(join(o, 'status_either.txt'), status)
if exception is not None:
writestr(join(o, 'exception.txt'), exception)
# Scan all immediate dependecnies of this build, propagate the 'LEFT' status
# if any of them has it.
for dref_dep in store_deps([dref], S):
for rg in context_deref(ctx, dref_dep, S):
rref = rg[tag_out()]
status = tryreadstr_def(
join(store_rref2path(rref, S), 'status_either.txt'),
'RIGHT')
if status == 'RIGHT':
continue
elif status == 'LEFT':
outpaths = [{
tag_out():
Path(mkdtemp(prefix="either_tmp", dir=tmp))
}]
_mark_status(outpaths, 'LEFT')
return outpaths
else:
assert False, f"Invalid either status {status}"
# Execute the original build
try:
outpaths = f(S, dref, ctx, ra)
_mark_status(outpaths, 'RIGHT')
except KeyboardInterrupt:
raise
except BuildError as be:
outpaths = be.outgroups
_mark_status(outpaths, 'LEFT', format_exc())
except Exception:
outpaths = [{
tag_out(): Path(mkdtemp(prefix="either_tmp", dir=tmp))
}]
_mark_status(outpaths, 'LEFT', format_exc())
# Return either valid artifacts or a LEFT-substitute
return outpaths
def mkdrv_(c: Config, S: SPath) -> DRef:
""" Create new derivation in storage `S`.
We attempt to do it atomically by creating temp directory first and then
renaming it right into it's place in the storage.
FIXME: Assert or handle possible (but improbable) hash collision [*]
"""
assert_store_initialized(S)
# c=cp.config
assert_valid_config(c)
assert_rref_deps(c)
refname = config_name(c)
dhash = config_hash(c)
dref = mkdref(trimhash(dhash), refname)
o = Path(mkdtemp(prefix=refname, dir=tempdir()))
with open(join(o, 'config.json'), 'w') as f:
f.write(config_serialize(c))
filero(Path(join(o, 'config.json')))
drefpath = store_dref2path(dref, S)
dreftmp = Path(drefpath + '.tmp')
replace(o, dreftmp)
try:
replace(dreftmp, drefpath)
except OSError as err:
if err.errno == ENOTEMPTY:
# Existing folder means that it has a matched content [*]
dirrm(dreftmp, ignore_not_found=False)
else:
raise
return dref
def mklogdir(tag: str,
logrootdir: Path,
subdirs: list = [],
symlinks: bool = True,
timetag: Optional[str] = None) -> Path:
""" Create `<logrootdir>/<tag>_<time>` folder and set
`<logrootdir>/_<tag>_latest` symlink to point to it. """
logpath = logdir(tag, logrootdir=logrootdir, timetag=timetag)
makedirs(logpath, exist_ok=True)
if symlinks:
linkname = join(
logrootdir,
(('_' + str(tag) + '_latest') if len(tag) > 0 else '_latest'))
try:
symlink(basename(logpath), linkname)
except OSError as e:
if e.errno == EEXIST:
remove(linkname)
symlink(basename(logpath), linkname)
else:
raise e
for sd in subdirs:
makedirs(join(logpath, sd), exist_ok=True)
return Path(logpath)
def du() -> Dict[DRef, Tuple[int, Dict[RRef, int]]]:
""" Calculates the disk usage, in bytes. For every derivation, return it's
total disk usage and disk usages per realizations. Note, that total disk usage
of a derivation is slightly bigger than sum of it's realization's usages."""
res = {}
for dref in alldrefs():
rref_res = {}
dref_total = 0
for gr in store_rrefs_(dref):
for rref in gr.values():
usage = dirsize(store_rref2path(rref))
rref_res[rref] = usage
dref_total += usage
dref_total += getsize(join(store_dref2path(dref), 'config.json'))
res[dref] = (dref_total, rref_res)
return res
def linkrref(rref: RRef,
destdir: Optional[Path] = None,
name: Optional[str] = None,
withtime: bool = False,
S=None) -> Path:
""" Helper function that creates a symbolic link to a particular realization
reference. The link is created under the current directory by default or under
the `destdir` directory.
Create a symlink pointing to realization `rref`. Other arguments define
symlink name and location. Informally,
`{tgtpath}/{timeprefix}{name} --> $PYLIGHTNIX_STORE/{dref}/{rref}`.
Overwrite existing symlinks. Folder named `tgtpath` should exist.
"""
destdir_ = '.' if destdir is None else destdir
name_: str = name if name is not None else (
'_result-' +
config_name(store_config(rref, S)) if destdir is None else 'result-' +
config_name(store_config(rref, S)))
ts: Optional[str] = store_buildtime(rref, S) if withtime else None
timetag_ = f'{ts}_' if ts is not None else ''
symlink = Path(join(destdir_, f"{timetag_}{name_}"))
forcelink(Path(relpath(store_rref2path(rref, S), destdir_)), symlink)
return symlink
def _realize(b:Build)->None:
o=build_outpath(b)
for an,av in artifacts.items():
with open(join(o,an),'wb') as f:
f.write(av)
def rrefdata(rref: RRef, S=None) -> Iterable[Path]:
""" Return the paths of top-level artifacts """
root = store_rref2path(rref, S)
for fd in scandir(root):
if not (fd.is_file() and fd.name in PYLIGHTNIX_RESERVED):
yield Path(join(root, fd.name))
def either_status(rref: RRef, S=None) -> str:
return readstr(join(store_rref2path(rref, S), 'status_either.txt'))
def assert_promise_fulfilled(k: str, p: PromisePath, o: Path) -> None:
ppath = join(o, *p[1:])
assert isfile(ppath) or isdir(ppath) or islink(ppath), (
f"Promise '{k}' of {p[0]} is not fulfilled. "
f"{ppath} is expected to be a file or a directory.")
def mkrealization(dref: DRef,
l: Context,
o: Path,
leader: Optional[Tuple[Tag, RRef]] = None,
S=None) -> RRef:
""" Create the [Realization](#pylightnix.types.RRef) object in the storage
`S`. Return new Realization reference.
Parameters:
- `dref:DRef`: Derivation reference to create the realization of.
- `l:Context`: Context which stores dependency information.
- `o:Path`: Path to temporal (build) folder which contains artifacts,
prepared by the [Realizer](#pylightnix.types.Realizer).
- `leader`: Tag name and Group identifier of the Group leader. By default,
we use name `out` and derivation's own rref.
FIXME: Assert or handle possible but improbable hash collision[*]
FIXME: Consider(not sure) writing group.json for all realizations[**]
"""
c = store_config(dref, S)
assert_valid_config(c)
(dhash, nm) = undref(dref)
assert isdir(o), (
f"While realizing {dref}: Outpath is expected to be a path to existing "
f"directory, but got {o}")
for fn in PYLIGHTNIX_RESERVED:
assert not isfile(join(o, fn)), (
f"While realizing {dref}: output folder '{o}' contains file '{fn}'. "
f"This name is reserved, please use another name. List of reserved "
f"names: {PYLIGHTNIX_RESERVED}")
with open(reserved(o, 'context.json'), 'w') as f:
f.write(context_serialize(l))
if leader is not None: # [**]
tag, group_rref = leader
with open(reserved(o, 'group.json'), 'w') as f:
json_dump({'tag': tag, 'group': group_rref}, f)
rhash = dirhash(o)
rref = mkrref(trimhash(rhash), dhash, nm)
rrefpath = store_rref2path(rref, S)
rreftmp = Path(rrefpath + '.tmp')
replace(o, rreftmp)
dirchmod(rreftmp, 'ro')
try:
replace(rreftmp, rrefpath)
except OSError as err:
if err.errno == ENOTEMPTY:
# Folder name contain the hash of the content, so getting here
# probably[*] means that we already have this object in storage so we
# just remove temp folder.
dirrm(rreftmp, ignore_not_found=False)
else:
# Attempt to roll-back
dirchmod(rreftmp, 'rw')
replace(rreftmp, o)
raise
return rref
def lsdref_(r: DRef) -> Iterable[str]:
p = store_dref2path(r)
for d in listdir(p):
p2 = join(d, p)
if isdir(p2):
yield d
def logdir(tag: str, logrootdir: Path, timetag: Optional[str] = None):
timetag = timestring() if timetag is None else timetag
return join(logrootdir,
((str(tag) + '_') if len(tag) > 0 else '') + timetag)
def lsrref_(r: RRef, fn: List[str] = []) -> Iterable[str]:
p = join(store_rref2path(r), *fn)
for d in listdir(p):
yield d
def catrref_(r: RRef, fn: List[str]) -> Iterable[str]:
with open(join(store_rref2path(r), *fn), 'r') as f:
for l in f.readlines():
yield l
def fetchurl(m: Manager,
url: str,
sha256: Optional[str] = None,
sha1: Optional[str] = None,
mode: str = 'unpack,remove',
name: Optional[str] = None,
filename: Optional[str] = None,
force_download: bool = False,
check_promises: bool = True,
**kwargs) -> DRef:
""" Download and unpack an URL addess.
Downloading is done by calling `wget` application. Optional unpacking is
performed with the `aunpack` script from `atool` package. `sha256` defines the
expected SHA-256 hashsum of the stored data. `mode` allows to tweak the
stage's behavior: adding word 'unpack' instructs fetchurl to unpack the
package, adding 'remove' instructs it to remove the archive after unpacking.
If 'unpack' is not expected, then the promise named 'out_path' is created.
Agruments:
- `m:Manager` the dependency resolution [Manager](#pylightnix.types.Manager).
- `url:str` URL to download from. Should point to a single file.
- `sha256:str` SHA-256 hash sum of the file.
- `model:str='unpack,remove'` Additional options. Format: `[unpack[,remove]]`.
- `name:Optional[str]`: Name of the Derivation. The stage will attempt to
deduce the name if not specified.
- `filename:Optional[str]=None` Name of the filename on disk after downloading.
Stage will attempt to deduced it if not specified.
- `force_download:bool=False` If False, resume the last download if
possible.
- `check_promises:bool=True` Passed to `mkdrv` as-is.
Example:
```python
def hello_src(m:Manager)->DRef:
hello_version = '2.10'
return fetchurl(
m,
name='hello-src',
url=f'http://ftp.gnu.org/gnu/hello/hello-{hello_version}.tar.gz',
sha256='31e066137a962676e89f69d1b65382de95a7ef7d914b8cb956f41ea72e0f516b')
rref:RRef=realize(instantiate(hello_src))
print(store_rref2path(rref))
```
"""
import pylightnix.core
tmpfetchdir = join(pylightnix.core.PYLIGHTNIX_TMP, 'fetchurl')
fname = filename or basename(urlparse(url).path)
assert len(fname) > 0, ("Downloadable filename shouldn't be empty. "
"Try specifying a valid `filename` argument")
def _instantiate() -> Config:
assert WGET() is not None
if 'unpack' in mode:
assert AUNPACK() is not None
assert (sha256 is None) or (sha1 is None)
makedirs(tmpfetchdir, exist_ok=True)
if sha256 is not None:
kwargs.update({
'name': name or 'fetchurl',
'url': url,
'sha256': sha256,
'mode': mode
})
elif sha1 is not None:
kwargs.update({
'name': name or 'fetchurl',
'url': url,
'sha1': sha1,
'mode': mode
})
else:
assert False, 'Either sha256 or sha1 arguments should be set'
if 'unpack' not in mode:
kwargs.update({'out_path': [promise, fname]})
return mkconfig(kwargs)
def _realize(b: Build) -> None:
c = build_cattrs(b)
o = build_outpath(b)
download_dir = o if force_download else tmpfetchdir
try:
partpath = join(download_dir, fname + '.tmp')
p = Popen(
[WGET(), "--continue", '--output-document', partpath, c.url],
cwd=download_dir)
p.wait()
assert p.returncode == 0, f"Download failed, errcode '{p.returncode}'"
assert isfile(partpath), f"Can't find output file '{partpath}'"
with open(partpath, "rb") as f:
if sha256 is not None:
realhash = sha256sum(f.read()).hexdigest()
assert realhash == c.sha256, (
f"Expected sha256 checksum '{c.sha256}', "
f"but got '{realhash}'")
elif sha1 is not None:
realhash = sha1sum(f.read()).hexdigest()
assert realhash == c.sha1, (
f"Expected sha1 checksum '{c.sha1}', "
f"but got '{realhash}'")
else:
assert False, 'Either sha256 or sha1 arguments should be set'
fullpath = join(o, fname)
rename(partpath, fullpath)
if 'unpack' in c.mode:
_unpack_inplace(o, fullpath, 'remove' in c.mode)
except Exception as e:
error(f"Download failed: {e}")
error(f"Keeping temporary directory {o}")
raise
return mkdrv(m,
_instantiate(),
match_only(),
build_wrapper(_realize),
check_promises=check_promises)
