def _make(b: Build) -> None:
c = build_cattrs(b)
o = build_outpath(b)
download_dir = o if force_download else tmpfetchdir
partpath = join(download_dir, filename_ + '.tmp')
try:
p = Popen(
[CURL(), "--continue-at", "-", "--output", partpath, url],
cwd=download_dir)
p.wait()
assert p.returncode == 0, f"Download failed, errcode '{p.returncode}'"
assert isfile(partpath), f"Can't find output file '{partpath}'"
with open(partpath, "rb") as f:
if sha256 is not None:
realhash = sha256sum(f.read()).hexdigest()
assert realhash == c.sha256, (
f"Expected sha256 checksum '{c.sha256}', "
f"but got '{realhash}'")
if sha1 is not None:
realhash = sha1sum(f.read()).hexdigest()
assert realhash == c.sha1, (
f"Expected sha1 checksum '{c.sha1}', "
f"but got '{realhash}'")
fullpath = join(o, filename_)
rename(partpath, fullpath)
except Exception as e:
error(f"Download failed: {e}")
error(f"Keeping temporary directory {o}")
raise
def _realize(b: Build) -> None:
c = build_cattrs(b)
o = build_outpath(b)
try:
assert path_ is not None
partpath = join(o, fname) + '.tmp'
fullpath = join(o, fname)
copyfile(path_, partpath)
assert isfile(partpath), f"Can't copy '{path_}' to '{partpath}'"
with open(partpath, "rb") as f:
realhash = sha256sum(f.read()).hexdigest()
assert realhash == c.sha256, (
f"Expected sha256 checksum '{c.sha256}', "
f"but got '{realhash}'")
rename(partpath, fullpath)
if 'unpack' in c.mode:
_unpack_inplace(o, fullpath, 'remove' in c.mode)
except Exception as e:
error(f"Copying failed: {e}")
error(f"Keeping temporary directory {o}")
raise
def shell(r: Union[Build, RRef, DRef, Path, str, None] = None) -> None:
""" Open the Unix Shell in the directory associated with the argument passed.
Path to the shell executable is read from the `SHELL` environment variable,
defaulting to `/bin/sh`. If `r` is None, open the shell in the root of the
Pylightnix storage.
The function is expected to be run in REPL Python shells like IPython.
"""
cwd: str
if r is None:
import pylightnix.core
cwd = pylightnix.core.PYLIGHTNIX_STORE
elif isrref(r):
cwd = store_rref2path(RRef(r))
elif isdref(r):
cwd = store_dref2path(DRef(r))
elif isinstance(r, Build):
assert len(r.outgroups) > 0, (
"Shell function requires at least one build output path to be defined"
)
cwd = r.outgroups[0][Tag('out')]
elif isdir(r):
cwd = str(r)
elif isfile(r):
cwd = dirname(str(r))
else:
assert False, (
f"Expecting `RRef`, `DRef`, a directory or file path (either a string or "
f"a `Path`), or None. Got {r}")
Popen([environ.get('SHELL', '/bin/sh')], shell=False, cwd=cwd).wait()
def dirrw(o: Path) -> None:
for root, dirs, files in walk(o):
for d in dirs:
mode = stat(join(root, d))[ST_MODE]
chmod(join(root, d), mode | (S_IWRITE))
for f in files:
if isfile(f):
filerw(Path(join(root, f)))
if islink(f):
warning(
f"Pylightnix doesn't guarantee the consistency of symlink '{f}'"
)
chmod(o, stat(o)[ST_MODE] | (S_IWRITE | S_IWGRP | S_IWOTH))
def _realize(b: Build) -> None:
c = build_cattrs(b)
o = build_outpath(b)
download_dir = o if force_download else tmpfetchdir
try:
partpath = join(download_dir, fname + '.tmp')
p = Popen(
[WGET(), "--continue", '--output-document', partpath, c.url],
cwd=download_dir)
p.wait()
assert p.returncode == 0, f"Download failed, errcode '{p.returncode}'"
assert isfile(partpath), f"Can't find output file '{partpath}'"
with open(partpath, "rb") as f:
if sha256 is not None:
realhash = sha256sum(f.read()).hexdigest()
assert realhash == c.sha256, (
f"Expected sha256 checksum '{c.sha256}', "
f"but got '{realhash}'")
elif sha1 is not None:
realhash = sha1sum(f.read()).hexdigest()
assert realhash == c.sha1, (
f"Expected sha1 checksum '{c.sha1}', "
f"but got '{realhash}'")
else:
assert False, 'Either sha256 or sha1 arguments should be set'
fullpath = join(o, fname)
rename(partpath, fullpath)
if 'unpack' in c.mode:
_unpack_inplace(o, fullpath, 'remove' in c.mode)
except Exception as e:
error(f"Download failed: {e}")
error(f"Keeping temporary directory {o}")
raise
def filero(f: Path) -> None:
assert isfile(f), f"'{f}' is not a file"
chmod(f, stat(f)[ST_MODE] & ~(S_IWRITE | S_IWGRP | S_IWOTH))
def filerw(f: Path) -> None:
assert isfile(f), f"'{f}' is not a file"
chmod(f, stat(f)[ST_MODE] | (S_IWRITE))
def filehash(path: Path) -> Hash:
assert isfile(path), f"filehash() expects a file path, not '{path}'"
with open(path, 'rb') as f:
return datahash([(path, f.read())])
def fetchurl2(m: Manager,
url: str,
sha256: Optional[str] = None,
sha1: Optional[str] = None,
name: Optional[str] = None,
filename: Optional[str] = None,
force_download: bool = False,
**kwargs) -> DRef:
""" Download file given it's URL addess.
Downloading is done by calling `wget` application. Optional unpacking is
performed with the `aunpack` script from `atool` package. `sha256` defines the
expected SHA-256 hashsum of the stored data. `mode` allows to tweak the
stage's behavior: adding word 'unpack' instructs fetchurl to unpack the
package, adding 'remove' instructs it to remove the archive after unpacking.
If 'unpack' is not expected, then the promise named 'out_path' is created.
Agruments:
- `m:Manager` the dependency resolution [Manager](#pylightnix.types.Manager).
- `url:str` URL to download from. Should point to a single file.
- `sha256:str` SHA-256 hash sum of the file.
- `model:str='unpack,remove'` Additional options. Format: `[unpack[,remove]]`.
- `name:Optional[str]`: Name of the Derivation. The stage will attempt to
deduce the name if not specified.
- `filename:Optional[str]=None` Name of the filename on disk after downloading.
Stage will attempt to deduced it if not specified.
- `force_download:bool=False` If False, resume the last download if
possible.
- `check_promises:bool=True` Passed to `mkdrv` as-is.
Example:
```python
def hello_src(m:Manager)->DRef:
hello_version = '2.10'
return fetchurl2(
m,
name='hello-src',
url=f'http://ftp.gnu.org/gnu/hello/hello-{hello_version}.tar.gz',
sha256='31e066137a962676e89f69d1b65382de95a7ef7d914b8cb956f41ea72e0f516b')
rref:RRef=realize(instantiate(hello_src))
print(store_rref2path(rref))
```
"""
import pylightnix.core
tmpfetchdir = join(pylightnix.core.PYLIGHTNIX_TMP, 'fetchurl2')
assert isabs(tmpfetchdir), (f"Expect absolute PYLIGHTNIX_TMP path, "
f"got {tmpfetchdir}")
filename_ = filename or basename(urlparse(url).path)
assert len(filename_) > 0, ("Downloadable filename shouldn't be empty. "
"Try specifying a valid `filename` argument")
assert CURL() is not None
makedirs(tmpfetchdir, exist_ok=True)
if name is None:
name = 'fetchurl2'
if sha256 is None and sha1 is None:
if isfile(url):
sha256 = filehash(Path(url))
url = f'file://{url}'
else:
assert False, (
"Either `sha256` or `sha1` arguments should be specified "
"for URLs")
def _config() -> dict:
args: dict = {'name': name}
if sha1 is not None:
args.update({'sha1': sha1})
if sha256 is not None:
args.update({'sha256': sha256})
args.update({'out': [promise, filename_]})
args.update(**kwargs)
return args
def _make(b: Build) -> None:
c = build_cattrs(b)
o = build_outpath(b)
download_dir = o if force_download else tmpfetchdir
partpath = join(download_dir, filename_ + '.tmp')
try:
p = Popen(
[CURL(), "--continue-at", "-", "--output", partpath, url],
cwd=download_dir)
p.wait()
assert p.returncode == 0, f"Download failed, errcode '{p.returncode}'"
assert isfile(partpath), f"Can't find output file '{partpath}'"
with open(partpath, "rb") as f:
if sha256 is not None:
realhash = sha256sum(f.read()).hexdigest()
assert realhash == c.sha256, (
f"Expected sha256 checksum '{c.sha256}', "
f"but got '{realhash}'")
if sha1 is not None:
realhash = sha1sum(f.read()).hexdigest()
assert realhash == c.sha1, (
f"Expected sha1 checksum '{c.sha1}', "
f"but got '{realhash}'")
fullpath = join(o, filename_)
rename(partpath, fullpath)
except Exception as e:
error(f"Download failed: {e}")
error(f"Keeping temporary directory {o}")
raise
return mkdrv(m, mkconfig(_config()), match_only(), build_wrapper(_make))
def assert_promise_fulfilled(k: str, p: PromisePath, o: Path) -> None:
ppath = join(o, *p[1:])
assert isfile(ppath) or isdir(ppath) or islink(ppath), (
f"Promise '{k}' of {p[0]} is not fulfilled. "
f"{ppath} is expected to be a file or a directory.")
def mkrealization(dref: DRef,
l: Context,
o: Path,
leader: Optional[Tuple[Tag, RRef]] = None,
S=None) -> RRef:
""" Create the [Realization](#pylightnix.types.RRef) object in the storage
`S`. Return new Realization reference.
Parameters:
- `dref:DRef`: Derivation reference to create the realization of.
- `l:Context`: Context which stores dependency information.
- `o:Path`: Path to temporal (build) folder which contains artifacts,
prepared by the [Realizer](#pylightnix.types.Realizer).
- `leader`: Tag name and Group identifier of the Group leader. By default,
we use name `out` and derivation's own rref.
FIXME: Assert or handle possible but improbable hash collision[*]
FIXME: Consider(not sure) writing group.json for all realizations[**]
"""
c = store_config(dref, S)
assert_valid_config(c)
(dhash, nm) = undref(dref)
assert isdir(o), (
f"While realizing {dref}: Outpath is expected to be a path to existing "
f"directory, but got {o}")
for fn in PYLIGHTNIX_RESERVED:
assert not isfile(join(o, fn)), (
f"While realizing {dref}: output folder '{o}' contains file '{fn}'. "
f"This name is reserved, please use another name. List of reserved "
f"names: {PYLIGHTNIX_RESERVED}")
with open(reserved(o, 'context.json'), 'w') as f:
f.write(context_serialize(l))
if leader is not None: # [**]
tag, group_rref = leader
with open(reserved(o, 'group.json'), 'w') as f:
json_dump({'tag': tag, 'group': group_rref}, f)
rhash = dirhash(o)
rref = mkrref(trimhash(rhash), dhash, nm)
rrefpath = store_rref2path(rref, S)
rreftmp = Path(rrefpath + '.tmp')
replace(o, rreftmp)
dirchmod(rreftmp, 'ro')
try:
replace(rreftmp, rrefpath)
except OSError as err:
if err.errno == ENOTEMPTY:
# Folder name contain the hash of the content, so getting here
# probably[*] means that we already have this object in storage so we
# just remove temp folder.
dirrm(rreftmp, ignore_not_found=False)
else:
# Attempt to roll-back
dirchmod(rreftmp, 'rw')
replace(rreftmp, o)
raise
return rref
