def put(self, key: str, outputs: Env.Bindings[Value.Base]) -> None:
if not self._cfg["call_cache"].get_bool("put"):
return
def cache(v: Union[Value.File, Value.Directory]) -> str:
_cached_files[inode(v.value)] = (key, outputs)
return ""
with _uploaded_files_lock:
Value.rewrite_env_paths(outputs, cache)
cache_put(self._cfg, self._logger, key, outputs)
def _call_eager(self, expr: E.Apply, arguments: List[V.Base]) -> V.Base:
ans_type = self.infer_type(expr)
try:
ans = self.op(arguments[0].coerce(ans_type).value, arguments[1].coerce(ans_type).value)
except ZeroDivisionError:
# TODO: different runtime error?
raise Error.IncompatibleOperand(expr.arguments[1], "Division by zero") from None
if ans_type == T.Int():
assert isinstance(ans, int)
return V.Int(ans)
assert isinstance(ans, float)
return V.Float(ans)
def eval(self, env: Env.Values) -> V.Base:
""
assert isinstance(self.type, T.Map)
eitems = []
for k, v in self.items:
eitems.append((k.eval(env), v.eval(env)))
# TODO: complain of duplicate keys
return V.Map(self.type, eitems)
def _call_eager(self, expr: E.Apply, arguments: List[V.Base]) -> V.Base:
ans_type = self.infer_type(expr)
if not isinstance(ans_type, T.String):
return super()._call_eager(expr, arguments)
# TODO: in a command interpolation, return missing if either operand is missing
ans = self.op(
str(arguments[0].coerce(T.String()).value), str(arguments[1].coerce(T.String()).value)
)
assert isinstance(ans, str)
return V.String(ans)
def _basename(*args) -> V.String:
assert len(args) in (1, 2)
assert isinstance(args[0], V.String)
path = args[0].value
if len(args) > 1:
assert isinstance(args[1], V.String)
suffix = args[1].value
if path.endswith(suffix):
path = path[: -len(suffix)]
return V.String(os.path.basename(path))
def __call__(self, expr: E.Apply, env: E.Env) -> V.Base:
ans_type = self.infer_type(expr)
if not isinstance(ans_type, T.String):
return super().__call__(expr, env)
# TODO: return missing if either operand is missing
ans = self.op(
str(expr.arguments[0].eval(env).coerce(T.String()).value),
str(expr.arguments[1].eval(env).coerce(T.String()).value),
)
assert isinstance(ans, str)
return V.String(ans)
def eval(self, env: Env.Values) -> V.String:
""
ans = []
for part in self.parts:
if isinstance(part, Placeholder):
# evaluate interpolated expression & stringify
ans.append(part.eval(env).value)
elif isinstance(part, str):
# use python builtins to decode escape sequences
ans.append(str.encode(part).decode("unicode_escape"))
else:
assert False
# concatenate the stringified parts and trim the surrounding quotes
return V.String("".join(ans)[1:-1]) # pyre-ignore
def eval(self, env: Env.Values) -> V.String:
""
v = self.expr.eval(env)
if isinstance(v, V.Null):
if "default" in self.options:
return V.String(self.options["default"])
return V.String("")
if isinstance(v, V.String):
return v
if isinstance(v, V.Array):
return V.String(self.options["sep"].join(
str(item.value) for item in v.value))
if v == V.Boolean(True) and "true" in self.options:
return V.String(self.options["true"])
if v == V.Boolean(False) and "false" in self.options:
return V.String(self.options["false"])
return V.String(str(v))
def cache_put(cfg: config.Loader, logger: logging.Logger, key: str,
outputs: Env.Bindings[Value.Base]):
if not (cfg["call_cache"].get_bool("put") and cfg["call_cache"]["backend"]
== "s3_progressive_upload_call_cache_backend"):
return
missing = False
def cache(v: Union[Value.File, Value.Directory]) -> str:
nonlocal missing
missing = missing or inode(str(v.value)) not in _uploaded_files
if missing:
return ""
return _uploaded_files[inode(str(v.value))]
remapped_outputs = Value.rewrite_env_paths(outputs, cache)
if not missing and cfg.has_option("s3_progressive_upload", "uri_prefix"):
uri = os.path.join(get_s3_put_prefix(cfg), "cache", f"{key}.json")
s3_object(uri).put(
Body=json.dumps(values_to_json(remapped_outputs)).encode())
flag_temporary(uri)
logger.info(_("call cache insert", cache_file=uri))
def eval(self, env: Env.Values) -> V.Int:
""
return V.Int(self.value)
def eval(self, env: Env.Values) -> V.Boolean:
""
return V.Boolean(self.value)
def _call_eager(self, expr: E.Apply, arguments: List[V.Base]) -> V.Base:
pfx = arguments[0].coerce(T.String()).value
return V.Array(
T.Array(T.String()),
[V.String(pfx + s.coerce(T.String()).value) for s in arguments[1].value],
)
def _call_eager(self, expr: E.Apply, arguments: List[V.Base]) -> V.Base:
arg0 = arguments[0]
assert isinstance(arg0, V.Int)
if arg0.value < 0:
raise Error.EvalError(expr, "range() got negative argument")
return V.Array(T.Array(T.Int()), [V.Int(x) for x in range(arg0.value)])
def __call__(self, expr: E.Apply, env: Env.Values) -> V.Base:
v = expr.arguments[0].eval(env)
if isinstance(v, V.Null):
return V.Int(0)
assert isinstance(v.value, list)
return V.Int(len(v.value))
def eval(self, env: Env.Values) -> V.Base:
""
assert isinstance(self.type, T.Pair)
lv = self.left.eval(env)
rv = self.right.eval(env)
return V.Pair(self.type, (lv, rv))
def __call__(self, expr: E.Apply, env: Env.Values, stdlib: Base) -> V.Base:
lhs = expr.arguments[0].eval(env, stdlib=stdlib).expect(T.Boolean()).value
if lhs:
return V.Boolean(True)
return expr.arguments[1].eval(env, stdlib=stdlib).expect(T.Boolean())
def __init__(self):
# language built-ins
self._at = _At()
self._land = _And()
self._lor = _Or()
self._negate = StaticFunction(
"_negate", [T.Boolean()], T.Boolean(), lambda x: V.Boolean(not x.value)
)
self._add = _AddOperator()
self._sub = _ArithmeticOperator("-", lambda l, r: l - r)
self._mul = _ArithmeticOperator("*", lambda l, r: l * r)
self._div = _ArithmeticOperator("/", lambda l, r: l // r)
self._rem = StaticFunction(
"_rem", [T.Int(), T.Int()], T.Int(), lambda l, r: V.Int(l.value % r.value)
)
self._eqeq = _ComparisonOperator("==", lambda l, r: l == r)
self._neq = _ComparisonOperator("!=", lambda l, r: l != r)
self._lt = _ComparisonOperator("<", lambda l, r: l < r)
self._lte = _ComparisonOperator("<=", lambda l, r: l <= r)
self._gt = _ComparisonOperator(">", lambda l, r: l > r)
self._gte = _ComparisonOperator(">=", lambda l, r: l >= r)
# static stdlib functions
for (name, argument_types, return_type, F) in [
("floor", [T.Float()], T.Int(), lambda v: V.Int(math.floor(v.value))),
("ceil", [T.Float()], T.Int(), lambda v: V.Int(math.ceil(v.value))),
("round", [T.Float()], T.Int(), lambda v: V.Int(round(v.value))),
("length", [T.Array(T.Any())], T.Int(), lambda v: V.Int(len(v.value))),
("sub", [T.String(), T.String(), T.String()], T.String(), _sub),
("basename", [T.String(), T.String(optional=True)], T.String(), _basename),
(
"defined",
[T.Any(optional=True)],
T.Boolean(),
lambda v: V.Boolean(not isinstance(v, V.Null)),
),
# context-dependent:
("write_lines", [T.Array(T.String())], T.File(), _notimpl),
("write_tsv", [T.Array(T.Array(T.String()))], T.File(), _notimpl),
("write_map", [T.Map((T.Any(), T.Any()))], T.File(), _notimpl),
("write_json", [T.Any()], T.File(), _notimpl),
("stdout", [], T.File(), _notimpl),
("stderr", [], T.File(), _notimpl),
("glob", [T.String()], T.Array(T.File()), _notimpl),
("read_int", [T.File()], T.Int(), _notimpl),
("read_boolean", [T.File()], T.Boolean(), _notimpl),
("read_string", [T.File()], T.String(), _notimpl),
("read_float", [T.File()], T.Float(), _notimpl),
("read_array", [T.File()], T.Array(T.Any()), _notimpl),
("read_map", [T.File()], T.Map((T.Any(), T.Any())), _notimpl),
("read_lines", [T.File()], T.Array(T.Any()), _notimpl),
("read_tsv", [T.File()], T.Array(T.Array(T.String())), _notimpl),
("read_json", [T.File()], T.Any(), _notimpl),
]:
setattr(self, name, StaticFunction(name, argument_types, return_type, F))
# polymorphically typed stdlib functions which require specialized
# infer_type logic
self.range = _Range()
self.prefix = _Prefix()
self.size = _Size()
self.select_first = _SelectFirst()
self.select_all = _SelectAll()
self.zip = _Zip()
self.cross = _Zip() # FIXME
self.flatten = _Flatten()
self.transpose = _Transpose()
def _sub(input: V.String, pattern: V.String, replace: V.String) -> V.String:
return V.String(re.compile(pattern.value).sub(replace.value, input.value))
def __call__(self, expr: E.Apply, env: E.Env) -> V.Base:
assert len(expr.arguments) == 2
return V.Boolean(
# pyre-fixme
self.op(expr.arguments[0].eval(env).value,
expr.arguments[1].eval(env).value))
def eval(self, env: Env.Values) -> V.Float:
""
return V.Float(self.value)
def __call__(self, expr: E.Apply, env: E.Env) -> V.Base:
lhs = expr.arguments[0].eval(env).expect(T.Boolean()).value
if lhs:
return V.Boolean(True)
return expr.arguments[1].eval(env).expect(T.Boolean())
def _call_eager(self, expr: E.Apply, arguments: List[V.Base]) -> V.Base:
assert len(arguments) == 2
return V.Boolean(self.op(arguments[0].value, arguments[1].value))
def eval(self, env: Env.Values) -> V.Array:
""
assert isinstance(self.type, T.Array)
return V.Array(self.type, [
item.eval(env).coerce(self.type.item_type) for item in self.items
])
def _call_eager(self, expr: E.Apply, arguments: List[V.Base]) -> V.Base:
arr = arguments[0]
assert isinstance(arr, V.Array)
arrty = arr.type
assert isinstance(arrty, T.Array)
return V.Array(arrty, [arg for arg in arr.value if not isinstance(arg, V.Null)])
assert len(expr.arguments) == len(self.argument_types)
argument_values = [
arg.eval(env).coerce(ty)
for arg, ty in zip(expr.arguments, self.argument_types)
]
ans: V.Base = self.F(*argument_values)
return ans.coerce(self.return_type)
def _notimpl(one: Any = None, two: Any = None) -> None:
exec("raise NotImplementedError()")
_static_functions: List[Tuple[str, List[T.Base], T.Base, Any]] = [
("_negate", [T.Boolean()], T.Boolean(),
lambda x: V.Boolean(not x.value)), # pyre-fixme
("_rem", [T.Int(), T.Int()], T.Int(),
lambda l, r: V.Int(l.value % r.value)), # pyre-fixme
("stdout", [], T.File(), _notimpl),
("basename", [T.String(), T.String(optional=True)], T.String(), _notimpl),
# note: size() can take an empty value and probably returns 0 in that case.
# e.g. https://github.com/DataBiosphere/topmed-workflows/blob/31ba8a714b36ada929044f2ba3d130936e6c740e/CRAM-no-header-md5sum/md5sum/CRAM_md5sum.wdl#L39
("size", [T.File(optional=True),
T.String(optional=True)], T.Float(), _notimpl),
("ceil", [T.Float()], T.Int(), _notimpl),
("round", [T.Float()], T.Int(), _notimpl),
("glob", [T.String()], T.Array(T.File()), _notimpl),
("read_int", [T.String()], T.Int(), _notimpl),
("read_boolean", [T.String()], T.Boolean(), _notimpl),
("read_string", [T.String()], T.String(), _notimpl),
("read_float", [T.String()], T.Float(), _notimpl),
def __call__(self, expr: E.Apply, env: Env.Values) -> V.Base:
if isinstance(expr.arguments[0].eval(env), V.Null):
return V.Boolean(False)
return V.Boolean(True)
