def parse_document(
txt: str, version: Optional[str] = None, uri: str = "", imported: bool = False
) -> D.Document:
if version is None:
# for now assume the version is 1.0 if the first line is "version <number>"
# otherwise draft-2
version = "draft-2"
for line in txt.split("\n"):
line = line.strip()
if line and line[0] != "#":
if line.startswith("version ") and line[8].isdigit():
version = "1.0"
break
if not txt.strip():
return D.Document(
SourcePosition(filename=uri, line=0, column=0, end_line=0, end_column=0),
[],
[],
None,
imported,
)
try:
return _DocTransformer(uri, imported).transform(parse(txt, "document", version))
except lark.exceptions.UnexpectedCharacters as exn:
raise Err.ParserError(uri if uri != "" else "(in buffer)") from exn
except lark.exceptions.UnexpectedToken as exn:
raise Err.ParserError(uri if uri != "" else "(in buffer)") from exn
def infer_type(self, expr: E.Apply) -> T.Base:
if len(expr.arguments) != 1:
raise Error.WrongArity(expr, 1)
if not isinstance(expr.arguments[0].type, T.Array):
raise Error.StaticTypeMismatch(expr, T.Array(None),
expr.arguments[0].type)
return T.Int()
def __call__(self, expr: E.Apply, env: E.Env) -> V.Base:
assert len(expr.arguments) == 2
lhs = expr.arguments[0]
rhs = expr.arguments[1]
if isinstance(lhs.type, T.Array):
arr = lhs.eval(env)
assert isinstance(arr, V.Array)
assert isinstance(arr.type, T.Array)
assert isinstance(arr.value, list)
idx = rhs.eval(env).expect(T.Int()).value
if idx < 0 or idx >= len(arr.value):
raise Error.OutOfBounds(rhs)
return arr.value[idx] # pyre-fixme
if isinstance(lhs.type, T.Map):
mp = lhs.eval(env)
assert isinstance(mp, V.Map)
assert isinstance(mp.type, T.Map)
assert mp.type.item_type is not None
assert isinstance(mp.value, list)
ans = None
key = rhs.eval(env).expect(mp.type.item_type[0])
for k, v in mp.value:
if key == k:
ans = v.expect(mp.type.item_type[1])
if ans is None:
raise Error.OutOfBounds(rhs) # TODO: KeyNotFound
return ans # pyre-fixme
assert False # pyre-fixme
def document(self, items, meta):
imports = []
structs = {}
tasks = []
workflow = None
for item in items:
if isinstance(item, D.Task):
tasks.append(item)
elif isinstance(item, D.Workflow):
if workflow is not None:
raise Err.MultipleDefinitions(
sp(self.filename, meta),
"Document has multiple workflows")
workflow = item
elif isinstance(item, D.StructTypeDef):
if item.name in structs:
raise Err.MultipleDefinitions(
sp(self.filename, meta),
"multiple structs named " + item.name)
structs[item.name] = item
elif isinstance(item, lark.Tree) and item.data == "version":
pass
elif isinstance(item, D.DocImport):
imports.append(item)
else:
assert False
return D.Document(sp(self.filename, meta), imports, structs, tasks,
workflow)
def _infer_type(self, type_env: Env.Types) -> T.Base:
self.expr.infer_type(type_env, self._check_quant)
if isinstance(self.expr.type, T.Array):
if "sep" not in self.options:
raise Error.StaticTypeMismatch(
self, T.Array(None), self.expr.type,
"array command placeholder must have 'sep'")
# if sum(1 for t in [T.Int, T.Float, T.Boolean, T.String, T.File] if isinstance(self.expr.type.item_type, t)) == 0:
# raise Error.StaticTypeMismatch(self, T.Array(None), self.expr.type, "cannot use array of complex types for command placeholder")
elif "sep" in self.options:
raise Error.StaticTypeMismatch(
self,
T.Array(None),
self.expr.type,
"command placeholder has 'sep' option for non-Array expression",
)
if "true" in self.options or "false" in self.options:
if not isinstance(self.expr.type, T.Boolean):
raise Error.StaticTypeMismatch(
self,
T.Boolean(),
self.expr.type,
"command placeholder 'true' and 'false' options used with non-Boolean expression",
)
if not ("true" in self.options and "false" in self.options):
raise Error.StaticTypeMismatch(
self,
T.Boolean(),
self.expr.type,
"command placeholder with only one of 'true' and 'false' options",
)
return T.String()
def typecheck_input(self, type_env: Env.Types, doc: TVDocument,
check_quant: bool) -> None:
# Check the input expressions against the callee's inputs. One-time use
assert self.callee
# Make a set of the input names which are required for this call to
# typecheck. In the top-level workflow, nothing is actually required
# as missing call inputs become workflow inputs required at runtime.
required_inputs = (set(decl.name
for decl in self.callee.required_inputs)
if doc.imported else set())
# typecheck call inputs against task/workflow input declarations
for name, expr in self.inputs.items():
decl = None
if isinstance(self.callee, Task):
for d in self.callee.inputs + self.callee.postinputs:
if d.name == name:
decl = d
else:
assert isinstance(self.callee, Workflow)
for ele in self.callee.elements:
if isinstance(ele, Decl) and ele.name == name:
decl = ele
if decl is None:
raise Err.NoSuchInput(expr, name)
expr.infer_type(type_env, check_quant).typecheck(decl.type)
if name in required_inputs:
required_inputs.remove(name)
# Check whether any required inputs were missed
if required_inputs:
raise Err.MissingInput(self, self.name, required_inputs)
def typecheck(self, check_quant: bool = True) -> None:
"""Typecheck each task in the document, then the workflow, if any.
Documents returned by :func:`~WDL.load` have already been typechecked."""
names = set()
for _, namespace, _ in self.imports:
if namespace in names:
raise Err.MultipleDefinitions(
self, "Multiple imports with namespace " + namespace)
names.add(namespace)
names = set()
# typecheck each task
for task in self.tasks:
if task.name in names:
raise Err.MultipleDefinitions(
task, "Multiple tasks named " + task.name)
names.add(task.name)
task.typecheck(check_quant=check_quant)
# typecheck the workflow
if self.workflow:
if self.workflow.name in names:
raise Err.MultipleDefinitions(
self.workflow,
"Workflow name collides with a task also named " +
self.workflow.name,
)
self.workflow.typecheck(self, check_quant=check_quant)
def infer_type(self, expr: E.Apply) -> T.Base:
assert len(expr.arguments) == 2
for arg in expr.arguments:
if not isinstance(arg.type, T.Boolean):
raise Error.IncompatibleOperand(arg, "non-Boolean operand to ||")
if expr._check_quant and arg.type.optional:
raise Error.IncompatibleOperand(arg, "optional Boolean? operand to ||")
return T.Boolean()
def workflow(self, items, meta):
elements = []
inputs = None
outputs = None
output_idents = None
output_idents_pos = None
parameter_meta = None
meta_section = None
for item in items[1:]:
if isinstance(item, dict):
if "inputs" in item:
assert inputs is None
inputs = item["inputs"]
elif "outputs" in item:
if outputs is not None:
raise Err.MultipleDefinitions(
sp(self.filename, meta),
"redundant sections in workflow")
outputs = item["outputs"]
if "output_idents" in item:
assert output_idents is None
output_idents = item["output_idents"]
output_idents_pos = item["pos"]
elif "meta" in item:
if meta_section is not None:
raise Err.MultipleDefinitions(
sp(self.filename, meta),
"redundant sections in workflow")
meta_section = item["meta"]
elif "parameter_meta" in item:
if parameter_meta is not None:
raise Err.MultipleDefinitions(
sp(self.filename, meta),
"redundant sections in workflow")
parameter_meta = item["parameter_meta"]
else:
assert False
elif isinstance(item, (D.Call, D.Conditional, D.Decl, D.Scatter)):
elements.append(item)
else:
assert False
_check_keyword(sp(self.filename, meta), items[0].value)
return D.Workflow(
sp(self.filename, meta),
items[0].value,
inputs,
elements,
outputs,
parameter_meta or dict(),
meta_section or dict(),
output_idents,
output_idents_pos,
)
def infer_type(self, expr: E.Apply) -> T.Base:
if len(expr.arguments) != 1:
raise Error.WrongArity(expr, 1)
if not isinstance(expr.arguments[0].type, T.Array):
raise Error.StaticTypeMismatch(expr.arguments[0], T.Array(None),
expr.arguments[0].type)
if expr.arguments[0].type.item_type is None:
# TODO: error for 'indeterminate type'
raise Error.EmptyArray(expr.arguments[0])
ty = expr.arguments[0].type.item_type
assert isinstance(ty, T.Base)
return T.Array(ty.copy(optional=False))
def infer_type(self, expr: E.Apply) -> T.Base:
if len(expr.arguments) != 1:
raise Error.WrongArity(expr, 1)
expr.arguments[0].typecheck(T.Array(T.Any()))
# TODO: won't handle implicit coercion from T to Array[T]
assert isinstance(expr.arguments[0].type, T.Array)
if expr.arguments[0].type.item_type is None:
return T.Array(T.Any())
if not isinstance(expr.arguments[0].type.item_type, T.Array):
raise Error.StaticTypeMismatch(
expr.arguments[0], T.Array(T.Array(T.Any())), expr.arguments[0].type
)
return expr.arguments[0].type
def _call_eager(self, expr: E.Apply, arguments: List[V.Base]) -> V.Base:
arr = arguments[0]
assert isinstance(arr, V.Array)
for arg in arr.value:
if not isinstance(arg, V.Null):
return arg
raise Error.NullValue(expr)
def _infer_type(self, type_env: Env.Types) -> T.Base:
if not self.items:
return T.Array(None)
for item in self.items:
item.infer_type(type_env, self._check_quant)
# Start by assuming the type of the first item is the item type
item_type: T.Base = self.items[0].type
# Allow a mixture of Int and Float to construct Array[Float]
if isinstance(item_type, T.Int):
for item in self.items:
if isinstance(item.type, T.Float):
item_type = T.Float()
# If any item is String, assume item type is String
# If any item has optional quantifier, assume item type is optional
# If all items have nonempty quantifier, assume item type is nonempty
all_nonempty = len(self.items) > 0
for item in self.items:
if isinstance(item.type, T.String):
item_type = T.String(optional=item_type.optional)
if item.type.optional:
item_type = item_type.copy(optional=True)
if isinstance(item.type, T.Array) and not item.type.nonempty:
all_nonempty = False
if isinstance(item_type, T.Array):
item_type = item_type.copy(nonempty=all_nonempty)
# Check all items are coercible to item_type
for item in self.items:
try:
item.typecheck(item_type)
except Error.StaticTypeMismatch:
self._type = T.Array(item_type, optional=False, nonempty=True)
raise Error.StaticTypeMismatch(
self, item_type, item.type,
"(inconsistent types within array)") from None
return T.Array(item_type, optional=False, nonempty=True)
def task(self, items, meta):
d = {}
for item in items:
if isinstance(item, dict):
for k, v in item.items():
if k in d:
raise Err.MultipleDefinitions(
sp(self.filename, meta), "redundant sections in task"
)
d[k] = v
else:
assert isinstance(item, str)
assert "name" not in d
d["name"] = item.value
return D.Task(
sp(self.filename, meta),
d["name"],
d.get("inputs", []),
d.get("decls", []),
d["command"],
d.get("outputs", []),
d.get("parameter_meta", {}),
d.get("runtime", {}),
d.get("meta", {}),
)
def infer_type(self, expr: E.Apply) -> T.Base:
assert len(expr.arguments) == 2
if (
(
expr._check_quant
and expr.arguments[0].type.optional != expr.arguments[1].type.optional
)
or (
self.name not in ["==", "!="]
and (expr.arguments[0].type.optional or expr.arguments[1].type.optional)
)
or (
not (
expr.arguments[0].type.copy(optional=False)
== expr.arguments[1].type.copy(optional=False)
or (
isinstance(expr.arguments[0].type, T.Int)
and isinstance(expr.arguments[1].type, T.Float)
)
or (
isinstance(expr.arguments[0].type, T.Float)
and isinstance(expr.arguments[1].type, T.Int)
)
)
)
):
raise Error.IncompatibleOperand(
expr,
"Cannot compare {} and {}".format(
str(expr.arguments[0].type), str(expr.arguments[1].type)
),
)
return T.Boolean()
def _call_eager(self, expr: E.Apply, arguments: List[V.Base]) -> V.Base:
argument_values = [arg.coerce(ty) for arg, ty in zip(arguments, self.argument_types)]
try:
ans: V.Base = self.F(*argument_values)
except Exception as exn:
raise Error.EvalError(expr, "function evaluation failed") from exn
return ans.coerce(self.return_type)
def infer_type(self, expr: E.Apply) -> T.Base:
assert len(expr.arguments) == 1
if not isinstance(expr.arguments[0].type, T.Pair):
raise Error.NotAPair(expr.arguments[0])
return expr.arguments[
0].type.left_type if self.left else expr.arguments[
0].type.right_type
def load(uri: str,
path: List[str] = [],
check_quant: bool = True,
imported: Optional[bool] = False) -> Document:
for fn in [uri] + [os.path.join(dn, uri) for dn in reversed(path)]:
if os.path.exists(fn):
with open(fn, "r") as infile:
# read and parse the document
doc = WDL._parser.parse_document(infile.read(),
uri=uri,
imported=imported)
assert isinstance(doc, Document)
# recursively descend into document's imports, and store the imported
# documents into doc.imports
# TODO: limit recursion; prevent mutual recursion
for i in range(len(doc.imports)):
try:
subpath = [os.path.dirname(fn)] + path
subdoc = load(doc.imports[i][0],
subpath,
check_quant=check_quant,
imported=True)
except Exception as exn:
raise Err.ImportError(uri, doc.imports[i][0]) from exn
doc.imports[i] = (doc.imports[i][0], doc.imports[i][1],
subdoc)
doc.typecheck(check_quant=check_quant)
return doc
raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), uri)
def task(self, items, meta):
d = {"noninput_decls": []}
for item in items:
if isinstance(item, dict):
for k, v in item.items():
if k == "noninput_decl":
d["noninput_decls"].append(v)
elif k in d:
raise Err.MultipleDefinitions(
sp(self.filename, meta),
"redundant sections in task")
else:
d[k] = v
else:
assert isinstance(item, str)
assert "name" not in d
d["name"] = item.value
_check_keyword(sp(self.filename, meta), d["name"])
return D.Task(
sp(self.filename, meta),
d["name"],
d.get("inputs", None),
d["noninput_decls"],
d["command"],
d.get("outputs", []),
d.get("parameter_meta", {}),
d.get("runtime", {}),
d.get("meta", {}),
)
def placeholder(self, items, meta):
options = dict(items[:-1])
if len(options.items()) < len(items) - 1:
raise Err.MultipleDefinitions(
sp(self.filename, meta),
"duplicate options in expression placeholder")
return E.Placeholder(sp(self.filename, meta), options, items[-1])
def call_inputs(self, items, meta):
d = dict()
for k, v in items:
if k in d:
raise Err.MultipleDefinitions(sp(self.filename, meta),
"duplicate keys in call inputs")
d[k] = v
return d
def meta_object(self, items, meta):
d = dict()
for k, v in items:
if k in d:
raise Err.MultipleDefinitions(sp(self.filename, meta),
"duplicate keys in meta object")
d[k] = v
return d
def typecheck(self, expected: Optional[T.Base]) -> Base:
""
if not self.items and isinstance(expected, T.Array):
# the literal empty array satisfies any array type
# (unless it has the nonempty quantifier)
if expected.nonempty:
raise Error.EmptyArray(self)
return self
return super().typecheck(expected) # pyre-ignore
def __init__(self, pos: SourcePosition, function: str,
arguments: List[Base]) -> None:
super().__init__(pos)
try:
self.function = _stdlib[function]
self.function_name = function
except KeyError:
raise Error.NoSuchFunction(self, function) from None
self.arguments = arguments
def eval(self, env: Env.Values) -> V.Base:
""
try:
if self.condition.eval(env).expect(T.Boolean()).value:
ans = self.consequent.eval(env)
else:
ans = self.alternative.eval(env)
return ans
except ReferenceError:
raise Error.NullValue(self) from None
def infer_type(self, expr: E.Apply) -> T.Base:
if len(expr.arguments) != 2:
raise Error.WrongArity(expr, 2)
arg0ty: T.Base = expr.arguments[0].type
if not isinstance(arg0ty, T.Array) or (expr._check_quant and arg0ty.optional):
raise Error.StaticTypeMismatch(expr.arguments[0], T.Array(T.Any()), arg0ty)
if isinstance(arg0ty.item_type, T.Any):
# TODO: error for 'indeterminate type'
raise Error.EmptyArray(expr.arguments[0])
arg1ty: T.Base = expr.arguments[1].type
if not isinstance(arg1ty, T.Array) or (expr._check_quant and arg1ty.optional):
raise Error.StaticTypeMismatch(expr.arguments[1], T.Array(T.Any()), arg1ty)
if isinstance(arg1ty.item_type, T.Any):
# TODO: error for 'indeterminate type'
raise Error.EmptyArray(expr.arguments[1])
return T.Array(
T.Pair(arg0ty.item_type, arg1ty.item_type),
nonempty=(arg0ty.nonempty or arg1ty.nonempty),
)
def infer_type(self, expr: E.Apply) -> T.Base:
min_args = len(self.argument_types)
for ty in reversed(self.argument_types):
if ty.optional:
min_args = min_args - 1
else:
break
if len(expr.arguments) > len(self.argument_types) or len(expr.arguments) < min_args:
raise Error.WrongArity(expr, len(self.argument_types))
for i in range(len(expr.arguments)):
try:
expr.arguments[i].typecheck(self.argument_types[i])
except Error.StaticTypeMismatch:
raise Error.StaticTypeMismatch(
expr.arguments[i],
self.argument_types[i],
expr.arguments[i].type,
"for {} argument #{}".format(self.name, i + 1),
) from None
return self.return_type
def add_to_type_env(self, type_env: Env.Types) -> Env.Types:
# Add an appropriate binding in the type env, after checking for name
# collision.
try:
Env.resolve(type_env, [], self.name)
raise Err.MultipleDefinitions(
self, "Multiple declarations of " + self.name)
except KeyError:
pass
ans: Env.Types = Env.bind(self.name, self.type, type_env, ctx=self)
return ans
def infer_type(self, expr: E.Apply) -> T.Base:
if len(expr.arguments) != 2:
raise Error.WrongArity(expr, 2)
expr.arguments[0].typecheck(T.String())
expr.arguments[1].typecheck(T.Array(T.String()))
return T.Array(
T.String(),
nonempty=(
isinstance(expr.arguments[1].type, T.Array) and expr.arguments[1].type.nonempty
),
)
def struct(self, items, meta):
assert len(items) >= 1
name = items[0]
_check_keyword(sp(self.filename, meta), name)
members = {}
for d in items[1:]:
assert not d.expr
if d.name in members:
raise Err.MultipleDefinitions(sp(self.filename, meta),
"duplicate members in struct")
members[d.name] = d.type
return D.StructTypeDef(sp(self.filename, meta), name, members)
