def parse_document( txt: str, version: Optional[str] = None, uri: str = "", imported: bool = False ) -> D.Document: if version is None: # for now assume the version is 1.0 if the first line is "version <number>" # otherwise draft-2 version = "draft-2" for line in txt.split("\n"): line = line.strip() if line and line[0] != "#": if line.startswith("version ") and line[8].isdigit(): version = "1.0" break if not txt.strip(): return D.Document( SourcePosition(filename=uri, line=0, column=0, end_line=0, end_column=0), [], [], None, imported, ) try: return _DocTransformer(uri, imported).transform(parse(txt, "document", version)) except lark.exceptions.UnexpectedCharacters as exn: raise Err.ParserError(uri if uri != "" else "(in buffer)") from exn except lark.exceptions.UnexpectedToken as exn: raise Err.ParserError(uri if uri != "" else "(in buffer)") from exn
def infer_type(self, expr: E.Apply) -> T.Base: if len(expr.arguments) != 1: raise Error.WrongArity(expr, 1) if not isinstance(expr.arguments[0].type, T.Array): raise Error.StaticTypeMismatch(expr, T.Array(None), expr.arguments[0].type) return T.Int()
def __call__(self, expr: E.Apply, env: E.Env) -> V.Base: assert len(expr.arguments) == 2 lhs = expr.arguments[0] rhs = expr.arguments[1] if isinstance(lhs.type, T.Array): arr = lhs.eval(env) assert isinstance(arr, V.Array) assert isinstance(arr.type, T.Array) assert isinstance(arr.value, list) idx = rhs.eval(env).expect(T.Int()).value if idx < 0 or idx >= len(arr.value): raise Error.OutOfBounds(rhs) return arr.value[idx] # pyre-fixme if isinstance(lhs.type, T.Map): mp = lhs.eval(env) assert isinstance(mp, V.Map) assert isinstance(mp.type, T.Map) assert mp.type.item_type is not None assert isinstance(mp.value, list) ans = None key = rhs.eval(env).expect(mp.type.item_type[0]) for k, v in mp.value: if key == k: ans = v.expect(mp.type.item_type[1]) if ans is None: raise Error.OutOfBounds(rhs) # TODO: KeyNotFound return ans # pyre-fixme assert False # pyre-fixme
def document(self, items, meta): imports = [] structs = {} tasks = [] workflow = None for item in items: if isinstance(item, D.Task): tasks.append(item) elif isinstance(item, D.Workflow): if workflow is not None: raise Err.MultipleDefinitions( sp(self.filename, meta), "Document has multiple workflows") workflow = item elif isinstance(item, D.StructTypeDef): if item.name in structs: raise Err.MultipleDefinitions( sp(self.filename, meta), "multiple structs named " + item.name) structs[item.name] = item elif isinstance(item, lark.Tree) and item.data == "version": pass elif isinstance(item, D.DocImport): imports.append(item) else: assert False return D.Document(sp(self.filename, meta), imports, structs, tasks, workflow)
def _infer_type(self, type_env: Env.Types) -> T.Base: self.expr.infer_type(type_env, self._check_quant) if isinstance(self.expr.type, T.Array): if "sep" not in self.options: raise Error.StaticTypeMismatch( self, T.Array(None), self.expr.type, "array command placeholder must have 'sep'") # if sum(1 for t in [T.Int, T.Float, T.Boolean, T.String, T.File] if isinstance(self.expr.type.item_type, t)) == 0: # raise Error.StaticTypeMismatch(self, T.Array(None), self.expr.type, "cannot use array of complex types for command placeholder") elif "sep" in self.options: raise Error.StaticTypeMismatch( self, T.Array(None), self.expr.type, "command placeholder has 'sep' option for non-Array expression", ) if "true" in self.options or "false" in self.options: if not isinstance(self.expr.type, T.Boolean): raise Error.StaticTypeMismatch( self, T.Boolean(), self.expr.type, "command placeholder 'true' and 'false' options used with non-Boolean expression", ) if not ("true" in self.options and "false" in self.options): raise Error.StaticTypeMismatch( self, T.Boolean(), self.expr.type, "command placeholder with only one of 'true' and 'false' options", ) return T.String()
def typecheck_input(self, type_env: Env.Types, doc: TVDocument, check_quant: bool) -> None: # Check the input expressions against the callee's inputs. One-time use assert self.callee # Make a set of the input names which are required for this call to # typecheck. In the top-level workflow, nothing is actually required # as missing call inputs become workflow inputs required at runtime. required_inputs = (set(decl.name for decl in self.callee.required_inputs) if doc.imported else set()) # typecheck call inputs against task/workflow input declarations for name, expr in self.inputs.items(): decl = None if isinstance(self.callee, Task): for d in self.callee.inputs + self.callee.postinputs: if d.name == name: decl = d else: assert isinstance(self.callee, Workflow) for ele in self.callee.elements: if isinstance(ele, Decl) and ele.name == name: decl = ele if decl is None: raise Err.NoSuchInput(expr, name) expr.infer_type(type_env, check_quant).typecheck(decl.type) if name in required_inputs: required_inputs.remove(name) # Check whether any required inputs were missed if required_inputs: raise Err.MissingInput(self, self.name, required_inputs)
def typecheck(self, check_quant: bool = True) -> None: """Typecheck each task in the document, then the workflow, if any. Documents returned by :func:`~WDL.load` have already been typechecked.""" names = set() for _, namespace, _ in self.imports: if namespace in names: raise Err.MultipleDefinitions( self, "Multiple imports with namespace " + namespace) names.add(namespace) names = set() # typecheck each task for task in self.tasks: if task.name in names: raise Err.MultipleDefinitions( task, "Multiple tasks named " + task.name) names.add(task.name) task.typecheck(check_quant=check_quant) # typecheck the workflow if self.workflow: if self.workflow.name in names: raise Err.MultipleDefinitions( self.workflow, "Workflow name collides with a task also named " + self.workflow.name, ) self.workflow.typecheck(self, check_quant=check_quant)
def infer_type(self, expr: E.Apply) -> T.Base: assert len(expr.arguments) == 2 for arg in expr.arguments: if not isinstance(arg.type, T.Boolean): raise Error.IncompatibleOperand(arg, "non-Boolean operand to ||") if expr._check_quant and arg.type.optional: raise Error.IncompatibleOperand(arg, "optional Boolean? operand to ||") return T.Boolean()
def workflow(self, items, meta): elements = [] inputs = None outputs = None output_idents = None output_idents_pos = None parameter_meta = None meta_section = None for item in items[1:]: if isinstance(item, dict): if "inputs" in item: assert inputs is None inputs = item["inputs"] elif "outputs" in item: if outputs is not None: raise Err.MultipleDefinitions( sp(self.filename, meta), "redundant sections in workflow") outputs = item["outputs"] if "output_idents" in item: assert output_idents is None output_idents = item["output_idents"] output_idents_pos = item["pos"] elif "meta" in item: if meta_section is not None: raise Err.MultipleDefinitions( sp(self.filename, meta), "redundant sections in workflow") meta_section = item["meta"] elif "parameter_meta" in item: if parameter_meta is not None: raise Err.MultipleDefinitions( sp(self.filename, meta), "redundant sections in workflow") parameter_meta = item["parameter_meta"] else: assert False elif isinstance(item, (D.Call, D.Conditional, D.Decl, D.Scatter)): elements.append(item) else: assert False _check_keyword(sp(self.filename, meta), items[0].value) return D.Workflow( sp(self.filename, meta), items[0].value, inputs, elements, outputs, parameter_meta or dict(), meta_section or dict(), output_idents, output_idents_pos, )
def infer_type(self, expr: E.Apply) -> T.Base: if len(expr.arguments) != 1: raise Error.WrongArity(expr, 1) if not isinstance(expr.arguments[0].type, T.Array): raise Error.StaticTypeMismatch(expr.arguments[0], T.Array(None), expr.arguments[0].type) if expr.arguments[0].type.item_type is None: # TODO: error for 'indeterminate type' raise Error.EmptyArray(expr.arguments[0]) ty = expr.arguments[0].type.item_type assert isinstance(ty, T.Base) return T.Array(ty.copy(optional=False))
def infer_type(self, expr: E.Apply) -> T.Base: if len(expr.arguments) != 1: raise Error.WrongArity(expr, 1) expr.arguments[0].typecheck(T.Array(T.Any())) # TODO: won't handle implicit coercion from T to Array[T] assert isinstance(expr.arguments[0].type, T.Array) if expr.arguments[0].type.item_type is None: return T.Array(T.Any()) if not isinstance(expr.arguments[0].type.item_type, T.Array): raise Error.StaticTypeMismatch( expr.arguments[0], T.Array(T.Array(T.Any())), expr.arguments[0].type ) return expr.arguments[0].type
def _call_eager(self, expr: E.Apply, arguments: List[V.Base]) -> V.Base: arr = arguments[0] assert isinstance(arr, V.Array) for arg in arr.value: if not isinstance(arg, V.Null): return arg raise Error.NullValue(expr)
def _infer_type(self, type_env: Env.Types) -> T.Base: if not self.items: return T.Array(None) for item in self.items: item.infer_type(type_env, self._check_quant) # Start by assuming the type of the first item is the item type item_type: T.Base = self.items[0].type # Allow a mixture of Int and Float to construct Array[Float] if isinstance(item_type, T.Int): for item in self.items: if isinstance(item.type, T.Float): item_type = T.Float() # If any item is String, assume item type is String # If any item has optional quantifier, assume item type is optional # If all items have nonempty quantifier, assume item type is nonempty all_nonempty = len(self.items) > 0 for item in self.items: if isinstance(item.type, T.String): item_type = T.String(optional=item_type.optional) if item.type.optional: item_type = item_type.copy(optional=True) if isinstance(item.type, T.Array) and not item.type.nonempty: all_nonempty = False if isinstance(item_type, T.Array): item_type = item_type.copy(nonempty=all_nonempty) # Check all items are coercible to item_type for item in self.items: try: item.typecheck(item_type) except Error.StaticTypeMismatch: self._type = T.Array(item_type, optional=False, nonempty=True) raise Error.StaticTypeMismatch( self, item_type, item.type, "(inconsistent types within array)") from None return T.Array(item_type, optional=False, nonempty=True)
def task(self, items, meta): d = {} for item in items: if isinstance(item, dict): for k, v in item.items(): if k in d: raise Err.MultipleDefinitions( sp(self.filename, meta), "redundant sections in task" ) d[k] = v else: assert isinstance(item, str) assert "name" not in d d["name"] = item.value return D.Task( sp(self.filename, meta), d["name"], d.get("inputs", []), d.get("decls", []), d["command"], d.get("outputs", []), d.get("parameter_meta", {}), d.get("runtime", {}), d.get("meta", {}), )
def infer_type(self, expr: E.Apply) -> T.Base: assert len(expr.arguments) == 2 if ( ( expr._check_quant and expr.arguments[0].type.optional != expr.arguments[1].type.optional ) or ( self.name not in ["==", "!="] and (expr.arguments[0].type.optional or expr.arguments[1].type.optional) ) or ( not ( expr.arguments[0].type.copy(optional=False) == expr.arguments[1].type.copy(optional=False) or ( isinstance(expr.arguments[0].type, T.Int) and isinstance(expr.arguments[1].type, T.Float) ) or ( isinstance(expr.arguments[0].type, T.Float) and isinstance(expr.arguments[1].type, T.Int) ) ) ) ): raise Error.IncompatibleOperand( expr, "Cannot compare {} and {}".format( str(expr.arguments[0].type), str(expr.arguments[1].type) ), ) return T.Boolean()
def _call_eager(self, expr: E.Apply, arguments: List[V.Base]) -> V.Base: argument_values = [arg.coerce(ty) for arg, ty in zip(arguments, self.argument_types)] try: ans: V.Base = self.F(*argument_values) except Exception as exn: raise Error.EvalError(expr, "function evaluation failed") from exn return ans.coerce(self.return_type)
def infer_type(self, expr: E.Apply) -> T.Base: assert len(expr.arguments) == 1 if not isinstance(expr.arguments[0].type, T.Pair): raise Error.NotAPair(expr.arguments[0]) return expr.arguments[ 0].type.left_type if self.left else expr.arguments[ 0].type.right_type
def load(uri: str, path: List[str] = [], check_quant: bool = True, imported: Optional[bool] = False) -> Document: for fn in [uri] + [os.path.join(dn, uri) for dn in reversed(path)]: if os.path.exists(fn): with open(fn, "r") as infile: # read and parse the document doc = WDL._parser.parse_document(infile.read(), uri=uri, imported=imported) assert isinstance(doc, Document) # recursively descend into document's imports, and store the imported # documents into doc.imports # TODO: limit recursion; prevent mutual recursion for i in range(len(doc.imports)): try: subpath = [os.path.dirname(fn)] + path subdoc = load(doc.imports[i][0], subpath, check_quant=check_quant, imported=True) except Exception as exn: raise Err.ImportError(uri, doc.imports[i][0]) from exn doc.imports[i] = (doc.imports[i][0], doc.imports[i][1], subdoc) doc.typecheck(check_quant=check_quant) return doc raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), uri)
def task(self, items, meta): d = {"noninput_decls": []} for item in items: if isinstance(item, dict): for k, v in item.items(): if k == "noninput_decl": d["noninput_decls"].append(v) elif k in d: raise Err.MultipleDefinitions( sp(self.filename, meta), "redundant sections in task") else: d[k] = v else: assert isinstance(item, str) assert "name" not in d d["name"] = item.value _check_keyword(sp(self.filename, meta), d["name"]) return D.Task( sp(self.filename, meta), d["name"], d.get("inputs", None), d["noninput_decls"], d["command"], d.get("outputs", []), d.get("parameter_meta", {}), d.get("runtime", {}), d.get("meta", {}), )
def placeholder(self, items, meta): options = dict(items[:-1]) if len(options.items()) < len(items) - 1: raise Err.MultipleDefinitions( sp(self.filename, meta), "duplicate options in expression placeholder") return E.Placeholder(sp(self.filename, meta), options, items[-1])
def call_inputs(self, items, meta): d = dict() for k, v in items: if k in d: raise Err.MultipleDefinitions(sp(self.filename, meta), "duplicate keys in call inputs") d[k] = v return d
def meta_object(self, items, meta): d = dict() for k, v in items: if k in d: raise Err.MultipleDefinitions(sp(self.filename, meta), "duplicate keys in meta object") d[k] = v return d
def typecheck(self, expected: Optional[T.Base]) -> Base: "" if not self.items and isinstance(expected, T.Array): # the literal empty array satisfies any array type # (unless it has the nonempty quantifier) if expected.nonempty: raise Error.EmptyArray(self) return self return super().typecheck(expected) # pyre-ignore
def __init__(self, pos: SourcePosition, function: str, arguments: List[Base]) -> None: super().__init__(pos) try: self.function = _stdlib[function] self.function_name = function except KeyError: raise Error.NoSuchFunction(self, function) from None self.arguments = arguments
def eval(self, env: Env.Values) -> V.Base: "" try: if self.condition.eval(env).expect(T.Boolean()).value: ans = self.consequent.eval(env) else: ans = self.alternative.eval(env) return ans except ReferenceError: raise Error.NullValue(self) from None
def infer_type(self, expr: E.Apply) -> T.Base: if len(expr.arguments) != 2: raise Error.WrongArity(expr, 2) arg0ty: T.Base = expr.arguments[0].type if not isinstance(arg0ty, T.Array) or (expr._check_quant and arg0ty.optional): raise Error.StaticTypeMismatch(expr.arguments[0], T.Array(T.Any()), arg0ty) if isinstance(arg0ty.item_type, T.Any): # TODO: error for 'indeterminate type' raise Error.EmptyArray(expr.arguments[0]) arg1ty: T.Base = expr.arguments[1].type if not isinstance(arg1ty, T.Array) or (expr._check_quant and arg1ty.optional): raise Error.StaticTypeMismatch(expr.arguments[1], T.Array(T.Any()), arg1ty) if isinstance(arg1ty.item_type, T.Any): # TODO: error for 'indeterminate type' raise Error.EmptyArray(expr.arguments[1]) return T.Array( T.Pair(arg0ty.item_type, arg1ty.item_type), nonempty=(arg0ty.nonempty or arg1ty.nonempty), )
def infer_type(self, expr: E.Apply) -> T.Base: min_args = len(self.argument_types) for ty in reversed(self.argument_types): if ty.optional: min_args = min_args - 1 else: break if len(expr.arguments) > len(self.argument_types) or len(expr.arguments) < min_args: raise Error.WrongArity(expr, len(self.argument_types)) for i in range(len(expr.arguments)): try: expr.arguments[i].typecheck(self.argument_types[i]) except Error.StaticTypeMismatch: raise Error.StaticTypeMismatch( expr.arguments[i], self.argument_types[i], expr.arguments[i].type, "for {} argument #{}".format(self.name, i + 1), ) from None return self.return_type
def add_to_type_env(self, type_env: Env.Types) -> Env.Types: # Add an appropriate binding in the type env, after checking for name # collision. try: Env.resolve(type_env, [], self.name) raise Err.MultipleDefinitions( self, "Multiple declarations of " + self.name) except KeyError: pass ans: Env.Types = Env.bind(self.name, self.type, type_env, ctx=self) return ans
def infer_type(self, expr: E.Apply) -> T.Base: if len(expr.arguments) != 2: raise Error.WrongArity(expr, 2) expr.arguments[0].typecheck(T.String()) expr.arguments[1].typecheck(T.Array(T.String())) return T.Array( T.String(), nonempty=( isinstance(expr.arguments[1].type, T.Array) and expr.arguments[1].type.nonempty ), )
def struct(self, items, meta): assert len(items) >= 1 name = items[0] _check_keyword(sp(self.filename, meta), name) members = {} for d in items[1:]: assert not d.expr if d.name in members: raise Err.MultipleDefinitions(sp(self.filename, meta), "duplicate members in struct") members[d.name] = d.type return D.StructTypeDef(sp(self.filename, meta), name, members)