コード例 #1
0
ファイル: reader.py プロジェクト: codeaudit/hadrian
def jsonToAst(jsonInput):
    if isinstance(jsonInput, file):
        jsonInput = jsonInput.read()
    if isinstance(jsonInput, basestring):
        jsonInput = json.loads(jsonInput)
    
    avroTypeBuilder = AvroTypeBuilder()
            
    result = _readEngineConfig(jsonInput, avroTypeBuilder)
    avroTypeBuilder.resolveTypes()
    return result
コード例 #2
0
ファイル: reader.py プロジェクト: codeaudit/hadrian
def jsonToFcnDefs(jsonInput, where=""):
    if isinstance(jsonInput, file):
        jsonInput = jsonInput.read()
    if isinstance(jsonInput, basestring):
        jsonInput = json.loads(jsonInput)

    avroTypeBuilder = AvroTypeBuilder()
    
    result = _readFcnDefMap(jsonInput, where, avroTypeBuilder)
    avroTypeBuilder.resolveTypes()
    return result
コード例 #3
0
ファイル: reader.py プロジェクト: codeaudit/hadrian
def jsonToExpressionsAst(jsonInput, where=""):
    if isinstance(jsonInput, file):
        jsonInput = jsonInput.read()
    if isinstance(jsonInput, basestring):
        jsonInput = json.loads(jsonInput)

    avroTypeBuilder = AvroTypeBuilder()

    result = _readExpressionArray(jsonInput, where, avroTypeBuilder)
    avroTypeBuilder.resolveTypes()
    return result
コード例 #4
0
ファイル: reader.py プロジェクト: nkhuyu/hadrian
def jsonToFcnDefs(jsonInput, where=""):
    """Parse a JSON object of PFA function declarations as a PFA abstract syntax tree.

    :type x: open JSON file, JSON string, or Pythonized JSON
    :param x: PFA functions in a JSON object
    :rtype: dict of titus.pfaast.FcnDef
    :return: parsed functions as abstract syntax trees
    """
    if isinstance(jsonInput, file):
        jsonInput = jsonInput.read()
    if isinstance(jsonInput, basestring):
        jsonInput = json.loads(jsonInput)

    avroTypeBuilder = AvroTypeBuilder()
    
    result = _readFcnDefMap(jsonInput, where, avroTypeBuilder)
    avroTypeBuilder.resolveTypes()
    return result
コード例 #5
0
ファイル: reader.py プロジェクト: nkhuyu/hadrian
def jsonToExpressionsAst(jsonInput, where=""):
    """Parse a JSON array of PFA expressions as a PFA abstract syntax trees.

    :type x: open JSON file, JSON string, or Pythonized JSON
    :param x: PFA expressions in a JSON array
    :rtype: list of titus.pfaast.Expression
    :return: parsed expressions as a list of abstract syntax trees
    """
    if isinstance(jsonInput, file):
        jsonInput = jsonInput.read()
    if isinstance(jsonInput, basestring):
        jsonInput = json.loads(jsonInput)

    avroTypeBuilder = AvroTypeBuilder()

    result = _readExpressionArray(jsonInput, where, avroTypeBuilder)
    avroTypeBuilder.resolveTypes()
    return result
コード例 #6
0
ファイル: reader.py プロジェクト: Marigold/hadrian
def jsonToFcnDefs(jsonInput, where=""):
    """Parse a JSON object of PFA function declarations as a PFA abstract syntax tree.

    :type x: open JSON file, JSON string, or Pythonized JSON
    :param x: PFA functions in a JSON object
    :rtype: dict of titus.pfaast.FcnDef
    :return: parsed functions as abstract syntax trees
    """
    if isinstance(jsonInput, io.TextIOBase):
        jsonInput = jsonInput.read()
    if isinstance(jsonInput, str):
        jsonInput = json.loads(jsonInput)

    avroTypeBuilder = AvroTypeBuilder()
    
    result = _readFcnDefMap(jsonInput, where, avroTypeBuilder)
    avroTypeBuilder.resolveTypes()
    return result
コード例 #7
0
ファイル: reader.py プロジェクト: Marigold/hadrian
def jsonToExpressionsAst(jsonInput, where=""):
    """Parse a JSON array of PFA expressions as a PFA abstract syntax trees.

    :type x: open JSON file, JSON string, or Pythonized JSON
    :param x: PFA expressions in a JSON array
    :rtype: list of titus.pfaast.Expression
    :return: parsed expressions as a list of abstract syntax trees
    """
    if isinstance(jsonInput, io.TextIOBase):
        jsonInput = jsonInput.read()
    if isinstance(jsonInput, str):
        jsonInput = json.loads(jsonInput)

    avroTypeBuilder = AvroTypeBuilder()

    result = _readExpressionArray(jsonInput, where, avroTypeBuilder)
    avroTypeBuilder.resolveTypes()
    return result
コード例 #8
0
ファイル: reader.py プロジェクト: nkhuyu/hadrian
def jsonToAst(jsonInput):
    """Reads PFA from serialized JSON into an abstract syntax tree.

    :type jsonInput: open JSON file, JSON string, or Pythonized JSON
    :param jsonInput: input JSON
    :rtype: titus.pfaast.EngineConfig
    :return: a PFA configuration that has passed syntax but not semantics checks
    """

    if isinstance(jsonInput, file):
        jsonInput = jsonInput.read()
    if isinstance(jsonInput, basestring):
        jsonInput = json.loads(jsonInput)
    
    avroTypeBuilder = AvroTypeBuilder()
            
    result = _readEngineConfig(jsonInput, avroTypeBuilder)
    avroTypeBuilder.resolveTypes()
    return result
コード例 #9
0
ファイル: reader.py プロジェクト: Marigold/hadrian
def jsonToAst(jsonInput):
    """Reads PFA from serialized JSON into an abstract syntax tree.

    :type jsonInput: open JSON file, JSON string, or Pythonized JSON
    :param jsonInput: input JSON
    :rtype: titus.pfaast.EngineConfig
    :return: a PFA configuration that has passed syntax but not semantics checks
    """

    if isinstance(jsonInput, io.TextIOBase):
        jsonInput = jsonInput.read()
    if isinstance(jsonInput, str):
        jsonInput = json.loads(jsonInput)
    
    avroTypeBuilder = AvroTypeBuilder()
            
    result = _readEngineConfig(jsonInput, avroTypeBuilder)
    avroTypeBuilder.resolveTypes()
    return result
コード例 #10
0
def pmmlToAst(pmmlInput, options=None):
    """Load a PMML document and convert it to a PFA abstract syntax tree.

    :type pmmlInput: open XML file, gzip-compressed byte string, XML string, or file name string
    :param pmmlInput: input source for the PMML
    :type options: dict of option strings
    :param options: PMML-to-PFA conversion options
    :rtype: titus.pfaast.EngineConfig
    :return: converted PFA
    """

    if options is None:
        options = {}

    context = titus.pmml.version_independent.Context()
    context.avroTypeBuilder = AvroTypeBuilder()

    obj = loadPMML(pmmlInput, options.get("reader.processNamespaces", False))
    result = obj.toPFA(options, context)

    context.avroTypeBuilder.resolveTypes()
    return result
コード例 #11
0
ファイル: chain.py プロジェクト: animator/titus2
def ast(pfas,
        check=True,
        name=None,
        randseed=None,
        doc=None,
        version=None,
        metadata={},
        options={},
        tryYaml=False,
        verbose=False):
    """Create a single PFA from a chained workflow, returning the result as an abstract syntax tree.

    :type pfas: list of titus.pfaast.EngineConfig, Pythonized JSON, or JSON strings
    :param pfas: PFA documents for which the output of document *i* is the input to document *i + 1*
    :type check: bool
    :param check: test the chained PFA for validity
    :type name: string or ``None``
    :param name: optional name for the chained PFA
    :type randseed: integer or ``None``
    :param randseed: optional random number seed for the chained PFA
    :type doc: string or ``None``
    :param doc: optional documentation string for the chained PFA
    :type version: integer or ``None``
    :param version: optional version number for the chained PFA
    :type metadata: dict of strings
    :param metadata: metadata for the chained PFA (default is ``{}``)
    :type options: dict of Pythonized JSON
    :param options: implementation options for the chained PFA (default is ``{}``)
    :type tryYaml: bool
    :param tryYaml: if ``True``, attempt to interpret ``pfas`` as YAML (assuming they fail as JSON)
    :type verbose: bool
    :param verbose: if ``True``, write status messages to standard output
    :rtype: titus.pfaast.EngineConfig
    :return: a PFA document representing the chained workflow
    """

    # normalize all input forms to ASTs
    if verbose:
        sys.stderr.write(time.asctime() + " Converting all inputs to ASTs\n")
    asts = []
    for i, src in enumerate(pfas):
        if verbose:
            sys.stderr.write(time.asctime() + "     step {0}\n".format(i + 1))
        if isinstance(src, EngineConfig):
            ast = src
        elif isinstance(src, dict):
            ast = titus.reader.jsonToAst(src)
        else:
            try:
                ast = titus.reader.jsonToAst(src)
            except ValueError:
                if tryYaml:
                    ast = titus.reader.yamlToAst(src)
                else:
                    raise
        asts.append(ast)
    pfas = asts

    # helper functions for transforming names
    def split(t):
        if "." in t:
            return t[:t.rindex(".")], t[t.rindex(".") + 1:]
        else:
            return None, t

    def join(ns, n):
        if ns is None or ns == "":
            return n
        else:
            return ns + "." + n

    def prefixType(i, pfa, t):
        ns, n = split(t)
        return join(ns, "Step{0:d}_{1}_{2}".format(i + 1, pfa.name, n))

    def prefixAction(i, pfa):
        return "step{0:d}_{1}_action".format(i + 1, pfa.name)

    def prefixFcnRef(i, pfa, x):
        if x.startswith("u."):
            return "u.step{0:d}_{1}_fcn_{2}".format(i + 1, pfa.name, x[2:])
        else:
            return x

    def prefixFcnDef(i, pfa, x):
        return "step{0:d}_{1}_fcn_{2}".format(i + 1, pfa.name, x)

    def prefixCell(i, pfa, x):
        return "step{0:d}_{1}_{2}".format(i + 1, pfa.name, x)

    def prefixPool(i, pfa, x):
        return "step{0:d}_{1}_{2}".format(i + 1, pfa.name, x)

    # define new names for all types to avoid type name collisions
    if verbose:
        sys.stderr.write(time.asctime() +
                         " Changing type names to avoid collisions\n")
    originalNameToNewName = {}
    for i, pfa in enumerate(pfas):
        originalNameToNewName[i] = {}
        for typeName in list(pfa.inputPlaceholder.parser.names.names.keys()):
            keyTypeName = typeName
            if (typeName[0] == "."):
                keyTypeName = keyTypeName[1:]
            originalNameToNewName[i][keyTypeName] = prefixType(
                i, pfa, typeName)

    # but any names in the input to the first and the output from the last should not be changed
    def trivialName(i, avroType, memo):
        if isinstance(avroType, AvroArray):
            trivialName(i, avroType.items, memo)
        elif isinstance(avroType, AvroMap):
            trivialName(i, avroType.values, memo)
        elif isinstance(avroType, AvroUnion):
            for t in avroType.types:
                trivialName(i, t, memo)
        elif isinstance(avroType, (AvroFixed, AvroEnum)):
            t = avroType.fullName
            originalNameToNewName[i][t] = t
        elif isinstance(avroType, AvroRecord):
            t = avroType.fullName
            if t not in memo:
                memo.add(t)
                for f in avroType.fields:
                    trivialName(i, f.avroType, memo)
                originalNameToNewName[i][t] = t

    trivialName(0, pfas[0].input, set())
    trivialName(len(pfas) - 1, pfas[-1].output, set())

    # ensure that chained types match and will be given the same names
    if verbose:
        sys.stderr.write(
            time.asctime() +
            " Verifying that input/output schemas match along the chain\n")

    def chainPair(i, first, second, memo):
        if isinstance(first, AvroNull) and isinstance(second, AvroNull):
            return True
        elif isinstance(first, AvroBoolean) and isinstance(
                second, AvroBoolean):
            return True
        elif isinstance(first, AvroInt) and isinstance(second, AvroInt):
            return True
        elif isinstance(first, AvroLong) and isinstance(second, AvroLong):
            return True
        elif isinstance(first, AvroFloat) and isinstance(second, AvroFloat):
            return True
        elif isinstance(first, AvroDouble) and isinstance(second, AvroDouble):
            return True
        elif isinstance(first, AvroBytes) and isinstance(second, AvroBytes):
            return True
        elif isinstance(first, AvroFixed) and isinstance(second, AvroFixed):
            if first.size == second.size:
                originalNameToNewName[i + 1][
                    second.fullName] = originalNameToNewName[i][first.fullName]
                return True
            else:
                return False
        elif isinstance(first, AvroString) and isinstance(second, AvroString):
            return True
        elif isinstance(first, AvroEnum) and isinstance(second, AvroEnum):
            if first.symbols == second.symbols:
                originalNameToNewName[i + 1][
                    second.fullName] = originalNameToNewName[i][first.fullName]
                return True
            else:
                return False
        elif isinstance(first, AvroArray) and isinstance(second, AvroArray):
            return chainPair(i, first.items, second.items, memo)
        elif isinstance(first, AvroMap) and isinstance(second, AvroMap):
            return chainPair(i, first.values, second.values, memo)
        elif isinstance(first, AvroRecord) and isinstance(second, AvroRecord):
            if first.fullName not in memo:
                memo.add(first.fullName)
                if len(first.fields) != len(second.fields):
                    return False
                for f1, f2 in zip(first.fields, second.fields):
                    if f1.name != f2.name:
                        return False
                    elif not chainPair(i, f1.avroType, f2.avroType, memo):
                        return False
                originalNameToNewName[i + 1][
                    second.fullName] = originalNameToNewName[i][first.fullName]
                return True
        elif isinstance(first, AvroUnion) and isinstance(second, AvroUnion):
            for yt in second.types:
                if not any(chainPair(i, xt, yt, memo) for xt in first.types):
                    return False
            return True
        else:
            return False

    for i in range(len(pfas) - 1):
        first = pfas[i].output
        second = pfas[i + 1].input
        if not chainPair(i, first, second, set()):
            raise PFAChainError(
                "output of engine {0}: {1} not compatible with input of engine {2}: {3}"
                .format(i + 1, ts(first), i + 2, ts(second)))

    def rename(i, avroType, memo):
        if isinstance(avroType, AvroArray):
            return {"type": "array", "items": rename(i, avroType.items, memo)}
        elif isinstance(avroType, AvroMap):
            return {"type": "map", "values": rename(i, avroType.values, memo)}
        elif isinstance(avroType, AvroUnion):
            return [rename(i, t, memo) for t in avroType.types]
        elif isinstance(avroType, AvroFixed):
            ns, n = split(originalNameToNewName[i][avroType.fullName])
            out = {"type": "fixed", "name": n, "size": avroType.size}
            if ns is not None:
                out["namespace"] = ns
            return out
        elif isinstance(avroType, AvroEnum):
            ns, n = split(originalNameToNewName[i][avroType.fullName])
            out = {"type": "enum", "name": n, "symbols": avroType.symbols}
            if ns is not None:
                out["namespace"] = ns
            return out
        elif isinstance(avroType, AvroRecord):
            newName = originalNameToNewName[i][avroType.fullName]
            if newName in memo:
                return memo[newName]
            else:
                ns, n = split(newName)
                out = {"type": "record", "name": n, "fields": []}
                if ns is not None:
                    out["namespace"] = ns
                memo[newName] = join(ns, n)
                for f in avroType.fields:
                    newf = {
                        "name": f.name,
                        "type": rename(i, f.avroType, memo)
                    }
                    if f.default is not None:
                        newf["default"] = f.default
                    if f.order is not None:
                        newf["order"] = f.order
                    out["fields"].append(newf)
                return out
        else:
            return jsonlib.loads(repr(avroType))

    avroTypeBuilder = AvroTypeBuilder()
    memo = {}

    def newPlaceholder(i, oldAvroType):
        newAvroType = rename(i, oldAvroType, {})
        return avroTypeBuilder.makePlaceholder(jsonlib.dumps(newAvroType),
                                               memo)

    # combined name, if not explicitly set
    if name is None:
        name = "Chain_" + "_".join(pfa.name for pfa in pfas)

    # combined method (fold not supported yet, but could be)
    method = Method.MAP
    for pfa in pfas:
        if pfa.method == Method.EMIT:
            method = Method.EMIT
        elif pfa.method == Method.FOLD:
            raise NotImplementedError(
                "chaining of fold-type scoring engines has not been implemented yet"
            )

    # no zero or merge until we support fold method
    zero = None
    merge = None

    # input/output types from first and last
    inputPlaceholder = newPlaceholder(0, pfas[0].input)
    outputPlaceholder = newPlaceholder(len(pfas) - 1, pfas[-1].output)

    if verbose:
        sys.stderr.write(
            time.asctime() +
            " Adding [name, instance, metadata, actionsStarted, actionsFinished, version] as model parameters\n"
        )

    cells = {
        "name":
        Cell(newPlaceholder(0, AvroString()), jsonlib.dumps(""), False, False,
             CellPoolSource.EMBEDDED),
        "instance":
        Cell(newPlaceholder(0, AvroInt()), jsonlib.dumps(0), False, False,
             CellPoolSource.EMBEDDED),
        "metadata":
        Cell(newPlaceholder(0, AvroMap(AvroString())), jsonlib.dumps({}),
             False, False, CellPoolSource.EMBEDDED),
        "actionsStarted":
        Cell(newPlaceholder(0, AvroLong()), jsonlib.dumps(0), False, False,
             CellPoolSource.EMBEDDED),
        "actionsFinished":
        Cell(newPlaceholder(0, AvroLong()), jsonlib.dumps(0), False, False,
             CellPoolSource.EMBEDDED)
    }
    if version is not None:
        cells["version"] = Cell(newPlaceholder(0, AvroInt()), 0, False, False,
                                CellPoolSource.EMBEDDED)
    pools = {}

    if verbose:
        sys.stderr.write(time.asctime() +
                         " Converting scoring engine algorithm\n")

    # all code will go into user functions, including begin/action/end
    fcns = {}

    begin = [
        CellTo("name", [], Ref("name")),
        CellTo("instance", [], Ref("instance")),
        CellTo("metadata", [], Ref("metadata"))
    ]
    if version is not None:
        begin.append(CellTo("version", [], Ref("version")))

    action = [
        CellTo("actionsStarted", [], Ref("actionsStarted")),
        CellTo("actionsFinished", [], Ref("actionsFinished"))
    ]

    end = [
        CellTo("actionsStarted", [], Ref("actionsStarted")),
        CellTo("actionsFinished", [], Ref("actionsFinished"))
    ]

    for i, pfa in enumerate(pfas):
        if verbose:
            sys.stderr.write(time.asctime() +
                             "     step {0}: {1}\n".format(i + 1, pfa.name))

        thisActionFcnName = prefixAction(i, pfa)
        if i + 1 < len(pfas):
            nextActionFcnName = prefixAction(i + 1, pfas[i + 1])
        else:
            nextActionFcnName = None

        # this is a closure; it must be defined in the loop to pick up free variables
        lazyFcnReplacer = None

        def genericReplacer(expr, self):
            if isinstance(expr, FcnDef):
                return FcnDef(
                    [{
                        list(t.keys())[0]: newPlaceholder(
                            i,
                            list(t.values())[0])
                    } for t in expr.params],
                    newPlaceholder(i, expr.ret),
                    [
                        x.replace(lazyFcnReplacer) for x in expr.body
                    ],  # this is the one place where we should pass down fcnReplacer rather than self
                    expr.pos)
            elif isinstance(expr, FcnRef):
                return FcnRef(prefixFcnRef(i, pfa, expr.name), expr.pos)
            elif isinstance(expr, FcnRefFill):
                return FcnRefFill(
                    prefixFcnRef(i, pfa, expr.name),
                    dict((k, v.replace(self))
                         for k, v in list(expr.fill.items())), expr.pos)
            elif isinstance(
                    expr, CallUserFcn
            ):  # TODO: need to change the symbols of the corresponding enum
                return CallUserFcn(expr.name.replace(self),
                                   [x.replace(self) for x in expr.args],
                                   expr.pos)
            elif isinstance(expr, Call):
                if pfa.method == Method.EMIT and i + 1 < len(
                        pfas) and expr.name == "emit":
                    return Call("u." + nextActionFcnName,
                                [x.replace(self) for x in expr.args], expr.pos)
                else:
                    return Call(prefixFcnRef(i, pfa, expr.name),
                                [x.replace(self) for x in expr.args], expr.pos)
            elif isinstance(expr, Literal):
                return Literal(newPlaceholder(i, expr.avroType), expr.value,
                               expr.pos)
            elif isinstance(expr, NewObject):
                return NewObject(
                    dict((k, v.replace(self))
                         for k, v in list(expr.fields.items())),
                    newPlaceholder(i, expr.avroType), expr.pos)
            elif isinstance(expr, NewArray):
                return NewArray([x.replace(self) for x in expr.items],
                                newPlaceholder(i, expr.avroType), expr.pos)
            elif isinstance(expr, CellGet):
                return CellGet(prefixCell(i, pfa, expr.cell),
                               [x.replace(self) for x in expr.path], expr.pos)
            elif isinstance(expr, CellTo):
                return CellTo(prefixCell(i, pfa, expr.cell),
                              [x.replace(self) for x in expr.path],
                              expr.to.replace(self), expr.pos)
            elif isinstance(expr, PoolGet):
                return PoolGet(prefixPool(i, pfa, expr.pool),
                               [x.replace(self) for x in expr.path], expr.pos)
            elif isinstance(expr, PoolTo):
                return PoolTo(prefixPool(i, pfa, expr.pool),
                              [x.replace(self) for x in expr.path],
                              expr.to.replace(self), expr.init.replace(self),
                              expr.pos)
            elif isinstance(expr, CastCase):
                return CastCase(newPlaceholder(i, expr.avroType), expr.named,
                                [x.replace(self) for x in expr.body], expr.pos)
            elif isinstance(expr, Upcast):
                return Upcast(expr.expr.replace(self),
                              newPlaceholder(i, expr.avroType), expr.pos)

        genericReplacer.isDefinedAt = lambda x: isinstance(
            x, (FcnDef, FcnRef, FcnRefFill, CallUserFcn, Call, Literal,
                NewObject, CellGet, CellTo, PoolGet, PoolTo, CastCase, Upcast))

        def fcnReplacer(expr):
            return genericReplacer(expr, fcnReplacer)

        fcnReplacer.isDefinedAt = genericReplacer.isDefinedAt

        lazyFcnReplacer = fcnReplacer

        # add statements to begin
        def beginReplacer(expr):
            if isinstance(expr, Ref):
                if expr.name in ("name", "instance",
                                 "metadata") or (version is not None
                                                 and expr.name == "version"):
                    return CellGet(expr.name, [], expr.pos)
                else:
                    return expr
            else:
                return genericReplacer(expr, beginReplacer)

        beginReplacer.isDefinedAt = lambda x: isinstance(
            x, Ref) or genericReplacer.isDefinedAt(x)
        begin.extend([x.replace(beginReplacer) for x in pfa.begin])

        # add statements to end
        def endReplacer(expr):
            if isinstance(expr, Ref):
                if expr.name in ("name", "instance", "metadata",
                                 "actionsStarted", "actionsFinished") or (
                                     version is not None
                                     and expr.name == "version"):
                    return CellGet(expr.name, [], expr.pos)
                else:
                    return expr
            else:
                return genericReplacer(expr, endReplacer)

        endReplacer.isDefinedAt = lambda x: isinstance(
            x, Ref) or genericReplacer.isDefinedAt(x)
        end.extend([x.replace(endReplacer) for x in pfa.end])

        # convert the action into a user function
        def actionReplacer(expr):
            if isinstance(expr, Ref):
                if expr.name in ("name", "instance", "metadata",
                                 "actionsStarted", "actionsFinished") or (
                                     version is not None
                                     and expr.name == "version"):
                    return CellGet(expr.name, [], expr.pos)
                else:
                    return expr
            else:
                return genericReplacer(expr, actionReplacer)

        actionReplacer.isDefinedAt = lambda x: isinstance(
            x, Ref) or genericReplacer.isDefinedAt(x)

        body = [x.replace(actionReplacer) for x in pfa.action]

        if method == Method.MAP:
            # if the overall method is MAP, then we know that all of the individual engines are MAP
            # the overall action calls a nested chain of engines-as-functions and each engine-as-a-function just does its job and returns (body is unmodified)
            fcns[thisActionFcnName] = FcnDef(
                [{
                    "input": newPlaceholder(i, pfa.input)
                }], newPlaceholder(i, pfa.output), body)
            if i == 0:
                action.append(Call("u." + thisActionFcnName, [Ref("input")]))
            else:
                action[-1] = Call("u." + thisActionFcnName, [action[-1]])

        elif method == Method.EMIT:
            # if the overall method is EMIT, then some individual engines might be MAP or might be EMIT
            # the overall action calls the first engine-as-a-function and the engines-as-functions call each other (body is modified)
            if pfa.method == Method.MAP and i + 1 < len(pfas):
                body = [
                    Call("u." + nextActionFcnName, [Do(body)]),
                    LiteralNull()
                ]
            elif pfa.method == Method.MAP:
                body = [Call("emit", [Do(body)])]
            elif pfa.method == Method.EMIT:
                body.append(LiteralNull())

            fcns[thisActionFcnName] = FcnDef(
                [{
                    "input": newPlaceholder(i, pfa.input)
                }], newPlaceholder(i, AvroNull()), body)
            if i == 0:
                action.append(Call("u." + thisActionFcnName, [Ref("input")]))

        # convert all of the user functions into user functions
        for fcnName, fcnDef in list(pfa.fcns.items()):
            # note: some of these user-defined functions may call emit; if so, they'll call the right emit
            fcns[prefixFcnDef(i, pfa, fcnName)] = FcnDef(
                [{
                    list(t.keys())[0]: newPlaceholder(i,
                                                      list(t.values())[0])
                } for t in fcnDef.paramsPlaceholder],
                newPlaceholder(i, fcnDef.ret),
                [x.replace(fcnReplacer) for x in fcnDef.body], fcnDef.pos)

    if verbose:
        sys.stderr.write(time.asctime() +
                         " Create types for model parameters\n")

    for i, pfa in enumerate(pfas):
        if verbose and len(pfa.cells) > 0:
            sys.stderr.write(time.asctime() + "     step {0}:\n".format(i + 1))
        for cellName, cell in list(pfa.cells.items()):
            if verbose:
                sys.stderr.write(time.asctime() +
                                 "         cell {0}\n".format(cellName))
            newCell = Cell(newPlaceholder(i, cell.avroType), cell.init,
                           cell.shared, cell.rollback, cell.source, cell.pos)
            cells[prefixCell(i, pfa, cellName)] = newCell
            if cell.source == "embedded":

                def converter(avroType):
                    original = jsonDecoder(cell.avroType,
                                           jsonlib.loads(cell.init))
                    return jsonlib.dumps(jsonEncoder(avroType, original))

                newCell.converter = converter

    for i, pfa in enumerate(pfas):
        if verbose and len(pfa.pools) > 0:
            sys.stderr.write(time.asctime() + "     step {0}:\n".format(i + 1))
        for poolName, pool in list(pfa.pools.items()):
            if verbose:
                sys.stderr.write(time.asctime() +
                                 "         pool {0}\n".format(poolName))
            newPool = Pool(newPlaceholder(i, pool.avroType), pool.init,
                           pool.shared, pool.rollback, pool.source, pool.pos)
            pools[prefixPool(i, pfa, poolName)] = newPool
            if pool.source == "embedded":

                def converter(avroType):
                    original = jsonDecoder(pool.avroType,
                                           jsonlib.loads(pool.init))
                    return jsonlib.dumps(jsonEncoder(avroType, original))

                newPool.converter = converter

    # make sure all the types work together
    if verbose: sys.stderr.write(time.asctime() + " Resolving all types\n")
    avroTypeBuilder.resolveTypes()

    if verbose:
        sys.stderr.write(time.asctime() +
                         " Converting the model parameters themselves\n")

    for i, pfa in enumerate(pfas):
        if verbose and len(pfa.cells) > 0:
            sys.stderr.write(time.asctime() + "     step {0}:\n".format(i + 1))
        for cellName, cell in list(pfa.cells.items()):
            if verbose:
                sys.stderr.write(time.asctime() +
                                 "         cell {0}\n".format(cellName))
            if cell.source == "embedded":
                newCell = cells[prefixCell(i, pfa, cellName)]
                newCell.init = newCell.converter(newCell.avroType)

    for i, pfa in enumerate(pfas):
        if verbose and len(pfa.pools) > 0:
            sys.stderr.write(time.asctime() + "     step {0}:\n".format(i + 1))
        for poolName, pool in list(pfa.pools.items()):
            if verbose:
                sys.stderr.write(time.asctime() +
                                 "         pool {0}\n".format(poolName))
            if pool.source == "embedded":
                newPool = pools[prefixPool(i, pfa, poolName)]
                newPool.init = newPool.converter(newPool.avroType)

    # randseed, doc, version, metadata, and options need to be explicitly set

    # return a (possibly checked) AST
    out = EngineConfig(name, method, inputPlaceholder, outputPlaceholder,
                       begin, action, end, fcns, zero, merge, cells, pools,
                       randseed, doc, version, metadata, options)
    if check:
        if verbose:
            sys.stderr.write(time.asctime() + " Verifying PFA validity\n")
        PFAEngine.fromAst(out)

    if verbose: sys.stderr.write(time.asctime() + " Done\n")
    return out
コード例 #12
0
ファイル: chain.py プロジェクト: ajutzeler/hadrian
def ast(pfas, check=True, name=None, randseed=None, doc=None, version=None, metadata={}, options={}, tryYaml=False, verbose=False):
    """Create a single PFA from a chained workflow, returning the result as an abstract syntax tree.

    :type pfas: list of titus.pfaast.EngineConfig, Pythonized JSON, or JSON strings
    :param pfas: PFA documents for which the output of document *i* is the input to document *i + 1*
    :type check: bool
    :param check: test the chained PFA for validity
    :type name: string or ``None``
    :param name: optional name for the chained PFA
    :type randseed: integer or ``None``
    :param randseed: optional random number seed for the chained PFA
    :type doc: string or ``None``
    :param doc: optional documentation string for the chained PFA
    :type version: integer or ``None``
    :param version: optional version number for the chained PFA
    :type metadata: dict of strings
    :param metadata: metadata for the chained PFA (default is ``{}``)
    :type options: dict of Pythonized JSON
    :param options: implementation options for the chained PFA (default is ``{}``)
    :type tryYaml: bool
    :param tryYaml: if ``True``, attempt to interpret ``pfas`` as YAML (assuming they fail as JSON)
    :type verbose: bool
    :param verbose: if ``True``, write status messages to standard output
    :rtype: titus.pfaast.EngineConfig
    :return: a PFA document representing the chained workflow
    """

    # normalize all input forms to ASTs
    if verbose: sys.stderr.write(time.asctime() + " Converting all inputs to ASTs\n")
    asts = []
    for i, src in enumerate(pfas):
        if verbose: sys.stderr.write(time.asctime() + "     step {0}\n".format(i + 1))
        if isinstance(src, EngineConfig):
            ast = src
        elif isinstance(src, dict):
            ast = titus.reader.jsonToAst(src)
        else:
            try:
                ast = titus.reader.jsonToAst(src)
            except ValueError:
                if tryYaml:
                    ast = titus.reader.yamlToAst(src)
                else:
                    raise
        asts.append(ast)
    pfas = asts

    # helper functions for transforming names
    def split(t):
        if "." in t:
            return t[:t.rindex(".")], t[t.rindex(".") + 1:]
        else:
            return None, t

    def join(ns, n):
        if ns is None or ns == "":
            return n
        else:
            return ns + "." + n

    def prefixType(i, pfa, t):
        ns, n = split(t)
        return join(ns, "Step{0:d}_{1}_{2}".format(i + 1, pfa.name, n))

    def prefixAction(i, pfa):
        return "step{0:d}_{1}_action".format(i + 1, pfa.name)

    def prefixFcnRef(i, pfa, x):
        if x.startswith("u."):
            return "u.step{0:d}_{1}_fcn_{2}".format(i + 1, pfa.name, x[2:])
        else:
            return x

    def prefixFcnDef(i, pfa, x):
        return "step{0:d}_{1}_fcn_{2}".format(i + 1, pfa.name, x)

    def prefixCell(i, pfa, x):
        return "step{0:d}_{1}_{2}".format(i + 1, pfa.name, x)

    def prefixPool(i, pfa, x):
        return "step{0:d}_{1}_{2}".format(i + 1, pfa.name, x)

    # define new names for all types to avoid type name collisions
    if verbose: sys.stderr.write(time.asctime() + " Changing type names to avoid collisions\n")
    originalNameToNewName = {}
    for i, pfa in enumerate(pfas):
        originalNameToNewName[i] = {}
        for typeName in pfa.inputPlaceholder.parser.names.names.keys():
            originalNameToNewName[i][typeName] = prefixType(i, pfa, typeName)

    # but any names in the input to the first and the output from the last should not be changed
    def trivialName(i, avroType, memo):
        if isinstance(avroType, AvroArray):
            trivialName(i, avroType.items, memo)
        elif isinstance(avroType, AvroMap):
            trivialName(i, avroType.values, memo)
        elif isinstance(avroType, AvroUnion):
            for t in avroType.types:
                trivialName(i, t, memo)
        elif isinstance(avroType, (AvroFixed, AvroEnum)):
            t = avroType.fullName
            originalNameToNewName[i][t] = t
        elif isinstance(avroType, AvroRecord):
            t = avroType.fullName
            if t not in memo:
                memo.add(t)
                for f in avroType.fields:
                    trivialName(i, f.avroType, memo)
                originalNameToNewName[i][t] = t
    trivialName(0, pfas[0].input, set())
    trivialName(len(pfas) - 1, pfas[-1].output, set())

    # ensure that chained types match and will be given the same names
    if verbose: sys.stderr.write(time.asctime() + " Verifying that input/output schemas match along the chain\n")
    def chainPair(i, first, second, memo):
        if isinstance(first, AvroNull) and isinstance(second, AvroNull):
            return True
        elif isinstance(first, AvroBoolean) and isinstance(second, AvroBoolean):
            return True
        elif isinstance(first, AvroInt) and isinstance(second, AvroInt):
            return True
        elif isinstance(first, AvroLong) and isinstance(second, AvroLong):
            return True
        elif isinstance(first, AvroFloat) and isinstance(second, AvroFloat):
            return True
        elif isinstance(first, AvroDouble) and isinstance(second, AvroDouble):
            return True
        elif isinstance(first, AvroBytes) and isinstance(second, AvroBytes):
            return True
        elif isinstance(first, AvroFixed) and isinstance(second, AvroFixed):
            t = avroType.fullName
            if first.size == second.size:
                originalNameToNewName[i + 1][second.fullName] = originalNameToNewName[i][first.fullName]
                return True
            else:
                return False
        elif isinstance(first, AvroString) and isinstance(second, AvroString):
            return True
        elif isinstance(first, AvroEnum) and isinstance(second, AvroEnum):
            if first.symbols == second.symbols:
                originalNameToNewName[i + 1][second.fullName] = originalNameToNewName[i][first.fullName]
                return True
            else:
                return False
        elif isinstance(first, AvroArray) and isinstance(second, AvroArray):
            return chainPair(i, first.items, second.items, memo)
        elif isinstance(first, AvroMap) and isinstance(second, AvroMap):
            return chainPair(i, first.values, second.values, memo)
        elif isinstance(first, AvroRecord) and isinstance(second, AvroRecord):
            if first.fullName not in memo:
                memo.add(first.fullName)
                if len(first.fields) != len(second.fields):
                    return False
                for f1, f2 in zip(first.fields, second.fields):
                    if f1.name != f2.name:
                        return False
                    elif not chainPair(i, f1.avroType, f2.avroType, memo):
                        return False
                originalNameToNewName[i + 1][second.fullName] = originalNameToNewName[i][first.fullName]
                return True
        elif isinstance(first, AvroUnion) and isinstance(second, AvroUnion):
            for yt in second.types:
                if not any(chainPair(i, xt, yt, memo) for xt in first.types):
                    return False
            return True
        else:
            return False
    for i in xrange(len(pfas) - 1):
        first = pfas[i].output
        second = pfas[i + 1].input
        if not chainPair(i, first, second, set()):
            raise PFAChainError("output of engine {0}: {1} not compatible with input of engine {2}: {3}".format(i + 1, ts(first), i + 2, ts(second)))

    def rename(i, avroType, memo):
        if isinstance(avroType, AvroArray):
            return {"type": "array", "items": rename(i, avroType.items, memo)}
        elif isinstance(avroType, AvroMap):
            return {"type": "map", "values": rename(i, avroType.values, memo)}
        elif isinstance(avroType, AvroUnion):
            return [rename(i, t, memo) for t in avroType.types]
        elif isinstance(avroType, AvroFixed):
            ns, n = split(originalNameToNewName[i][avroType.fullName])
            out = {"type": "fixed", "name": n, "size": avroType.size}
            if ns is not None:
                out["namespace"] = ns
            return out
        elif isinstance(avroType, AvroEnum):
            ns, n = split(originalNameToNewName[i][avroType.fullName])
            out = {"type": "enum", "name": n, "symbols": avroType.symbols}
            if ns is not None:
                out["namespace"] = ns
            return out
        elif isinstance(avroType, AvroRecord):
            newName = originalNameToNewName[i][avroType.fullName]
            if newName in memo:
                return memo[newName]
            else:
                ns, n = split(newName)
                out = {"type": "record", "name": n, "fields": []}
                if ns is not None:
                    out["namespace"] = ns
                memo[newName] = join(ns, n)
                for f in avroType.fields:
                    newf = {"name": f.name, "type": rename(i, f.avroType, memo)}
                    if f.default is not None:
                        newf["default"] = f.default
                    if f.order is not None:
                        newf["order"] = f.order
                    out["fields"].append(newf)
                return out
        else:
            return jsonlib.loads(repr(avroType))

    avroTypeBuilder = AvroTypeBuilder()
    memo = {}
    def newPlaceholder(i, oldAvroType):
        newAvroType = rename(i, oldAvroType, {})
        return avroTypeBuilder.makePlaceholder(jsonlib.dumps(newAvroType), memo)
            
    # combined name, if not explicitly set
    if name is None:
        name = "Chain_" + "_".join(pfa.name for pfa in pfas)

    # combined method (fold not supported yet, but could be)
    method = Method.MAP
    for pfa in pfas:
        if pfa.method == Method.EMIT:
            method = Method.EMIT
        elif pfa.method == Method.FOLD:
            raise NotImplementedError("chaining of fold-type scoring engines has not been implemented yet")

    # no zero or merge until we support fold method
    zero = None
    merge = None

    # input/output types from first and last
    inputPlaceholder = newPlaceholder(0, pfas[0].input)
    outputPlaceholder = newPlaceholder(len(pfas) - 1, pfas[-1].output)

    if verbose: sys.stderr.write(time.asctime() + " Adding [name, instance, metadata, actionsStarted, actionsFinished, version] as model parameters\n")

    cells = {"name": Cell(newPlaceholder(0, AvroString()), jsonlib.dumps(""), False, False, CellPoolSource.EMBEDDED),
             "instance": Cell(newPlaceholder(0, AvroInt()), jsonlib.dumps(0), False, False, CellPoolSource.EMBEDDED),
             "metadata": Cell(newPlaceholder(0, AvroMap(AvroString())), jsonlib.dumps({}), False, False, CellPoolSource.EMBEDDED),
             "actionsStarted": Cell(newPlaceholder(0, AvroLong()), jsonlib.dumps(0), False, False, CellPoolSource.EMBEDDED),
             "actionsFinished": Cell(newPlaceholder(0, AvroLong()), jsonlib.dumps(0), False, False, CellPoolSource.EMBEDDED)}
    if version is not None:
        cells["version"] = Cell(newPlaceholder(0, AvroInt()), 0, False, False, CellPoolSource.EMBEDDED)
    pools = {}

    if verbose: sys.stderr.write(time.asctime() + " Converting scoring engine algorithm\n")

    # all code will go into user functions, including begin/action/end
    fcns = {}

    begin = [CellTo("name", [], Ref("name")),
             CellTo("instance", [], Ref("instance")),
             CellTo("metadata", [], Ref("metadata"))]
    if version is not None:
        begin.append(CellTo("version", [], Ref("version")))

    action = [CellTo("actionsStarted", [], Ref("actionsStarted")),
              CellTo("actionsFinished", [], Ref("actionsFinished"))]

    end = [CellTo("actionsStarted", [], Ref("actionsStarted")),
           CellTo("actionsFinished", [], Ref("actionsFinished"))]

    for i, pfa in enumerate(pfas):
        if verbose: sys.stderr.write(time.asctime() + "     step {0}: {1}\n".format(i + 1, pfa.name))

        thisActionFcnName = prefixAction(i, pfa)
        if i + 1 < len(pfas):
            nextActionFcnName = prefixAction(i + 1, pfas[i + 1])
        else:
            nextActionFcnName = None

        # this is a closure; it must be defined in the loop to pick up free variables
        lazyFcnReplacer = None
        def genericReplacer(expr, self):
            if isinstance(expr, FcnDef):
                return FcnDef([{t.keys()[0]: newPlaceholder(i, t.values()[0])} for t in expr.params],
                              newPlaceholder(i, expr.ret),
                              [x.replace(lazyFcnReplacer) for x in expr.body],     # this is the one place where we should pass down fcnReplacer rather than self
                              expr.pos)
            elif isinstance(expr, FcnRef):
                return FcnRef(prefixFcnRef(i, pfa, expr.name), epxr.pos)
            elif isinstance(expr, FcnRefFill):
                return FcnRefFill(prefixFcnRef(i, pfa, expr.name),
                                  dict((k, v.replace(self)) for k, v in expr.fill.items()),
                                  expr.pos)
            elif isinstance(expr, CallUserFcn):   # TODO: need to change the symbols of the corresponding enum
                return CallUserFcn(expr.name.replace(self),
                                   [x.replace(self) for x in expr.args],
                                   expr.pos)
            elif isinstance(expr, Call):
                if pfa.method == Method.EMIT and i + 1 < len(pfas) and expr.name == "emit":
                    return Call("u." + nextActionFcnName,
                                [x.replace(self) for x in expr.args],
                                expr.pos)
                else:
                    return Call(prefixFcnRef(i, pfa, expr.name),
                                [x.replace(self) for x in expr.args],
                                expr.pos)
            elif isinstance(expr, Literal):
                return Literal(newPlaceholder(i, expr.avroType),
                               expr.value,
                               expr.pos)
            elif isinstance(expr, NewObject):
                return NewObject(dict((k, v.replace(self)) for k, v in expr.fields.items()),
                                 newPlaceholder(i, expr.avroType),
                                 expr.pos)
            elif isinstance(expr, NewArray):
                return NewArray([x.replace(self) for x in expr.items],
                                newPlaceholder(i, expr.avroType),
                                expr.pos)
            elif isinstance(expr, CellGet):
                return CellGet(prefixCell(i, pfa, expr.cell),
                               [x.replace(self) for x in expr.path],
                               expr.pos)
            elif isinstance(expr, CellTo):
                return CellTo(prefixCell(i, pfa, expr.cell),
                              [x.replace(self) for x in expr.path],
                              expr.to.replace(self),
                              expr.pos)
            elif isinstance(expr, PoolGet):
                return PoolGet(prefixPool(i, pfa, expr.pool),
                               [x.replace(self) for x in expr.path],
                               expr.pos)
            elif isinstance(expr, PoolTo):
                return PoolTo(prefixPool(i, pfa, expr.pool),
                              [x.replace(self) for x in expr.path],
                              expr.to.replace(self),
                              expr.init.replace(self),
                              expr.pos)
            elif isinstance(expr, CastCase):
                return CastCase(newPlaceholder(i, expr.avroType),
                                expr.named,
                                [x.replace(self) for x in expr.body],
                                expr.pos)
            elif isinstance(expr, Upcast):
                return Upcast(expr.expr.replace(self),
                              newPlaceholder(i, expr.avroType),
                              expr.pos)
        genericReplacer.isDefinedAt = lambda x: isinstance(x, (FcnDef, FcnRef, FcnRefFill, CallUserFcn, Call, Literal, NewObject, CellGet, CellTo, PoolGet, PoolTo, CastCase, Upcast))

        def fcnReplacer(expr):
            return genericReplacer(expr, fcnReplacer)
        fcnReplacer.isDefinedAt = genericReplacer.isDefinedAt

        lazyFcnReplacer = fcnReplacer

        # add statements to begin
        def beginReplacer(expr):
            if isinstance(expr, Ref):
                if expr.name in ("name", "instance", "metadata") or (version is not None and expr.name == "version"):
                    return CellGet(expr.name, [], expr.pos)
                else:
                    return expr
            else:
                return genericReplacer(expr, beginReplacer)
        beginReplacer.isDefinedAt = lambda x: isinstance(x, Ref) or genericReplacer.isDefinedAt(x)
        begin.extend([x.replace(beginReplacer) for x in pfa.begin])

        # add statements to end
        def endReplacer(expr):
            if isinstance(expr, Ref):
                if expr.name in ("name", "instance", "metadata", "actionsStarted", "actionsFinished") or (version is not None and expr.name == "version"):
                    return CellGet(expr.name, [], expr.pos)
                else:
                    return expr
            else:
                return genericReplacer(expr, endReplacer)
        endReplacer.isDefinedAt = lambda x: isinstance(x, Ref) or genericReplacer.isDefinedAt(x)
        end.extend([x.replace(endReplacer) for x in pfa.end])

        # convert the action into a user function
        def actionReplacer(expr):
            if isinstance(expr, Ref):
                if expr.name in ("name", "instance", "metadata", "actionsStarted", "actionsFinished") or (version is not None and expr.name == "version"):
                    return CellGet(expr.name, [], expr.pos)
                else:
                    return expr
            else:
                return genericReplacer(expr, actionReplacer)
        actionReplacer.isDefinedAt = lambda x: isinstance(x, Ref) or genericReplacer.isDefinedAt(x)

        body = [x.replace(actionReplacer) for x in pfa.action]

        if method == Method.MAP:
            # if the overall method is MAP, then we know that all of the individual engines are MAP
            # the overall action calls a nested chain of engines-as-functions and each engine-as-a-function just does its job and returns (body is unmodified)
            fcns[thisActionFcnName] = FcnDef([{"input": newPlaceholder(i, pfa.input)}], newPlaceholder(i, pfa.output), body)
            if i == 0:
                action.append(Call("u." + thisActionFcnName, [Ref("input")]))
            else:
                action[-1] = Call("u." + thisActionFcnName, [action[-1]])

        elif method == Method.EMIT:
            # if the overall method is EMIT, then some individual engines might be MAP or might be EMIT
            # the overall action calls the first engine-as-a-function and the engines-as-functions call each other (body is modified)
            if pfa.method == Method.MAP and i + 1 < len(pfas):
                body = [Call("u." + nextActionFcnName, [Do(body)]), LiteralNull()]
            elif pfa.method == Method.MAP:
                body = [Call("emit", [Do(body)])]
            elif pfa.method == Method.EMIT:
                body.append(LiteralNull())

            fcns[thisActionFcnName] = FcnDef([{"input": newPlaceholder(i, pfa.input)}], newPlaceholder(i, AvroNull()), body)
            if i == 0:
                action.append(Call("u." + thisActionFcnName, [Ref("input")]))

        # convert all of the user functions into user functions
        for fcnName, fcnDef in pfa.fcns.items():
            # note: some of these user-defined functions may call emit; if so, they'll call the right emit
            fcns[prefixFcnDef(i, pfa, fcnName)] = FcnDef([{t.keys()[0]: newPlaceholder(i, t.values()[0])} for t in fcnDef.paramsPlaceholder],
                                                         newPlaceholder(i, fcnDef.ret),
                                                         [x.replace(fcnReplacer) for x in fcnDef.body],
                                                         fcnDef.pos)

    if verbose: sys.stderr.write(time.asctime() + " Create types for model parameters\n")

    for i, pfa in enumerate(pfas):
        if verbose and len(pfa.cells) > 0: sys.stderr.write(time.asctime() + "     step {0}:\n".format(i + 1))
        for cellName, cell in pfa.cells.items():
            if verbose: sys.stderr.write(time.asctime() + "         cell {0}\n".format(cellName))
            newCell = Cell(newPlaceholder(i, cell.avroType), cell.init, cell.shared, cell.rollback, cell.source, cell.pos)
            cells[prefixCell(i, pfa, cellName)] = newCell
            if cell.source == "embedded":
                def converter(avroType):
                    original = jsonDecoder(cell.avroType, jsonlib.loads(cell.init))
                    return jsonlib.dumps(jsonEncoder(avroType, original))
                newCell.converter = converter
                
    for i, pfa in enumerate(pfas):
        if verbose and len(pfa.pools) > 0: sys.stderr.write(time.asctime() + "     step {0}:\n".format(i + 1))
        for poolName, pool in pfa.pools.items():
            if verbose: sys.stderr.write(time.asctime() + "         pool {0}\n".format(poolName))
            newPool = Pool(newPlaceholder(i, pool.avroType), pool.init, pool.shared, pool.rollback, pool.source, pool.pos)
            pools[prefixPool(i, pfa, poolName)] = newPool
            if pool.source == "embedded":
                def converter(avroType):
                    original = jsonDecoder(pool.avroType, jsonlib.loads(pool.init))
                    return jsonlib.dumps(jsonEncoder(avroType, original))
                newPool.converter = converter
                
    # make sure all the types work together
    if verbose: sys.stderr.write(time.asctime() + " Resolving all types\n")
    avroTypeBuilder.resolveTypes()

    if verbose: sys.stderr.write(time.asctime() + " Converting the model parameters themselves\n")

    for i, pfa in enumerate(pfas):
        if verbose and len(pfa.cells) > 0: sys.stderr.write(time.asctime() + "     step {0}:\n".format(i + 1))
        for cellName, cell in pfa.cells.items():
            if verbose: sys.stderr.write(time.asctime() + "         cell {0}\n".format(cellName))
            if cell.source == "embedded":
                newCell = cells[prefixCell(i, pfa, cellName)]
                newCell.init = newCell.converter(newCell.avroType)

    for i, pfa in enumerate(pfas):
        if verbose and len(pfa.pools) > 0: sys.stderr.write(time.asctime() + "     step {0}:\n".format(i + 1))
        for poolName, pool in pfa.pools.items():
            if verbose: sys.stderr.write(time.asctime() + "         pool {0}\n".format(poolName))
            if pool.source == "embedded":
                newPool = pools[prefixPool(i, pfa, poolName)]
                newPool.init = newPool.converter(newPool.avroType)

    # randseed, doc, version, metadata, and options need to be explicitly set

    # return a (possibly checked) AST
    out = EngineConfig(name,
                       method,
                       inputPlaceholder,
                       outputPlaceholder,
                       begin,
                       action,
                       end,
                       fcns,
                       zero,
                       merge,
                       cells,
                       pools,
                       randseed,
                       doc,
                       version,
                       metadata,
                       options)
    if check:
        if verbose: sys.stderr.write(time.asctime() + " Verifying PFA validity\n")
        PFAEngine.fromAst(out)

    if verbose: sys.stderr.write(time.asctime() + " Done\n")
    return out