def load(ctx):
ctx.readme = libcell(".readme")
def recreate(ctx, name0, with_buffer, with_schema, inchannels):
name = "." + name0
ctx.struc = context(name="struc",context=ctx)
ctx.struc.storage = libcell(name+".storage")
ctx.struc.form = libcell(name+".form")
ctx.struc.data = libmixedcell(name+".data",
form_cell = ctx.struc.form,
storage_cell = ctx.struc.storage,
)
schema = None
if with_schema:
ctx.struc.schema = libcell(name+".schema")
schema = ctx.struc.schema
bufferwrapper = None
if with_buffer:
ctx.struc.buffer_storage = libcell(name+".buffer_storage")
ctx.struc.buffer_form = libcell(name+".buffer_form")
ctx.struc.buffer_data = libmixedcell(name+".buffer_data",
form_cell = ctx.struc.buffer_form,
storage_cell = ctx.struc.buffer_storage,
)
bufferwrapper = BufferWrapper(
ctx.struc.buffer_data,
ctx.struc.buffer_storage,
ctx.struc.buffer_form
)
ctx.hub = StructuredCell(
"hub",
ctx.struc.data,
storage = ctx.struc.storage,
form = ctx.struc.form,
schema = schema,
buffer = bufferwrapper,
inchannels = inchannels,
outchannels = [],
)
ctx.auth_json = context(name="auth_json", context=ctx)
recreate(ctx.auth_json, "auth_json.struc", with_buffer=False, with_schema=False, inchannels=[])
ctx.auth = context(name="auth", context=ctx)
recreate(ctx.auth, "auth.struc", with_buffer=False, with_schema=True, inchannels=[])
ctx.err = context(name="err", context=ctx)
recreate(ctx.err, "err.struc", with_buffer=True, with_schema=True, inchannels=[])
ctx.nauth = context(name="nauth", context=ctx)
try:
recreate(ctx.nauth, "nauth.struc", with_buffer=False, with_schema=False, inchannels=[("z",)])
except:
import traceback; traceback.print_exc()
def translate_module(node, root, namespace, inchannels, outchannels):
module_type = node["module_type"]
language = node["language"]
dependencies = node["dependencies"]
path = node["path"]
parent = get_path(root, path[:-1], None, None)
name = path[-1]
for c in inchannels + outchannels:
assert not len(c) #should have been checked by highlevel
subcontext = context(toplevel=False)
setattr(parent, name, subcontext)
codecell = core_cell("plain")
subcontext.code = codecell
if node.get("fingertip_no_recompute"):
codecell._fingertip_recompute = False
if node.get("fingertip_no_remote"):
codecell._fingertip_remote = False
pathstr = "." + ".".join(path)
checksum = node.get("checksum")
if checksum is not None:
codecell._set_checksum(checksum, initial=True)
if "mount" in node:
codecell2 = core_cell("text")
subcontext.code2 = codecell2
codecell2.mount(**node["mount"])
mode = node["mount"].get("mode", "rw")
if mode == "rw":
codecell2.bilink(codecell)
elif mode == "r":
codecell2.connect(codecell)
elif mode == "w":
codecell.connect(codecell2)
subcontext.module_cell = core_cell("plain")
subcontext.gen_module_cell = transformer({
"module_type": ("input", "str"),
"language": ("input", "str"),
"module_code": ("input", "plain"),
"dependencies": ("input", "plain"),
"result": ("output", "plain")
})
subcontext.gen_module_cell.code.cell().set(
inspect.getsource(gen_module_cell))
subcontext.gen_module_cell.module_type.cell().set(module_type)
subcontext.gen_module_cell.language.cell().set(language)
subcontext.gen_module_cell.dependencies.cell().set(dependencies)
codecell.connect(subcontext.gen_module_cell.module_code)
subcontext.gen_module_cell.result.connect(subcontext.module_cell)
namespace[path, "source"] = subcontext.module_cell, node
namespace[path, "target"] = codecell, node
return subcontext
def main():
global ctx, compute
ctx = context(toplevel=True)
ctx.select_compute = libcell("compute.select")
ctx.compute = macro(select_params, lib="compute")
ctx.select_compute.connect(ctx.compute.code)
ctx.compute.which.cell().set("pi")
compute = ctx.compute.gen_context
ctx.iterations = cell().set(10000)
ctx.iterations.connect(compute.iterations)
ctx.result = cell()
compute.result.connect(ctx.result)
async def main():
ctx = context(toplevel=True)
ctx.cell1 = cell("int").set(1)
ctx.cell2 = cell("int").set(2)
#ctx.code = cell("transformer")
#ctx.code = cell("transformer").set("c = 'test'")
#ctx.code = cell("transformer").set("raise Exception")
#ctx.code = cell("transformer").set("import time; time.sleep(2); c = a + b")
ctx.code = cell("transformer").set("a + b")
ctx.result = cell("int")
ctx.tf = transformer({
"a": "input",
"b": "input",
"c": "output"
})
ctx.cell1_unilink = unilink(ctx.cell1)
ctx.cell1_unilink.connect(ctx.tf.a)
ctx.cell2.connect(ctx.tf.b)
ctx.code_copy = cell("transformer")
ctx.code.connect(ctx.code_copy)
ctx.code_copy.connect(ctx.tf.code)
ctx.result_unilink = unilink(ctx.result)
ctx.tf.c.connect(ctx.result_unilink)
ctx.result_copy = cell("int")
ctx.result.connect(ctx.result_copy)
await ctx.computation(1)
print("STOP")
print(ctx.cell1.value, ctx.cell1, ctx.cell1.status)
print(ctx.cell2.value, ctx.cell2, ctx.cell2.status)
print(ctx.code.value, ctx.code, ctx.code.status)
print(ctx.code_copy.value, ctx.code_copy, ctx.code_copy.status)
print(ctx.result.value, ctx.result, ctx.result.status)
print(ctx.result_copy.value, ctx.result_copy, ctx.result_copy.status)
print(ctx.tf.value, ctx.tf, ctx.tf.status)
print(ctx.status)
print(ctx.tf.exception)
await ctx.computation()
ctx.cell1.set(10)
await ctx.computation()
print(ctx.result.value, ctx.status)
ctx.code.set("c = a + b + 1000")
await ctx.computation()
print(ctx.result.value, ctx.status)
print("Introduce delay...")
ctx.code.set("import time; time.sleep(2); c = -(a + b)")
await ctx.computation(1.0)
print("after 1.0 sec...")
print(ctx.result.value, ctx.status)
print("...")
await ctx.computation()
print(ctx.result.value, ctx.status)
def build_structured_cell(ctx,
name,
inchannels,
outchannels,
*,
fingertip_no_remote,
fingertip_no_recompute,
mount=None,
return_context=False,
hash_pattern=None):
#print("build_structured_cell", name)
name2 = name + STRUC_ID
c = context(toplevel=False)
setattr(ctx, name2, c)
if mount is not None:
mount.pop("as_directory", None)
c.mount(**mount)
c.data = core_cell("mixed")
c.data._hash_pattern = hash_pattern
c.auth = core_cell("mixed")
c.auth._hash_pattern = hash_pattern
c.schema = core_cell("plain")
c.buffer = core_cell("mixed")
c.buffer._hash_pattern = hash_pattern
sc = StructuredCell(data=c.data,
auth=c.auth,
schema=c.schema,
buffer=c.buffer,
inchannels=inchannels,
outchannels=outchannels,
hash_pattern=hash_pattern)
c.example_data = core_cell("mixed")
c.example_buffer = core_cell("mixed")
c.example = StructuredCell(c.example_data,
buffer=c.example_buffer,
schema=c.schema)
for cc in (c.data, c.buffer, c.schema, c.auth, c.example_data,
c.example_buffer):
if fingertip_no_recompute:
cc._fingertip_recompute = False
if fingertip_no_remote:
cc._fingertip_remote = False
if return_context:
return sc, c
else:
return sc
def define_ctx():
with macro_mode_on():
ctx = context(toplevel=True)
ctx.cell1 = cell("json").set(1)
ctx.cell2 = cell("json").set(2)
ctx.result = cell("json")
ctx.tf = transformer({"a": "input", "b": "input", "c": "output"})
ctx.cell1_link = link(ctx.cell1)
ctx.cell1_link.connect(ctx.tf.a)
ctx.cell2.connect(ctx.tf.b)
ctx.code = pytransformercell().set("c = a + b")
ctx.code.connect(ctx.tf.code)
ctx.result_link = link(ctx.result)
ctx.tf.c.connect(ctx.result_link)
return ctx
def main():
global ctx, compute
ctx = context(toplevel=True)
ctx.select_compute = libcell("compute.select")
ctx.compute = macro(select_params, lib="compute")
ctx.select_compute.connect(ctx.compute.code)
###ctx.compute.which.cell().set("pi") # TODO: malfunctioning
### KLUDGE
ctx.which_cell = cell().set("pi")
ctx.which_cell.connect(ctx.compute.which)
### /KLUDGE
compute = ctx.compute.ctx
ctx.iterations = cell().set(10000)
ctx.iterations.connect(compute.iterations)
ctx.result = cell()
compute.result.connect(ctx.result)
def reset_backend(share_schemas=True, with_hash_pattern=True):
hp = hash_pattern if with_hash_pattern else None
global ctx, s, s2, s3
if ctx is not None:
ctx.compute() # makes no difference, but could be easier debugging
ctx.destroy()
ctx = context(toplevel=True)
ctx.data = cell("mixed", hash_pattern=hp)
ctx.buffer = cell("mixed", hash_pattern=hp)
ctx.schema = cell("plain")
ctx.sc = StructuredCell(
buffer=ctx.buffer,
data=ctx.data,
schema=ctx.schema,
hash_pattern=hp
)
s = ctx.sc.handle
ctx.data2 = cell("mixed", hash_pattern=hp)
ctx.buffer2 = cell("mixed", hash_pattern=hp)
if share_schemas:
schema2 = ctx.schema
else:
ctx.schema2 = cell("plain")
schema2 = ctx.schema2
ctx.sc2 = StructuredCell(
buffer=ctx.buffer2,
data=ctx.data2,
schema=schema2,
hash_pattern=hp
)
s2 = ctx.sc2.handle
hp3 = None # never use hash pattern for this one
ctx.data3 = cell("mixed", hash_pattern=hp3)
ctx.buffer3 = cell("mixed", hash_pattern=hp3)
if share_schemas:
schema3 = ctx.schema
else:
ctx.schema3 = cell("plain")
schema3 = ctx.schema3
ctx.sc3 = StructuredCell(
buffer=ctx.buffer3,
data=ctx.data3,
schema=schema3,
hash_pattern=hp3
)
s3 = ctx.sc3.handle
def define_ctx():
with macro_mode_on():
ctx = context(toplevel=True)
ctx.cell1 = cell().set(1)
ctx.cell2 = cell().set(2)
ctx.result = cell()
ctx.tf = transformer({
"a": "input",
"b": "input",
"c": "output"
})
ctx.cell1_link = link(ctx.cell1)
ctx.cell1_link.connect(ctx.tf.a)
ctx.cell2.connect(ctx.tf.b)
ctx.code = pytransformercell().set("c = a + b")
ctx.code.connect(ctx.tf.code)
ctx.result_link = link(ctx.result)
ctx.tf.c.connect(ctx.result_link)
return ctx
def transform_job(rqdata):
x = decode(rqdata, as_cells=True)
transformer_params, output_signature, cells, _, _ = x
inputs = cells.keys()
with macro_mode_on():
ctx = context(toplevel=True)
tf = ctx.TRANSFORMER = transformer(transformer_params)
for k in inputs:
setattr(ctx, k, cells[k])
cells[k].connect(getattr(tf, k))
outputpin = getattr(tf, tf._output_name)
for n, outp in enumerate(output_signature):
name = "RESULT" + str(n + 1)
if outp == "mixed":
c2 = cell("text")
setattr(ctx, name + "_storage", c2)
c3 = cell("json")
setattr(ctx, name + "_form", c3)
c = cell("mixed", storage_cell=c2, form_cell=c3)
else:
c = cell(outp)
setattr(ctx, name, c)
outputpin.connect(c)
while tf.status() not in ("OK", "ERROR"):
ctx.equilibrate(1)
result = {}
if tf.status() != "OK":
result["ERROR"] = ctx.tf.transformer.EXCEPTION
else:
for n, outp in enumerate(output_signature):
name = "RESULT" + str(n + 1)
data = getattr(ctx, name)
if outp == "mixed":
storage = getattr(ctx, name + "_storage")
result["STORAGE"] = storage
form = getattr(ctx, name + "_form")
result["FORM"] = form
result[outp] = data
for key, c in list(result.items()):
val = c.serialize_buffer()
result[key] = val
return result
def macro_code(ctx, param):
ctx.sub = context()
ctx.a = cell().set(1000 + param)
ctx.b = cell().set(2000 + param)
ctx.result = cell()
ctx.tf = transformer({
"a": "input",
"b": "input",
"c": "output"
})
ctx.a.connect(ctx.tf.a)
ctx.b.connect(ctx.tf.b)
ctx.code = cell("transformer").set("print('TRANSFORM'); import time; time.sleep(2); c = a + b")
ctx.code.connect(ctx.tf.code)
ctx.tf.c.connect(ctx.result)
assert param != 999 # on purpose
if param > 1:
ctx.d = cell().set(42)
# raise Exception("on purpose") #causes the macro reconstruction to fail
pass # For some reason, comments at the end are not captured with inspect.get_source?
def create(ctx, with_buffer, with_schema, inchannels):
with macro_mode_on():
ctx.struc = context(name="struc", context=ctx)
ctx.struc.storage = cell("text")
ctx.struc.form = cell("json")
ctx.struc.data = cell(
"mixed",
form_cell=ctx.struc.form,
storage_cell=ctx.struc.storage,
)
schema = None
if with_schema:
ctx.struc.schema = cell("json")
schema = ctx.struc.schema
bufferwrapper = None
if with_buffer:
ctx.struc.buffer_storage = cell("text")
ctx.struc.buffer_form = cell("json")
ctx.struc.buffer_data = cell(
"mixed",
form_cell=ctx.struc.buffer_form,
storage_cell=ctx.struc.buffer_storage,
)
bufferwrapper = BufferWrapper(ctx.struc.buffer_data,
ctx.struc.buffer_storage,
ctx.struc.buffer_form)
ctx.hub = StructuredCell(
"hub",
ctx.struc.data,
storage=ctx.struc.storage,
form=ctx.struc.form,
schema=schema,
buffer=bufferwrapper,
inchannels=inchannels,
outchannels=[],
)
def recreate(ctx, name0, with_buffer, with_schema, inchannels):
name = "." + name0
ctx.struc = context(name="struc",context=ctx)
ctx.struc.storage = libcell(name+".storage")
ctx.struc.form = libcell(name+".form")
ctx.struc.data = libmixedcell(name+".data",
form_cell = ctx.struc.form,
storage_cell = ctx.struc.storage,
)
schema = None
if with_schema:
ctx.struc.schema = libcell(name+".schema")
schema = ctx.struc.schema
bufferwrapper = None
if with_buffer:
ctx.struc.buffer_storage = libcell(name+".buffer_storage")
ctx.struc.buffer_form = libcell(name+".buffer_form")
ctx.struc.buffer_data = libmixedcell(name+".buffer_data",
form_cell = ctx.struc.buffer_form,
storage_cell = ctx.struc.buffer_storage,
)
bufferwrapper = BufferWrapper(
ctx.struc.buffer_data,
ctx.struc.buffer_storage,
ctx.struc.buffer_form
)
ctx.hub = StructuredCell(
"hub",
ctx.struc.data,
storage = ctx.struc.storage,
form = ctx.struc.form,
schema = schema,
buffer = bufferwrapper,
inchannels = inchannels,
outchannels = [],
)
def create(ctx, with_buffer, with_schema, inchannels):
with macro_mode_on():
ctx.struc = context(name="struc",context=ctx)
ctx.struc.storage = cell("text")
ctx.struc.form = cell("json")
ctx.struc.data = cell("mixed",
form_cell = ctx.struc.form,
storage_cell = ctx.struc.storage,
)
schema = None
if with_schema:
ctx.struc.schema = cell("json")
schema = ctx.struc.schema
bufferwrapper = None
if with_buffer:
ctx.struc.buffer_storage = cell("text")
ctx.struc.buffer_form = cell("json")
ctx.struc.buffer_data = cell("mixed",
form_cell = ctx.struc.buffer_form,
storage_cell = ctx.struc.buffer_storage,
)
bufferwrapper = BufferWrapper(
ctx.struc.buffer_data,
ctx.struc.buffer_storage,
ctx.struc.buffer_form
)
ctx.hub = StructuredCell(
"hub",
ctx.struc.data,
storage = ctx.struc.storage,
form = ctx.struc.form,
schema = schema,
buffer = bufferwrapper,
inchannels = inchannels,
outchannels = [],
)
def recreate(ctx, name0, with_buffer, with_schema, inchannels):
name = "." + name0
ctx.struc = context(name="struc", context=ctx)
ctx.struc.storage = libcell(name + ".storage")
ctx.struc.form = libcell(name + ".form")
ctx.struc.data = libmixedcell(
name + ".data",
form_cell=ctx.struc.form,
storage_cell=ctx.struc.storage,
)
schema = None
if with_schema:
ctx.struc.schema = libcell(name + ".schema")
schema = ctx.struc.schema
bufferwrapper = None
if with_buffer:
ctx.struc.buffer_storage = libcell(name + ".buffer_storage")
ctx.struc.buffer_form = libcell(name + ".buffer_form")
ctx.struc.buffer_data = libmixedcell(
name + ".buffer_data",
form_cell=ctx.struc.buffer_form,
storage_cell=ctx.struc.buffer_storage,
)
bufferwrapper = BufferWrapper(ctx.struc.buffer_data,
ctx.struc.buffer_storage,
ctx.struc.buffer_form)
ctx.hub = StructuredCell(
"hub",
ctx.struc.data,
storage=ctx.struc.storage,
form=ctx.struc.form,
schema=schema,
buffer=bufferwrapper,
inchannels=inchannels,
outchannels=[],
)
import seamless
from seamless.core import macro_mode_on
from seamless.core import context, cell, transformer, StructuredCell
import numpy as np
with macro_mode_on():
ctx = context(toplevel=True)
ctx.inp_struc = context(name="inp_struc",context=ctx)
ctx.inp_struc.storage = cell("text")
ctx.inp_struc.form = cell("json")
ctx.inp_struc.data = cell("mixed",
form_cell = ctx.inp_struc.form,
storage_cell = ctx.inp_struc.storage,
)
ctx.inp_struc.schema = cell("json")
ctx.inp = StructuredCell(
"inp",
ctx.inp_struc.data,
storage = ctx.inp_struc.storage,
form = ctx.inp_struc.form,
schema = ctx.inp_struc.schema,
buffer = None,
inchannels = [("a",)],
outchannels = [()]
)
ctx.a = cell("json")
ctx.inp.connect_inchannel(ctx.a, ("a",))
ctx.tf = transformer({
"inp": ("input", "copy", "silk"),
"c": "output",
raise NotImplementedError
import seamless
from seamless.core import macro_mode_on
from seamless.core import context, cell, transformer, StructuredCell
from seamless.core.structured_cell import BufferWrapper, StructuredCellState
import numpy as np
with macro_mode_on():
ctx = context(toplevel=True)
def create(ctx, mount, state=None):
with macro_mode_on():
ctx.hub_struc = context(name="hub_struc",context=ctx)
ctx.hub_struc.storage = cell("text")
ctx.hub_struc.form = cell("json")
ctx.hub_struc.data = cell("mixed",
form_cell = ctx.hub_struc.form,
storage_cell = ctx.hub_struc.storage,
)
ctx.hub_struc.schema = cell("json")
ctx.hub_struc.buffer_storage = cell("text")
ctx.hub_struc.buffer_form = cell("json")
ctx.hub_struc.buffer_data = cell("mixed",
form_cell = ctx.hub_struc.buffer_form,
storage_cell = ctx.hub_struc.buffer_storage,
)
bufferwrapper = BufferWrapper(
ctx.hub_struc.buffer_data,
ctx.hub_struc.buffer_storage,
ctx.hub_struc.buffer_form
def translate_compiled_transformer(node, root, namespace, inchannels,
outchannels, lib_path00, is_lib):
#TODO: still a lot of common code with translate_py_transformer, put in functions
inchannels = [ic for ic in inchannels if ic[0] != "code"]
main_module_inchannels = [("objects", ) + ic[1:] for ic in inchannels
if ic[0] == "_main_module"]
inchannels = [ic for ic in inchannels if ic[0] != "_main_module"]
parent = get_path(root, node["path"][:-1], None, None)
name = node["path"][-1]
lib_path0 = lib_path00 + "." + name if lib_path00 is not None else None
ctx = context(context=parent, name=name)
setattr(parent, name, ctx)
input_name = node["INPUT"]
result_name = node["RESULT"]
if len(inchannels):
lib_path0 = None #partial authority or no authority; no library update in either case
for c in inchannels:
assert (not len(
c)) or c[0] != result_name #should have been checked by highlevel
with_result = node["with_result"]
assert with_result #compiled transformers must have with_result
buffered = node["buffered"]
mount = node.get("mount", {})
plain = node["plain"]
input_state = node.get("stored_state_input", None)
if input_state is None:
input_state = node.get("cached_state_input", None)
inp, inp_ctx = build_structured_cell(ctx,
input_name,
True,
plain,
buffered,
inchannels, [()],
input_state,
lib_path0,
return_context=True)
setattr(ctx, input_name, inp)
if "input_schema" in mount:
inp_ctx.schema.mount(**mount["input_schema"])
for inchannel in inchannels:
path = node["path"] + inchannel
namespace[path, True] = inp.inchannels[inchannel], node
assert result_name not in node[
"pins"] #should have been checked by highlevel
assert "translator_result_" not in node[
"pins"] #should have been checked by highlevel
all_pins = {}
inputpins = []
for pinname, pin in node["pins"].items():
p = {"io": "input"}
p.update(pin)
if p["io"] == "input":
inputpins.append(pinname)
all_pins[pinname] = p
all_pins[result_name] = "output"
if node["SCHEMA"]:
assert with_result
all_pins[node["SCHEMA"]] = {
"io": "input",
"transfer_mode": "json",
"access_mode": "json",
"content_type": "json"
}
in_equilibrium = node.get("in_equilibrium", False)
temp = node.get("TEMP")
if temp is None:
temp = {}
# Compiler
ctx.language = cell("text").set(node["language"])
main_module_state = node.get("stored_state_main_module", None)
if main_module_state is None:
main_module_state = node.get("cached_state_main_module", None)
ctx.main_module = build_structured_cell(
ctx,
"main_module",
False,
True,
False,
main_module_inchannels,
[()],
main_module_state,
lib_path00,
)
if "_main_module" in temp and len(temp["_main_module"]):
temp_main_module = temp["_main_module"]
main_module_handle = ctx.main_module.handle
main_module_data = ctx.main_module.data.value
if main_module_data is None:
ctx.main_module.monitor.set_path((), {"objects": {}}, forced=True)
main_module_data = ctx.main_module.data.value
elif "objects" not in main_module_data:
main_module_handle["objects"] = {}
for objname, obj in temp_main_module.items():
for key, value in obj.items():
if objname in main_module_data["objects"] and \
key in main_module_data["objects"][objname]:
msg = "WARNING: %s main module object '%s': %s already defined"
print(msg % (node["path"], objname, key))
continue
if objname not in main_module_data["objects"]:
ctx.main_module.monitor.set_path(("objects", objname), {},
forced=True)
main_module_handle["objects"][objname][key] = value
elif main_module_state is None:
ctx.main_module.monitor.set_path((), {}, forced=True)
for ic in main_module_inchannels:
icpath = node["path"] + ("_main_module", ) + ic[1:]
namespace[icpath, True] = ctx.main_module.inchannels[ic], node
compiler_verbose = node["main_module"]["compiler_verbose"]
ctx.compiler_verbose = cell("json").set(compiler_verbose)
target = node["main_module"].get("target")
if target is not None:
ctx.main_module.monitor.set_path(("target", ), target, forced=True)
# Transformer itself
ctf = ctx.tf = context(name="tf", context=ctx)
debug = node["debug"]
_init_from_library(ctf, debug)
if lib_path00 is not None:
lib_path = lib_path00 + "." + name + ".code"
ctx.code = libcell(lib_path)
else:
ctx.code = cell("text")
ctx.code.set_file_extension(node["file_extension"])
if "code" in mount:
ctx.code.mount(**mount["code"])
plain_result = node["plain_result"]
result_state = node.get("cached_state_result", None)
result, result_ctx = build_structured_cell(ctx,
result_name,
True,
plain_result,
False, [()],
outchannels,
result_state,
lib_path0,
return_context=True)
if "result_schema" in mount:
result_ctx.schema.mount(**mount["result_schema"])
setattr(ctx, result_name, result)
assert not node["SCHEMA"]
ctx.pins = cell("json").set(all_pins)
ctx.inputpins = cell("json").set(inputpins)
c_inp = getattr(ctx, input_name + STRUC_ID)
c_result = getattr(ctx, result_name + STRUC_ID)
_finalize(ctx, ctf, inp, c_inp, result, c_result, input_name, result_name)
if "header" in mount:
ctx.header.mount(**mount["header"])
code = node.get("code")
if code is None:
code = node.get("cached_code")
if code is not None:
ctx.code.set(code)
if "code" in temp:
ctx.code.set(temp["code"])
inphandle = inp.handle
for k, v in temp.items():
if k in ("code", "_main_module"):
continue
setattr(inphandle, k, v)
namespace[node["path"] + ("code", ), True] = ctx.code, node
namespace[node["path"] + ("code", ), False] = ctx.code, node
if not is_lib: #clean up cached state and in_equilibrium, unless a library context
node.pop("cached_state_input", None)
if not in_equilibrium:
node.pop("cached_state_result", None)
node.pop("in_equilibrium", None)
namespace[node["path"], True] = inp, node
namespace[node["path"], False] = result, node
node.pop("TEMP", None)
def map_dict_chunk(ctx, chunksize, graph, inp, keyorder, has_uniform, elision,
lib_module_dict):
#print("map_dict_chunk", inp)
from seamless.core import Cell as CoreCell
from seamless.core import cell, context, macro, path, transformer
from seamless.core.structured_cell import StructuredCell
from seamless.core.HighLevelContext import HighLevelContext
from seamless.core.unbound_context import UnboundContext
import math
pseudo_connections = []
ctx.sc_data = cell("mixed")
ctx.sc_buffer = cell("mixed")
inpkeys = keyorder
nchunks = math.ceil(len(inpkeys) / chunksize)
ctx.sc = StructuredCell(
data=ctx.sc_data,
buffer=ctx.sc_buffer,
inchannels=[(n + 1, ) for n in range(nchunks)],
outchannels=[()],
)
if has_uniform:
ctx.uniform = cell("mixed")
first = True
for n in range(nchunks):
pos = chunksize * n
hc = HighLevelContext(graph)
subctx = "subctx_" + str(n + 1)
setattr(ctx, subctx, hc)
if first:
if not hasattr(hc, "inp"):
raise TypeError(
"map_dict_chunk context must have a cell called 'inp'")
hci = hc.inp
if has_uniform:
if first:
if not hasattr(hc, "uniform"):
raise TypeError(
"map_dict_chunk context must have a cell called 'uniform'"
)
if isinstance(hc.uniform, StructuredCell):
raise TypeError(
"map_dict_chunk context has a cell called 'uniform', but its celltype must be mixed, not structured"
)
if not isinstance(hc.uniform, CoreCell):
raise TypeError(
"map_dict_chunk context must have an attribute 'uniform' that is a cell, not a {}"
.format(type(hc.uniform)))
ctx.uniform.connect(hc.uniform)
con = ["..uniform"], ["ctx", subctx, "uniform"]
pseudo_connections.append(con)
if first:
if isinstance(hci, StructuredCell):
raise TypeError(
"map_dict_chunk context has a cell called 'inp', but its celltype must be mixed, not structured"
)
if not isinstance(hci, CoreCell):
raise TypeError(
"map_dict_chunk context must have an attribute 'inp' that is a cell, not a {}"
.format(type(hci)))
if hci.celltype != "mixed":
raise TypeError(
"map_dict_chunk context has a cell called 'inp', but its celltype must be mixed, not {}"
.format(hci.celltype))
con = ["..inp"], ["ctx", subctx, "inp"]
pseudo_connections.append(con)
inputchunk = {k: inp[k] for k in inpkeys[pos:pos + chunksize]}
#print("CHUNK", list(inputchunk.keys()))
chunk_ctx = context()
setattr(ctx, "chunk_%d" % (n + 1), chunk_ctx)
chunk_ctx.inputchunk_deep = cell("mixed", hash_pattern={"*": "#"})
chunk_ctx.inputchunk_deep.set(inputchunk)
chunk_ctx.inputchunk_deep2 = cell("plain")
chunk_ctx.inputchunk_deep.connect(chunk_ctx.inputchunk_deep2)
chunk_ctx.inputchunk_checksum = cell("checksum")
chunk_ctx.inputchunk_deep2.connect(chunk_ctx.inputchunk_checksum)
# chunk_ctx.inputchunk_checksum has the correct checksum, but there is no valid conversion
# (because it is unsafe).
# Use a macro to do it
chunk_ctx.get_inputchunk = macro(
{"inputchunk_checksum": {
"io": "input",
"celltype": "checksum"
}})
get_inputchunk = lib_module_dict["helper"]["get_inputchunk_dict"]
chunk_ctx.get_inputchunk.code.cell().set(get_inputchunk)
chunk_ctx.inputchunk_checksum.connect(
chunk_ctx.get_inputchunk.inputchunk_checksum)
p = path(chunk_ctx.get_inputchunk.ctx).inputchunk
chunk_ctx.inputchunk = cell("mixed", hash_pattern={"*": "#"})
p.connect(chunk_ctx.inputchunk)
if first:
if not hasattr(hc, "result"):
raise TypeError(
"map_dict_chunk context must have a cell called 'result'")
if isinstance(hc.result, StructuredCell):
raise TypeError(
"map_dict_chunk context has a cell called 'result', but its celltype must be mixed, not structured"
)
if not isinstance(hc.result, CoreCell):
raise TypeError(
"map_dict_chunk context must have an attribute 'result' that is a cell, not a {}"
.format(type(hc.result)))
chunk_ctx.inputchunk.connect(hci)
chunk_ctx.result = cell("mixed", hash_pattern={"*": "#"})
chunk_ctx.result_deep = cell("checksum")
hc.result.connect(chunk_ctx.result)
chunk_ctx.result.connect(chunk_ctx.result_deep)
chunk_ctx.result_deep.connect(ctx.sc.inchannels[(n + 1, )])
con = ["ctx", subctx, "result"], ["..result"]
pseudo_connections.append(con)
first = False
ctx.subresults = cell("plain")
ctx.sc.outchannels[()].connect(ctx.subresults)
merge_subresults = lib_module_dict["helper"]["merge_subresults_chunk"]
ctx.merge_subresults = transformer({
"subresults": {
"io": "input",
"celltype": "plain"
},
"result": {
"io": "output",
"celltype": "plain"
}
})
ctx.merge_subresults.code.cell().set(merge_subresults)
ctx.subresults.connect(ctx.merge_subresults.subresults)
ctx.result_deep = cell("plain")
ctx.merge_subresults.result.connect(ctx.result_deep)
# ctx.result_deep has the correct checksum, but there is no valid conversion
# (because it is unsafe).
# Use a macro to do it
ctx.get_result = macro(
{"result_checksum": {
"io": "input",
"celltype": "checksum"
}})
get_result = lib_module_dict["helper"]["get_result_dict"]
ctx.get_result.code.cell().set(get_result)
ctx.result_deep.connect(ctx.get_result.result_checksum)
p = path(ctx.get_result.ctx).result
ctx.result = cell("mixed", hash_pattern={"*": "#"})
p.connect(ctx.result)
if not elision:
ctx._pseudo_connections = pseudo_connections
def translate_compiled_transformer(node, root, namespace, inchannels,
outchannels, *, has_meta_connection):
from .translate import set_structured_cell_from_checksum
from ..highlevel.Environment import Environment
env0 = Environment(None)
env0._load(node.get("environment"))
env = env0._to_lowlevel()
inchannels = [ic for ic in inchannels if ic[0] != "code"]
main_module_inchannels = [
("objects", ) + ic[1:] for ic in inchannels if ic[0] == "_main_module"
] + [("link_options", )]
inchannels = [ic for ic in inchannels if ic[0] != "_main_module"]
parent = get_path(root, node["path"][:-1], None, None)
name = node["path"][-1]
ctx = context(toplevel=False)
setattr(parent, name, ctx)
result_name = node["RESULT"]
input_name = node["INPUT"]
inputpins = []
all_inchannels = set(inchannels)
pin_cells = {}
for pin in list(node["pins"].keys()):
pin_cell_name = pin + "_PIN"
assert pin_cell_name not in all_inchannels
assert pin_cell_name not in node["pins"]
pin_cell = cell("mixed")
cell_setattr(node, ctx, pin_cell_name, pin_cell)
pin_cells[pin] = pin_cell
mount = node.get("mount", {})
inp, inp_ctx = build_structured_cell(
ctx,
input_name,
inchannels, [()],
fingertip_no_remote=node.get("fingertip_no_remote", False),
fingertip_no_recompute=node.get("fingertip_no_recompute", False),
hash_pattern=node.get("hash_pattern"),
return_context=True)
setattr(ctx, input_name, inp)
namespace[node["path"] + ("SCHEMA", ), "source"] = inp.schema, node
if "input_schema" in mount:
inp_ctx.schema.mount(**mount["input_schema"])
for inchannel in inchannels:
path = node["path"] + inchannel
namespace[path, "target"] = inp.inchannels[inchannel], node
assert result_name not in node[
"pins"] #should have been checked by highlevel
assert "executor_result_" not in node[
"pins"] #should have been checked by highlevel
all_pins = {}
inputpins = []
for pinname, pin in node["pins"].items():
p = {"io": "input"}
p.update(pin)
all_pins[pinname] = p
inputpins.append(pinname)
all_pins[result_name] = {"io": "output"}
if node["SCHEMA"]:
all_pins[node["SCHEMA"]] = {"io": "input", "celltype": "mixed"}
# Compiler
ctx.language = cell("str").set(node["language"])
ctx.main_module = build_structured_cell(
ctx,
"main_module",
main_module_inchannels,
[()],
fingertip_no_remote=node.get("fingertip_no_remote", False),
fingertip_no_recompute=node.get("fingertip_no_recompute", False),
)
for ic in main_module_inchannels:
if ic == "link_options":
continue
icpath = node["path"] + ("_main_module", ) + ic[1:]
namespace[icpath, "target"] = ctx.main_module.inchannels[ic], node
# Transformer itself
ctf = ctx.tf = context()
debug = node["debug"] or node.get("compiled_debug")
if debug is None:
debug = False
_init_from_graph(ctf, debug)
ctx.code = cell("text")
ctx.code.set_file_extension(node["file_extension"])
if "code" in mount:
ctx.code.mount(**mount["code"])
checksum = node.get("checksum", {})
if "code" in checksum:
ctx.code._set_checksum(checksum["code"], initial=True)
main_module_checksum = checksum.get(
"main_module",
'd0a1b2af1705c1b8495b00145082ef7470384e62ac1c4d9b9cdbbe0476c28f8c' # {}
)
set_structured_cell_from_checksum(ctx.main_module,
{"auth": main_module_checksum})
inp_checksum = convert_checksum_dict(checksum, "input")
set_structured_cell_from_checksum(inp, inp_checksum)
namespace[node["path"] + ("code", ), "target"] = ctx.code, node
namespace[node["path"] + ("code", ), "source"] = ctx.code, node
if has_meta_connection:
ctx.meta = cell("plain")
ctx.meta.connect(ctf.executor.META)
namespace[node["path"] + ("meta", ), "target"] = ctx.meta, node
else:
meta = node.get("meta")
if meta is not None:
ctf.executor.meta = meta
result, result_ctx = build_structured_cell(
ctx,
result_name, [()],
outchannels,
fingertip_no_remote=node.get("fingertip_no_remote", False),
fingertip_no_recompute=node.get("fingertip_no_recompute", False),
return_context=True)
namespace[node["path"] + ("RESULTSCHEMA", ),
"source"] = result.schema, node
if "result_schema" in mount:
result_ctx.schema.mount(**mount["result_schema"])
setattr(ctx, result_name, result)
assert not node["SCHEMA"]
result_checksum = {}
for k in checksum:
if not k.startswith("result"):
continue
k2 = "value" if k == "result" else k[len("result_"):]
result_checksum[k2] = checksum[k]
set_structured_cell_from_checksum(result, result_checksum)
ctx.pins = cell("plain").set(all_pins)
ctx.inputpins = cell("plain").set(inputpins)
c_inp = getattr(ctx, input_name + STRUC_ID)
c_result = getattr(ctx, result_name + STRUC_ID)
_finalize(ctx, ctf, inp, c_inp, result, c_result, input_name, result_name,
inchannels, node)
if "header" in mount:
ctx.header.mount(**mount["header"])
namespace[node["path"] + ("header", ), "source"] = ctx.header, node
if env is not None:
ctf.executor.env = env
namespace[node["path"], "target"] = inp, node
namespace[node["path"], "source"] = result, node
import seamless
from seamless.core import macro_mode_on
from seamless.core import context, cell, transformer, macro, libcell, StructuredCell
from seamless.core.structured_cell import BufferWrapper, StructuredCellState
from seamless.core import library
with macro_mode_on():
ctx = context(toplevel=True)
def create(ctx, with_buffer, with_schema, inchannels):
with macro_mode_on():
ctx.struc = context(name="struc", context=ctx)
ctx.struc.storage = cell("text")
ctx.struc.form = cell("json")
ctx.struc.data = cell(
"mixed",
form_cell=ctx.struc.form,
storage_cell=ctx.struc.storage,
)
schema = None
if with_schema:
ctx.struc.schema = cell("json")
schema = ctx.struc.schema
bufferwrapper = None
if with_buffer:
ctx.struc.buffer_storage = cell("text")
ctx.struc.buffer_form = cell("json")
ctx.struc.buffer_data = cell(
"mixed",
form_cell=ctx.struc.buffer_form,
import seamless
from seamless.core import macro_mode_on
from seamless.core import context, cell, transformer, StructuredCell
import numpy as np
with macro_mode_on():
ctx = context(toplevel=True)
ctx.mount("/tmp/mount-test",
persistent=None) #directory remains, but empty
ctx.inp_struc = context(name="inp_struc", context=ctx)
ctx.inp_struc.storage = cell("text")
ctx.inp_struc.form = cell("json")
ctx.inp_struc.data = cell(
"mixed",
form_cell=ctx.inp_struc.form,
storage_cell=ctx.inp_struc.storage,
)
ctx.inp = StructuredCell("inp",
ctx.inp_struc.data,
storage=ctx.inp_struc.storage,
form=ctx.inp_struc.form,
schema=None,
buffer=None,
inchannels=None,
outchannels=[()])
ctx.tf = transformer({
"inp": ("input", "copy", "silk"),
"c": "output",
})
ctx.inp.connect_outchannel((), ctx.tf.inp)
def translate_macro(node, root, namespace, inchannels, outchannels):
from .translate import set_structured_cell_from_checksum
parent = get_path(root, node["path"][:-1], None, None)
name = node["path"][-1]
ctx = context(toplevel=False)
setattr(parent, name, ctx)
param_name = node["PARAM"]
all_inchannels = set(inchannels)
param_inchannels = []
interchannels = []
pin_cells = {}
pin_mpaths0 = {}
for pinname in list(node["pins"].keys()):
pin = node["pins"][pinname]
if pin["io"] == "parameter":
pin_cell_name = pinname + "_PARAM"
assert pin_cell_name not in node["pins"]
assert pin_cell_name not in all_inchannels
pinname2 = as_tuple(pinname)
interchannels.append(pinname2)
if pinname2 in inchannels:
param_inchannels.append(pinname2)
elif pin["io"] in ("input", "output", "edit"):
pin_cell_name = pinname
else:
raise ValueError((pin["io"], pinname))
pin_hash_pattern = pin.get("hash_pattern")
celltype = pin.get("celltype", "mixed")
if celltype == "mixed":
pin_cell = cell(celltype, hash_pattern=pin_hash_pattern)
else:
pin_cell = cell(celltype)
cell_setattr(node, ctx, pin_cell_name, pin_cell)
pin_cells[pinname] = pin_cell
if pin["io"] != "parameter":
pin_mpaths0[pinname] = (pin["io"] in ("input", "edit"))
mount = node.get("mount", {})
param = None
if len(interchannels):
param, param_ctx = build_structured_cell(ctx,
param_name,
param_inchannels,
interchannels,
return_context=True,
fingertip_no_remote=False,
fingertip_no_recompute=False,
hash_pattern={"*": "#"})
setattr(ctx, param_name, param)
namespace[node["path"] + ("SCHEMA", ), "source"] = param.schema, node
if "param_schema" in mount:
param_ctx.schema.mount(**mount["param_schema"])
param_pins = {}
for pinname, pin in node["pins"].items():
if pin["io"] != "parameter":
continue
p = {"io": "input"}
p.update(pin)
param_pins[pinname] = p
ctx.macro = macro(param_pins)
if node.get("elision"):
ctx.macro.allow_elision = True
elision = {"macro": ctx.macro, "input_cells": {}, "output_cells": {}}
for pinname in pin_mpaths0:
is_input = pin_mpaths0[pinname]
pin_mpath = getattr(core_path(ctx.macro.ctx), pinname)
pin_cell = pin_cells[pinname]
if is_input:
if node["pins"][pinname]["io"] == "edit":
pin_cell.bilink(pin_mpath)
else:
pin_cell.connect(pin_mpath)
elision["input_cells"][pin_cell] = pin_mpath
else:
pin_mpath.connect(pin_cell)
elision["output_cells"][pin_cell] = pin_mpath
ctx._get_manager().set_elision(**elision)
ctx.code = cell("macro")
if "code" in mount:
ctx.code.mount(**mount["code"])
ctx.code.connect(ctx.macro.code)
checksum = node.get("checksum", {})
if "code" in checksum:
ctx.code._set_checksum(checksum["code"], initial=True)
if param is not None:
param_checksum = convert_checksum_dict(checksum, "param")
set_structured_cell_from_checksum(param, param_checksum)
namespace[node["path"] + ("code", ), "target"] = ctx.code, node
namespace[node["path"] + ("code", ), "source"] = ctx.code, node
for pinname in node["pins"]:
path = node["path"] + as_tuple(pinname)
pin = node["pins"][pinname]
if pin["io"] == "parameter":
pinname2 = as_tuple(pinname)
if pinname2 in inchannels:
namespace[path, "target"] = param.inchannels[pinname], node
target = getattr(ctx.macro, pinname)
assert target is not None, pinname
pin_cell = pin_cells[pinname]
param.outchannels[pinname2].connect(pin_cell)
pin_cell.connect(target)
elif pin["io"] == "edit":
namespace[path, "edit"] = pin_cells[pinname], node
else:
cmode = "target" if pin["io"] == "input" else "source"
namespace[path, cmode] = pin_cells[pinname], node
def load(ctx):
ctx.readme = libcell(".readme")
def recreate(ctx, name0, with_buffer, with_schema, inchannels):
name = "." + name0
ctx.struc = context(name="struc", context=ctx)
ctx.struc.storage = libcell(name + ".storage")
ctx.struc.form = libcell(name + ".form")
ctx.struc.data = libmixedcell(
name + ".data",
form_cell=ctx.struc.form,
storage_cell=ctx.struc.storage,
)
schema = None
if with_schema:
ctx.struc.schema = libcell(name + ".schema")
schema = ctx.struc.schema
bufferwrapper = None
if with_buffer:
ctx.struc.buffer_storage = libcell(name + ".buffer_storage")
ctx.struc.buffer_form = libcell(name + ".buffer_form")
ctx.struc.buffer_data = libmixedcell(
name + ".buffer_data",
form_cell=ctx.struc.buffer_form,
storage_cell=ctx.struc.buffer_storage,
)
bufferwrapper = BufferWrapper(ctx.struc.buffer_data,
ctx.struc.buffer_storage,
ctx.struc.buffer_form)
ctx.hub = StructuredCell(
"hub",
ctx.struc.data,
storage=ctx.struc.storage,
form=ctx.struc.form,
schema=schema,
buffer=bufferwrapper,
inchannels=inchannels,
outchannels=[],
)
ctx.auth_json = context(name="auth_json", context=ctx)
recreate(ctx.auth_json,
"auth_json.struc",
with_buffer=False,
with_schema=False,
inchannels=[])
ctx.auth = context(name="auth", context=ctx)
recreate(ctx.auth,
"auth.struc",
with_buffer=False,
with_schema=True,
inchannels=[])
ctx.err = context(name="err", context=ctx)
recreate(ctx.err,
"err.struc",
with_buffer=True,
with_schema=True,
inchannels=[])
ctx.nauth = context(name="nauth", context=ctx)
try:
recreate(ctx.nauth,
"nauth.struc",
with_buffer=False,
with_schema=False,
inchannels=[("z", )])
except:
import traceback
traceback.print_exc()
assert which in ("pi", "e")
ctx.readme = libcell(".readme")
ctx.loader = macro({})
if which == "pi":
ctx.load = libcell(".pi.load")
else:
ctx.load = libcell(".e.load")
ctx.load.connect(ctx.loader.code)
compute = ctx.loader.gen_context
ctx.iterations = cell()
ctx.iterations.connect(compute.iterations)
ctx.result = cell()
compute.result.connect(ctx.result)
lctx = context(toplevel=True)
lctx.readme = cell("text").set("Compute pi or e iteratively")
lctx.pi = context(context=lctx, name="pi")
lctx.pi.code = cell("python").set(compute_pi)
lctx.pi.load = cell("macro").set(load_pi)
lctx.e = context(context=lctx, name="e")
lctx.e.code = cell("python").set(compute_e)
lctx.e.load = cell("macro").set(load_e)
lctx.select = cell("macro").set(select)
lctx.equilibrate()
lib = library.build(lctx)
library.register("compute", lib)
select_params = {
"which": ("ref", "text"),
}
def translate_bash_transformer(node, root, namespace, inchannels, outchannels, lib_path00, is_lib):
#TODO: simple translation, without a structured cell
#TODO: there is a lot of common code with py transformer
# Just to register the "bash_transformer" lib
from seamless.lib.bash_transformer import bash_transformer as _
inchannels = [ic for ic in inchannels if ic[0] != "code"]
parent = get_path(root, node["path"][:-1], None, None)
name = node["path"][-1]
lib_path0 = lib_path00 + "." + name if lib_path00 is not None else None
ctx = context(context=parent, name=name)
setattr(parent, name, ctx)
result_name = node["RESULT"]
input_name = node["INPUT"]
if len(inchannels):
lib_path0 = None #partial authority or no authority; no library update in either case
for c in inchannels:
assert (not len(c)) or c[0] != result_name #should have been checked by highlevel
with_result = node["with_result"]
buffered = node["buffered"]
pins = node["pins"].copy()
for extrapin in ("bashcode", "pins"):
assert extrapin not in node["pins"], extrapin
pins[extrapin] = {
"transfer_mode": "ref",
"access_mode": "default",
"content_type": None,
}
ctx.pins = core_cell("json").set(list(pins.keys()))
interchannels = [as_tuple(pin) for pin in pins]
plain = node["plain"]
input_state = node.get("stored_state_input", None)
mount = node.get("mount", {})
if input_state is None:
input_state = node.get("cached_state_input", None)
inp, inp_ctx = build_structured_cell(
ctx, input_name, True, plain, buffered, inchannels, interchannels,
input_state, lib_path0,
return_context=True
)
setattr(ctx, input_name, inp)
if "input_schema" in mount:
inp_ctx.schema.mount(**mount["input_schema"])
for inchannel in inchannels:
path = node["path"] + inchannel
namespace[path, True] = inp.inchannels[inchannel], node
assert result_name not in pins #should have been checked by highlevel
all_pins = {}
for pinname, pin in pins.items():
p = {"io": "input"}
p.update(pin)
all_pins[pinname] = p
all_pins[result_name] = {"io": "output", "transfer_mode": "copy"}
if node["SCHEMA"]:
assert with_result
all_pins[node["SCHEMA"]] = {
"io": "input", "transfer_mode": "json",
"access_mode": "json", "content_type": "json"
}
in_equilibrium = node.get("in_equilibrium", False)
ctx.tf = transformer(all_pins, in_equilibrium=in_equilibrium)
if node["debug"]:
ctx.tf.debug = True
if lib_path00 is not None:
lib_path = lib_path00 + "." + name + ".code"
ctx.code = libcell(lib_path)
else:
ctx.code = core_cell("json")
if "code" in mount:
ctx.code.mount(**mount["code"])
ctx.code._sovereign = True
ctx.pins.connect(ctx.tf.pins)
ctx.code.connect(ctx.tf.bashcode)
code = node.get("code")
if code is None:
code = node.get("cached_code")
ctx.code.set(code)
temp = node.get("TEMP")
if temp is None:
temp = {}
if "code" in temp:
ctx.code.set(temp["code"])
with library.bind("bash_transformer"):
ctx.executor_code = libcell(".executor_code")
ctx.executor_code.connect(ctx.tf.code)
inphandle = inp.handle
for k,v in temp.items():
if k == "code":
continue
setattr(inphandle, k, v)
namespace[node["path"] + ("code",), True] = ctx.code, node
namespace[node["path"] + ("code",), False] = ctx.code, node
for pin in list(node["pins"].keys()):
target = getattr(ctx.tf, pin)
inp.connect_outchannel( (pin,) , target )
if with_result:
plain_result = node["plain_result"]
result_state = node.get("cached_state_result", None)
result, result_ctx = build_structured_cell(
ctx, result_name, True, plain_result, False, [()],
outchannels, result_state, lib_path0,
return_context=True
)
if "result_schema" in mount:
result_ctx.schema.mount(**mount["result_schema"])
setattr(ctx, result_name, result)
result_pin = getattr(ctx.tf, result_name)
result.connect_inchannel(result_pin, ())
if node["SCHEMA"]:
schema_pin = getattr(ctx.tf, node["SCHEMA"])
result.schema.connect(schema_pin)
else:
for c in outchannels:
assert len(c) == 0 #should have been checked by highlevel
result = getattr(ctx.tf, result_name)
namespace[node["path"] + (result_name,), False] = result, node
if not is_lib: #clean up cached state and in_equilibrium, unless a library context
node.pop("cached_state_input", None)
if not in_equilibrium:
node.pop("cached_state_result", None)
node.pop("in_equilibrium", None)
namespace[node["path"], True] = inp, node
namespace[node["path"], False] = result, node
node.pop("TEMP", None)
def build_structured_cell(ctx,
name,
silk,
plain,
buffered,
inchannels,
outchannels,
state,
lib_path0,
*,
editchannels=[],
mount=None,
return_context=False):
#print("build_structured_cell", name, lib_path)
name2 = name + STRUC_ID
c = context(name=name2, context=ctx)
setattr(ctx, name2, c)
if mount is not None:
c.mount(**mount)
lib_path = lib_path0 + "." + name2 if lib_path0 is not None else None
sovereign = True
if lib_path:
path = lib_path + ".form"
cc = libcell(path)
else:
cc = core_cell("json")
cc._sovereign = sovereign
c.form = cc
if plain:
if lib_path:
path = lib_path + ".data"
cc = libcell(path)
else:
cc = core_cell("json")
cc._sovereign = sovereign
c.data = cc
storage = None
else:
if lib_path:
path = lib_path + ".storage"
storage = libcell(path)
else:
storage = core_cell("text")
storage._sovereign = sovereign
c.storage = storage
if lib_path:
path = lib_path + ".data"
c.data = libmixedcell(path,
form_cell=c.form,
storage_cell=c.storage)
else:
c.data = core_cell("mixed",
form_cell=c.form,
storage_cell=c.storage)
c.data._sovereign = sovereign
if silk:
if lib_path:
path = lib_path + ".schema"
schema = libcell(path)
else:
schema = core_cell("json")
c.schema = schema
else:
schema = None
if buffered:
if lib_path:
path = lib_path + ".buffer_form"
cc = libcell(path)
else:
cc = core_cell("json")
cc._sovereign = sovereign
c.buffer_form = cc
if plain:
if lib_path:
path = lib_path + ".buffer_data"
cc = libcell(path)
else:
cc = core_cell("json")
cc._sovereign = sovereign
c.buffer_data = cc
buffer_storage = None
else:
if lib_path:
path = lib_path + ".buffer_storage"
buffer_storage = libcell(path)
else:
buffer_storage = core_cell("text")
buffer_storage._sovereign = sovereign
c.buffer_storage = buffer_storage
if lib_path:
path = lib_path + ".buffer_data"
c.buffer_data = libmixedcell(
path,
form_cell=c.buffer_form,
storage_cell=c.buffer_storage,
)
else:
c.buffer_data = core_cell(
"mixed",
form_cell=c.buffer_form,
storage_cell=c.buffer_storage,
)
c.buffer_data._sovereign = sovereign
bufferwrapper = BufferWrapper(c.buffer_data, buffer_storage,
c.buffer_form)
else:
bufferwrapper = None
sc = StructuredCell(name,
c.data,
storage=storage,
form=c.form,
schema=schema,
buffer=bufferwrapper,
inchannels=inchannels,
outchannels=outchannels,
state=state,
editchannels=editchannels)
if return_context:
return sc, c
else:
return sc
import seamless
from seamless.core import macro_mode_on
from seamless.core import context, textcell, transformer, pytransformercell
with macro_mode_on():
ctx = context(toplevel=True)
ctx.cell1 = textcell().set(1)
ctx.cell2 = textcell().set(2)
ctx.result = textcell()
ctx.tf = transformer({"a": "input", "b": "input", "c": "output"})
ctx.cell1.connect(ctx.tf.a)
ctx.cell2.connect(ctx.tf.b)
ctx.code = pytransformercell().set("c = float(a) + float(b)")
ctx.code.connect(ctx.tf.code)
ctx.tf.c.connect(ctx.result)
ctx.result.mount("/tmp/mount-test/myresult", persistent=True)
ctx.mount("/tmp/mount-test")
ctx.sub = context(toplevel=False, context=ctx, name="sub")
ctx.sub.mycell = textcell().set("This is my cell\nend")
ctx.equilibrate()
print(ctx.result.value)
ctx.cell1.set(10)
ctx.equilibrate()
print(ctx.result.value)
print(ctx.result.value)
ctx.code.set("c = float(a) + float(b) + 1000")
ctx.equilibrate()
print(ctx.result.value)
print(ctx.status())
def select(ctx, which):
assert which in ("pi", "e")
ctx.readme = libcell(".readme")
ctx.loader = macro({})
if which == "pi":
ctx.load = libcell(".pi.load")
else:
ctx.load = libcell(".e.load")
ctx.load.connect(ctx.loader.code)
compute = ctx.loader.ctx
ctx.iterations = cell()
ctx.iterations.connect(compute.iterations)
ctx.result = cell()
compute.result.connect(ctx.result)
lctx = context(toplevel=True)
lctx.readme = cell("text").set("Compute pi or e iteratively")
lctx.pi = context()
lctx.pi.code = cell("python").set(compute_pi)
lctx.pi.load = cell("macro").set(load_pi)
lctx.e = context()
lctx.e.code = cell("python").set(compute_e)
lctx.e.load = cell("macro").set(load_e)
lctx.select = cell("macro").set(select)
lctx.equilibrate()
lib = library.build(lctx)
library.register("compute", lib)
select_params = {
"which": ("ref", "text"),
import seamless
from seamless.core import macro_mode_on
from seamless.core import context, cell, transformer, StructuredCell
from seamless.core.structured_cell import BufferWrapper
import numpy as np
with macro_mode_on():
ctx = context(toplevel=True)
ctx.hub_struc = context(name="hub_struc", context=ctx)
ctx.hub_struc.storage = cell("text")
ctx.hub_struc.form = cell("json")
ctx.hub_struc.data = cell(
"mixed",
form_cell=ctx.hub_struc.form,
storage_cell=ctx.hub_struc.storage,
)
ctx.hub_struc.schema = cell("json")
ctx.hub_struc.buffer_storage = cell("text")
ctx.hub_struc.buffer_form = cell("json")
ctx.hub_struc.buffer_data = cell(
"mixed",
form_cell=ctx.hub_struc.buffer_form,
storage_cell=ctx.hub_struc.buffer_storage,
)
bufferwrapper = BufferWrapper(ctx.hub_struc.buffer_data,
ctx.hub_struc.buffer_storage,
ctx.hub_struc.buffer_form)
ctx.hub = StructuredCell("hub",
ctx.hub_struc.data,
storage=ctx.hub_struc.storage,
form=ctx.hub_struc.form,
from math import pi
from seamless.core import context, cell
ctx0 = context(toplevel=True)
ctx0.cell = cell("mixed").set(pi)
ctx0.cell2 = cell("python").set("lambda a: 2 * a")
ctx0.cell.set({"a": pi})
import json
graph = json.load(open("twopi.seamless"))
from seamless.highlevel import load_graph
ctx = load_graph(graph, cache_ctx=ctx0)
ctx.translate(force=True)
print(ctx.pi.checksum)
print(ctx.pi.value)
print("equilibrate")
ctx.equilibrate()
print(ctx.twopi.value)
graph2 = json.load(open("twopi-result.seamless"))
ctx2 = load_graph(graph2, cache_ctx=ctx0)
print(ctx2.pi.checksum)
print(ctx2.pi.value)
print("equilibrate")
ctx2.equilibrate()
print(ctx2.twopi.value)
def translate_bash_transformer(node, root, namespace, inchannels, outchannels, lib_path00, is_lib):
#TODO: simple translation, without a structured cell
#TODO: there is a lot of common code with py transformer
assert not "code" in node ### node["code"] is an outdated attribute
# Just to register the "bash_transformer" lib
from seamless.core.macro_mode import get_macro_mode, curr_macro
from seamless.lib.bash_transformer import bash_transformer as _
inchannels = [ic for ic in inchannels if ic[0] != "code"]
parent = get_path(root, node["path"][:-1], None, None)
name = node["path"][-1]
lib_path0 = lib_path00 + "." + name if lib_path00 is not None else None
ctx = context(toplevel=False)
setattr(parent, name, ctx)
result_name = node["RESULT"]
input_name = node["INPUT"]
for c in inchannels:
assert (not len(c)) or c[0] != result_name #should have been checked by highlevel
with_result = node["with_result"]
buffered = node["buffered"]
pins = node["pins"].copy()
for extrapin in ("bashcode", "pins"):
assert extrapin not in node["pins"], extrapin
pins[extrapin] = {
"transfer_mode": "ref",
"access_mode": "plain",
"content_type": None,
}
ctx.pins = core_cell("plain").set(list(pins.keys()))
interchannels = [as_tuple(pin) for pin in pins]
plain = node["plain"]
mount = node.get("mount", {})
inp, inp_ctx = build_structured_cell(
ctx, input_name, True, plain, buffered, inchannels, interchannels,
lib_path0,
return_context=True
)
setattr(ctx, input_name, inp)
if "input_schema" in mount:
inp_ctx.schema.mount(**mount["input_schema"])
for inchannel in inchannels:
path = node["path"] + inchannel
namespace[path, True] = inp.inchannels[inchannel], node
assert result_name not in pins #should have been checked by highlevel
all_pins = {}
for pinname, pin in pins.items():
p = {"io": "input"}
p.update(pin)
all_pins[pinname] = p
all_pins[result_name] = {"io": "output", "transfer_mode": "copy"}
if node["SCHEMA"]:
assert with_result
all_pins[node["SCHEMA"]] = {
"io": "input", "transfer_mode": "json",
"access_mode": "json", "content_type": "json"
}
ctx.tf = transformer(all_pins)
if node["debug"]:
ctx.tf.debug = True
if lib_path00 is not None:
lib_path = lib_path00 + "." + name + ".code"
ctx.code = libcell(lib_path)
else:
ctx.code = core_cell("text")
if "code" in mount:
ctx.code.mount(**mount["code"])
ctx.code._sovereign = True
ctx.pins.connect(ctx.tf.pins)
ctx.code.connect(ctx.tf.bashcode)
checksum = node.get("checksum", {})
if "code" in checksum:
ctx.code.set_checksum(checksum["code"])
if "input" in checksum:
inp.set_checksum(checksum["input"])
with library.bind("bash_transformer"):
ctx.executor_code = libcell(".executor_code")
ctx.executor_code.connect(ctx.tf.code)
namespace[node["path"] + ("code",), True] = ctx.code, node
namespace[node["path"] + ("code",), False] = ctx.code, node
for pin in list(node["pins"].keys()):
target = getattr(ctx.tf, pin)
inp.outchannels[(pin,)].connect(target)
if with_result:
plain_result = node["plain_result"]
result, result_ctx = build_structured_cell(
ctx, result_name, True, plain_result, False, [()],
outchannels, lib_path0,
return_context=True
)
if "result_schema" in mount:
result_ctx.schema.mount(**mount["result_schema"])
setattr(ctx, result_name, result)
result_pin = getattr(ctx.tf, result_name)
result_pin.connect(result.inchannels[()])
if node["SCHEMA"]:
schema_pin = getattr(ctx.tf, node["SCHEMA"])
result.schema.connect(schema_pin)
if "result" in checksum:
result.set_checksum(checksum["result"])
if "schema" in checksum:
result.schema.set_checksum(checksum["schema"])
else:
for c in outchannels:
assert len(c) == 0 #should have been checked by highlevel
result = getattr(ctx.tf, result_name)
namespace[node["path"] + (result_name,), False] = result, node
namespace[node["path"], True] = inp, node
namespace[node["path"], False] = result, node
import seamless
from seamless.core import macro_mode_on
from seamless.core import context,textcell, cell, transformer, pytransformercell
with macro_mode_on():
ctx = context(toplevel=True)
ctx.cell1 = cell().set(1)
ctx.cell2 = cell().set(2)
ctx.result = cell()
ctx.tf = transformer({
"a": "input",
"b": "input",
"c": "output"
})
ctx.cell1.connect(ctx.tf.a)
ctx.cell2.connect(ctx.tf.b)
ctx.code = pytransformercell().set("c = a + b")
ctx.code.connect(ctx.tf.code)
ctx.tf.c.connect(ctx.result)
ctx.result.mount("/tmp/mount-test/myresult", persistent=True)
ctx.mount("/tmp/mount-test")
ctx.sub = context(toplevel=False)
ctx.sub.mycell = textcell().set("This is my cell\nend")
ctx.equilibrate()
print(ctx.result.value)
ctx.cell1.set(10)
ctx.equilibrate()
print(ctx.result.value)
print(ctx.result.value)
ctx.code.set("c = float(a) + float(b) + 1000")
def translate_bash_transformer(node, root, namespace, inchannels, outchannels,
*, has_meta_connection):
from .translate import set_structured_cell_from_checksum
from ..highlevel.Environment import Environment
from ..core.environment import (validate_capabilities,
validate_conda_environment,
validate_docker)
env0 = Environment(None)
env0._load(node.get("environment"))
env = env0._to_lowlevel()
is_docker_transformer = False
if env is not None and env.get("docker") is not None:
ok1 = validate_capabilities(env)[0]
ok2 = validate_conda_environment(env)[0]
ok3 = validate_docker(env)[0]
if not (ok1 or ok2 or ok3):
is_docker_transformer = True
if is_docker_transformer:
from .translate_bashdocker_transformer import translate_bashdocker_transformer
docker = env.pop("docker")
docker_image = docker["name"]
docker_options = docker["options"]
# TODO: pass on version and checksum as well?
if "powers" not in env:
env["powers"] = []
env["powers"].append("docker")
return translate_bashdocker_transformer(
node,
root,
namespace,
inchannels,
outchannels,
has_meta_connection=has_meta_connection,
env=env,
docker_image=docker_image,
docker_options=docker_options)
inchannels = [ic for ic in inchannels if ic[0] != "code"]
parent = get_path(root, node["path"][:-1], None, None)
name = node["path"][-1]
ctx = context(toplevel=False)
setattr(parent, name, ctx)
result_name = node["RESULT"]
input_name = node["INPUT"]
result_cell_name = result_name + "_CELL"
forbidden = [result_name, result_cell_name, "bashcode", "pins_"]
pin_intermediate = {}
for pin in node["pins"].keys():
pin_intermediate[pin] = input_name + "_PIN_" + pin
forbidden.append(pin_intermediate[pin])
for c in inchannels:
assert (
not len(c)
) or c[0] not in forbidden #should have been checked by highlevel
pins = node["pins"].copy()
pins["bashcode"] = {"celltype": "text"}
pins["pins_"] = {"celltype": "plain"}
ctx.pins = cell("plain").set(list(pins.keys()))
interchannels = [as_tuple(pin) for pin in pins]
mount = node.get("mount", {})
inp, inp_ctx = build_structured_cell(
ctx,
input_name,
inchannels,
interchannels,
fingertip_no_remote=node.get("fingertip_no_remote", False),
fingertip_no_recompute=node.get("fingertip_no_recompute", False),
hash_pattern=node.get("hash_pattern"),
return_context=True)
setattr(ctx, input_name, inp)
namespace[node["path"] + ("SCHEMA", ), "source"] = inp.schema, node
if "input_schema" in mount:
inp_ctx.schema.mount(**mount["input_schema"])
for inchannel in inchannels:
path = node["path"] + inchannel
namespace[path, "target"] = inp.inchannels[inchannel], node
assert result_name not in pins #should have been checked by highlevel
all_pins = {}
for pinname, pin in pins.items():
p = {"io": "input"}
p.update(pin)
all_pins[pinname] = p
all_pins[result_name] = {"io": "output"}
if node["SCHEMA"]:
raise NotImplementedError
all_pins[node["SCHEMA"]] = {
"io": "input",
"transfer_mode": "json",
"access_mode": "json",
"content_type": "json"
}
ctx.tf = transformer(all_pins)
if node["debug"]:
ctx.tf.debug = True
ctx.code = cell("text")
if "code" in mount:
ctx.code.mount(**mount["code"])
ctx.pins.connect(ctx.tf.pins_)
ctx.code.connect(ctx.tf.bashcode)
checksum = node.get("checksum", {})
if "code" in checksum:
ctx.code._set_checksum(checksum["code"], initial=True)
inp_checksum = convert_checksum_dict(checksum, "input")
set_structured_cell_from_checksum(inp, inp_checksum)
ctx.executor_code = sctx.executor_code.cell()
ctx.executor_code.connect(ctx.tf.code)
namespace[node["path"] + ("code", ), "target"] = ctx.code, node
namespace[node["path"] + ("code", ), "source"] = ctx.code, node
for pinname, pin in node["pins"].items():
target = ctx.tf.get_pin(pinname)
celltype = pin.get("celltype", "mixed")
if celltype == "code":
celltype = "text"
intermediate_cell = cell(celltype)
cell_setattr(node, ctx, pin_intermediate[pinname], intermediate_cell)
inp.outchannels[(pinname, )].connect(intermediate_cell)
intermediate_cell.connect(target)
meta = deepcopy(node.get("meta", {}))
meta["transformer_type"] = "bash"
ctx.tf.meta = meta
if has_meta_connection:
ctx.meta = cell("plain")
ctx.meta.connect(ctx.tf.META)
namespace[node["path"] + ("meta", ), "target"] = ctx.meta, node
result, result_ctx = build_structured_cell(
ctx,
result_name, [()],
outchannels,
fingertip_no_remote=node.get("fingertip_no_remote", False),
fingertip_no_recompute=node.get("fingertip_no_recompute", False),
return_context=True)
namespace[node["path"] + ("RESULTSCHEMA", ),
"source"] = result.schema, node
if "result_schema" in mount:
result_ctx.schema.mount(**mount["result_schema"])
setattr(ctx, result_name, result)
result_pin = ctx.tf.get_pin(result_name)
result_cell = cell("mixed")
cell_setattr(node, ctx, result_cell_name, result_cell)
result_pin.connect(result_cell)
result_cell.connect(result.inchannels[()])
if node["SCHEMA"]:
schema_pin = ctx.tf.get_pin(node["SCHEMA"])
result.schema.connect(schema_pin)
result_checksum = {}
for k in checksum:
if not k.startswith("result"):
continue
k2 = "value" if k == "result" else k[len("result_"):]
result_checksum[k2] = checksum[k]
set_structured_cell_from_checksum(result, result_checksum)
if env is not None:
ctx.tf.env = env
namespace[node["path"], "target"] = inp, node
namespace[node["path"], "source"] = result, node
def build_structured_cell(
ctx, name, silk, plain, buffered,
inchannels, outchannels, lib_path0,
*, editchannels=[], mount=None, return_context=False
):
#print("build_structured_cell", name, lib_path)
name2 = name + STRUC_ID
c = context(toplevel=False)
setattr(ctx, name2, c)
if mount is not None:
c.mount(**mount)
lib_path = lib_path0 + "." + name2 if lib_path0 is not None else None
sovereign = True
if plain:
if lib_path:
path = lib_path + ".data"
cc = libcell(path)
else:
cc = core_cell("mixed")
cc._sovereign = sovereign
c.data = cc
storage = None
else:
if lib_path:
path = lib_path + ".data"
c.data = libcell(path, "mixed")
else:
c.data = core_cell("mixed")
c.data._sovereign = sovereign
if silk:
if lib_path:
path = lib_path + ".schema"
schema = libcell(path)
else:
schema = core_cell("plain")
c.schema = schema
else:
schema = None
if buffered:
if plain:
if lib_path:
path = lib_path + ".buffer"
cc = libcell(path)
else:
cc = core_cell("mixed")
cc._sovereign = sovereign
c.buffer = cc
storage = None
else:
if lib_path:
path = lib_path + ".buffer"
c.buffer = libcell(path, "mixed")
else:
c.buffer = core_cell("mixed")
c.buffer._sovereign = sovereign
buffer = c.buffer
else:
buffer = None
sc = StructuredCell(
name,
c.data,
schema=schema,
buffer=buffer,
plain=plain,
inchannels=inchannels,
outchannels=outchannels,
editchannels=editchannels
)
if return_context:
return sc, c
else:
return sc
raise NotImplementedError # Silk access is not working; the modified code below kludges around it
import seamless
from seamless.core import macro_mode_on
from seamless.core import context, cell, transformer, StructuredCell
import numpy as np
with macro_mode_on():
ctx = context(toplevel=True)
ctx.mount("/tmp/mount-test", persistent=None) #directory remains, but empty
ctx.inp_struc = context(toplevel=False)
ctx.inp_struc.data = cell("mixed")
ctx.inp = StructuredCell(
"inp",
ctx.inp_struc.data,
plain = False,
schema = None,
buffer = None,
inchannels = None,
outchannels = [()]
)
ctx.tf = transformer({
###"inp": ("input", "copy", "silk"),
"inp": ("input", "copy", "mixed"),
"c": "output",
})
ctx.inp.outchannels[()].connect(ctx.tf.inp)
###ctx.tf.code.cell().set("c = inp.a * inp.dat + inp.b")
ctx.tf.code.cell().set("c = inp['a'] * inp['dat'] + inp['b']")
def create(ctx, mount, state=None):
with macro_mode_on():
ctx.hub_struc = context(name="hub_struc", context=ctx)
ctx.hub_struc.storage = cell("text")
ctx.hub_struc.form = cell("json")
ctx.hub_struc.data = cell(
"mixed",
form_cell=ctx.hub_struc.form,
storage_cell=ctx.hub_struc.storage,
)
ctx.hub_struc.schema = cell("json")
ctx.hub_struc.buffer_storage = cell("text")
ctx.hub_struc.buffer_form = cell("json")
ctx.hub_struc.buffer_data = cell(
"mixed",
form_cell=ctx.hub_struc.buffer_form,
storage_cell=ctx.hub_struc.buffer_storage,
)
bufferwrapper = BufferWrapper(ctx.hub_struc.buffer_data,
ctx.hub_struc.buffer_storage,
ctx.hub_struc.buffer_form)
ctx.hub = StructuredCell(
"hub",
ctx.hub_struc.data,
storage=ctx.hub_struc.storage,
form=ctx.hub_struc.form,
schema=ctx.hub_struc.schema,
buffer=bufferwrapper,
inchannels=[("a", "factor1"), ("a", "factor2"), ("b", )],
outchannels=[("a", ), ("b", ), ("c", )],
state=state,
)
ctx.code = cell("transformer").set(
"d = a.factor1 * b + a.factor2 * c + a.constant")
ctx.mixer = transformer({
"a": ("input", "ref", "silk"),
"b": ("input", "ref", "object"),
"c": ("input", "ref", "object"),
"d": ("output", "ref", "json"),
})
ctx.a_factor1 = cell("json")
ctx.a_factor2 = cell("json")
ctx.b = cell("json")
ctx.code.connect(ctx.mixer.code)
ctx.hub.connect_inchannel(ctx.a_factor1, ("a", "factor1"))
ctx.hub.connect_inchannel(ctx.a_factor2, ("a", "factor2"))
ctx.hub.connect_inchannel(ctx.b, ("b", ))
ctx.hub.connect_outchannel(("a", ), ctx.mixer.a)
ctx.hub.connect_outchannel(("b", ), ctx.mixer.b)
ctx.hub.connect_outchannel(("c", ), ctx.mixer.c)
ctx.result_struc = context(name="result_struc", context=ctx)
ctx.result_struc.storage = cell("text")
ctx.result_struc.form = cell("json")
ctx.result_struc.data = cell(
"mixed",
form_cell=ctx.result_struc.form,
storage_cell=ctx.result_struc.storage,
)
ctx.result_struc.schema = cell("json")
ctx.result_struc.buffer_storage = cell("text")
ctx.result_struc.buffer_form = cell("json")
ctx.result_struc.buffer_data = cell(
"mixed",
form_cell=ctx.result_struc.buffer_form,
storage_cell=ctx.result_struc.buffer_storage,
)
bufferwrapper = BufferWrapper(ctx.result_struc.buffer_data,
ctx.result_struc.buffer_storage,
ctx.result_struc.buffer_form)
ctx.result = StructuredCell("result",
ctx.result_struc.data,
storage=ctx.result_struc.storage,
form=ctx.result_struc.form,
schema=ctx.result_struc.schema,
buffer=bufferwrapper,
inchannels=[()],
outchannels=[])
ctx.result.connect_inchannel(ctx.mixer.d, ())
if mount:
ctx.mount("/tmp/mount-test")
import seamless
from seamless.core import macro_mode_on
from seamless.core import context, cell, transformer
with macro_mode_on():
topctx = context(toplevel=True)
ctx = topctx.sub = context(toplevel=False)
assert topctx.sub is ctx
ctx.cell1 = cell().set(1)
ctx.cell2 = cell().set(2)
ctx.result = cell()
ctx.tf = transformer({
"a": "input",
"b": "input",
"c": "output"
})
ctx.cell1.connect(ctx.tf.a)
ctx.cell2.connect(ctx.tf.b)
ctx.code = cell("transformer").set("c = a + b")
ctx.code.connect(ctx.tf.code)
ctx.tf.c.connect(ctx.result)
topctx.compute()
print(ctx.result.value)
ctx.cell1.set(10)
topctx.compute()
print(ctx.result.value)
ctx.code.set("c = a + b + 1000")
topctx.compute()
print(ctx.result.value)
print(ctx.status)
"""
Final test for compiled transformers, using stdlib.compiled_transformer
"""
from seamless.core import context, cell, transformer, macro, libcell, macro_mode_on
from seamless.core import StructuredCell
from seamless.core import library
from copy import deepcopy
# 1: set up example data
with macro_mode_on():
ctx = context(toplevel=True)
# 1a. Setup of StructuredCells
ctx.inp_struc = context(name="inp_struc",context=ctx)
ctx.inp_struc.storage = cell("text")
ctx.inp_struc.form = cell("json")
ctx.inp_struc.data = cell("mixed",
form_cell = ctx.inp_struc.form,
storage_cell = ctx.inp_struc.storage,
)
ctx.inp_struc.schema = cell("json")
ctx.inp = StructuredCell(
"inp",
ctx.inp_struc.data,
storage = ctx.inp_struc.storage,
form = ctx.inp_struc.form,
schema = ctx.inp_struc.schema,
buffer = None,
inchannels = [],
outchannels = [()]
def translate(graph, ctx, environment):
from ..core.macro_mode import curr_macro
if curr_macro() is None:
print_info("*" * 30 + "TRANSLATE" + "*" * 30)
#import traceback; stack = traceback.extract_stack(); print("TRANSLATE:"); print("".join(traceback.format_list(stack[:3])))
nodes, connections = graph["nodes"], graph["connections"]
contexts = {con["path"]: con for con in nodes if con["type"] == "context"}
for path in sorted(contexts.keys(), key=lambda k: len(k)):
parent = get_path(ctx, path[:-1], None, is_target=False)
name = path[-1]
c = context()
setattr(parent, name, c)
# No need to add it to namespace, as long as the low-level graph structure is imitated
connection_paths = [(con["source"], con["target"]) for con in connections
if con["type"] == "connection"]
namespace = {}
for node in nodes:
t = node["type"]
if t in ("context", "link"):
continue
path = node["path"]
for node in nodes:
t = node["type"]
if t in ("context", "link"):
continue
path = node["path"]
if t == "transformer":
inchannels, outchannels = find_channels(node["path"],
connection_paths)
try:
inchannels.remove(("meta", ))
has_meta_connection = True
except ValueError:
has_meta_connection = False
language = node["language"]
if node["compiled"]:
from .translate_compiled_transformer import translate_compiled_transformer
translate_compiled_transformer(
node,
ctx,
namespace,
inchannels,
outchannels,
has_meta_connection=has_meta_connection)
elif language == "bash":
translate_bash_transformer(
node,
ctx,
namespace,
inchannels,
outchannels,
has_meta_connection=has_meta_connection)
else:
ipy_template = None
py_bridge = None
if language not in ("python", "ipython"):
ok = False
if environment is not None:
try:
ipy_template = environment.get_ipy_template(
language)
ok = True
except KeyError:
pass
try:
py_bridge = environment.get_py_bridge(language)
ok = True
except KeyError:
pass
if ipy_template is not None and py_bridge is not None:
msg = "Language '{}' has an IPython template AND a Python bridge"
raise ValueError(msg.format(language))
if not ok:
raise NotImplementedError(language)
translate_py_transformer(
node,
ctx,
namespace,
inchannels,
outchannels,
ipy_template=ipy_template,
py_bridge=py_bridge,
has_meta_connection=has_meta_connection)
elif t == "macro":
if node["language"] != "python":
raise NotImplementedError(node["language"])
inchannels, outchannels = find_channels(node["path"],
connection_paths)
translate_macro(node, ctx, namespace, inchannels, outchannels)
elif t == "cell":
inchannels, outchannels = find_channels(path, connection_paths)
translate_cell(node, ctx, namespace, inchannels, outchannels)
elif t == "module":
inchannels, outchannels = find_channels(path, connection_paths)
translate_module(node, ctx, namespace, inchannels, outchannels)
elif t == "libinstance":
msg = "Libinstance '%s' was not removed during pre-translation"
raise TypeError(msg % str(path))
else:
raise TypeError(t)
node.pop("UNTRANSLATED", None)
node.pop("UNSHARE", None)
namespace2 = OrderedDict()
for k in sorted(namespace.keys(), key=lambda k: -len(k)):
namespace2[k] = namespace[k]
for connection in connections:
if connection["type"] == "connection":
translate_connection(connection, namespace2, ctx)
elif connection["type"] == "link":
translate_link(connection, namespace2, ctx)
elif connection["type"] == "virtual":
pass
else:
raise TypeError(connection["type"])
icellname = "input_param_" + param
icell = cell("mixed")
setattr(c, icellname, icell)
outchannel = c.input.outchannels[(param, )]
pin = getattr(c.tf, param)
outchannel.connect(icell)
icell.connect(pin)
c.example_data = cell("mixed")
c.example_buffer = cell("mixed")
c.example = StructuredCell(data=c.example_data,
buffer=c.example_buffer,
schema=c.input_schema)
with macro_mode_on():
ctx = context(toplevel=True)
tf_names = [("tf1", ), ("tf2", ), ("tf3", ), ("tf4", )]
channel_names = tf_names # TODO (long term): try numeric path
ctx.params_struc = context()
ctx.params_struc.data = cell("mixed")
ctx.params_struc.auth = cell("mixed")
ctx.params_struc.buffer = cell("mixed")
ctx.params_struc.schema = cell("plain")
ctx.params_struc.example_buffer = cell("mixed")
ctx.params_struc.example_data = cell("mixed")
ctx.params = StructuredCell(
ctx.params_struc.data,
auth=ctx.params_struc.auth,
buffer=ctx.params_struc.buffer,
schema=ctx.params_struc.schema,
"language": "python",
"transformer": True,
"checksum": 'b0480c66eb4dbb31f5311e09e89b9414c880360842b9d5ef7b6621fc31a5ab99',
}],
"connections": [{
"source": ("pi",),
"target": ("doubleit", "a"),
},
{
"source": ("doubleit",),
"target": ("twopi",),
},
{
"source": ("code",),
"target": ("doubleit", "code"),
}],
}
ctx0 = context(toplevel=True)
ctx0.pi = cell("mixed").set(math.pi)
assert ctx0.pi.checksum == '9809b7dfcfe29dd194c71c7d2da94af3aeef98f079eeff8e1d9e5099acef737c'
ctx0.code = cell("python").set("result = a * 2")
assert ctx0.code.checksum == 'b0480c66eb4dbb31f5311e09e89b9414c880360842b9d5ef7b6621fc31a5ab99'
with macro_mode_on():
ctx = context(toplevel=True, manager=ctx0._get_manager())
translate(graph, ctx, [], False)
ctx.equilibrate()
print(ctx.twopi.value)
import seamless
from seamless.core import macro_mode_on
from seamless.core import context, cell, transformer, StructuredCell
from seamless.core.structured_cell import BufferWrapper
import numpy as np
with macro_mode_on():
ctx = context(toplevel=True)
ctx.hub_struc = context(name="hub_struc",context=ctx)
ctx.hub_struc.storage = cell("text")
ctx.hub_struc.form = cell("json")
ctx.hub_struc.data = cell("mixed",
form_cell = ctx.hub_struc.form,
storage_cell = ctx.hub_struc.storage,
)
ctx.hub_struc.schema = cell("json")
ctx.hub_struc.buffer_storage = cell("text")
ctx.hub_struc.buffer_form = cell("json")
ctx.hub_struc.buffer_data = cell("mixed",
form_cell = ctx.hub_struc.buffer_form,
storage_cell = ctx.hub_struc.buffer_storage,
)
bufferwrapper = BufferWrapper(
ctx.hub_struc.buffer_data,
ctx.hub_struc.buffer_storage,
ctx.hub_struc.buffer_form
)
ctx.hub = StructuredCell(
"hub",
ctx.hub_struc.data,
storage = ctx.hub_struc.storage,
def translate_py_transformer(node, root, namespace, inchannels, outchannels, lib_path00, is_lib):
#TODO: simple translation, without a structured cell
assert not "code" in node ### node["code"] is an outdated attribute
inchannels = [ic for ic in inchannels if ic[0] != "code"]
parent = get_path(root, node["path"][:-1], None, None)
name = node["path"][-1]
lib_path0 = lib_path00 + "." + name if lib_path00 is not None else None
ctx = context(toplevel=False)
setattr(parent, name, ctx)
result_name = node["RESULT"]
if node["language"] == "ipython":
assert result_name == "result"
input_name = node["INPUT"]
for c in inchannels:
assert (not len(c)) or c[0] != result_name #should have been checked by highlevel
with_result = node["with_result"]
buffered = node["buffered"]
interchannels = [as_tuple(pin) for pin in node["pins"]]
plain = node["plain"]
mount = node.get("mount", {})
silk = (buffered or not plain)
inp, inp_ctx = build_structured_cell(
ctx, input_name, silk, plain, buffered, inchannels, interchannels,
lib_path0,
return_context=True
)
setattr(ctx, input_name, inp)
if "input_schema" in mount:
inp_ctx.schema.mount(**mount["input_schema"])
for inchannel in inchannels:
path = node["path"] + inchannel
namespace[path, True] = inp.inchannels[inchannel], node
assert result_name not in node["pins"] #should have been checked by highlevel
all_pins = {}
for pinname, pin in node["pins"].items():
p = {"io": "input"}
p.update(pin)
all_pins[pinname] = p
all_pins[result_name] = {"io": "output", "transfer_mode": "copy"}
if node["SCHEMA"]:
assert with_result
all_pins[node["SCHEMA"]] = {
"io": "input", "transfer_mode": "json",
"access_mode": "json", "content_type": "json"
}
ctx.tf = transformer(all_pins)
if node["debug"]:
ctx.tf.debug = True
if lib_path00 is not None:
lib_path = lib_path00 + "." + name + ".code"
ctx.code = libcell(lib_path)
else:
if node["language"] == "ipython":
ctx.code = core_cell("ipython")
else:
ctx.code = core_cell("transformer")
if "code" in mount:
ctx.code.mount(**mount["code"])
ctx.code._sovereign = True
ctx.code.connect(ctx.tf.code)
checksum = node.get("checksum", {})
if "code" in checksum:
ctx.code.set_checksum(checksum["code"])
if "input" in checksum:
inp.set_checksum(checksum["input"])
namespace[node["path"] + ("code",), True] = ctx.code, node
namespace[node["path"] + ("code",), False] = ctx.code, node
for pin in list(node["pins"].keys()):
target = getattr(ctx.tf, pin)
inp.outchannels[(pin,)].connect(target)
if with_result:
plain_result = node["plain_result"]
result, result_ctx = build_structured_cell(
ctx, result_name, True, plain_result, False, [()],
outchannels, lib_path0,
return_context=True
)
if "result_schema" in mount:
result_ctx.schema.mount(**mount["result_schema"])
setattr(ctx, result_name, result)
result_pin = getattr(ctx.tf, result_name)
result_pin.connect(result.inchannels[()])
if node["SCHEMA"]:
schema_pin = getattr(ctx.tf, node["SCHEMA"])
result.schema.connect(schema_pin)
if "result" in checksum:
result.set_checksum(checksum["result"])
if "schema" in checksum:
result.schema.set_checksum(checksum["schema"])
else:
for c in outchannels:
assert len(c) == 0 #should have been checked by highlevel
result = getattr(ctx.tf, result_name)
namespace[node["path"] + (result_name,), False] = result, node
namespace[node["path"], True] = inp, node
namespace[node["path"], False] = result, node
def translate_py_reactor(node, root, namespace, inchannels, outchannels,
editchannels, lib_path00, is_lib):
#TODO: simple-mode translation, without a structured cell
skip_channels = ("code_start", "code_update", "code_stop")
inchannels = [ic for ic in inchannels if ic[0] not in skip_channels]
editchannels = [ec for ec in editchannels if ec[0] not in skip_channels]
parent = get_path(root, node["path"][:-1], None, None)
name = node["path"][-1]
lib_path0 = lib_path00 + "." + name if lib_path00 is not None else None
ctx = context(context=parent, name=name)
setattr(parent, name, ctx)
io_name = node["IO"]
if len(inchannels):
lib_path0 = None #partial authority or no authority; no library update in either case
buffered = node["buffered"]
interchannels_in = [
as_tuple(p) for p, pin in node["pins"].items() if pin["io"] == "output"
]
interchannels_out = [
as_tuple(p) for p, pin in node["pins"].items() if pin["io"] == "input"
]
interchannels_edit = [
as_tuple(p) for p, pin in node["pins"].items() if pin["io"] == "edit"
]
all_inchannels = interchannels_in + inchannels #highlevel must check that there are no duplicates
all_outchannels = interchannels_out + [
p for p in outchannels if p not in interchannels_out
]
all_editchannels = interchannels_edit + [
p for p in editchannels if p not in interchannels_edit
]
plain = node["plain"]
io_state = node.get("stored_state_io", None)
if io_state is None:
io_state = node.get("cached_state_io", None)
io = build_structured_cell(ctx,
io_name,
True,
plain,
buffered,
all_inchannels,
all_outchannels,
io_state,
lib_path0,
editchannels=all_editchannels)
setattr(ctx, io_name, io)
for inchannel in inchannels:
path = node["path"] + inchannel
namespace[path, True] = io.inchannels[inchannel], node
for outchannel in outchannels:
path = node["path"] + outchannel
namespace[path, False] = io.outchannels[outchannel], node
for channel in editchannels:
path = node["path"] + channel
namespace[path, True] = io.editchannels[channel], node
namespace[path, False] = io.editchannels[channel], node
ctx.rc = reactor(node["pins"])
for attr in ("code_start", "code_stop", "code_update"):
if lib_path00 is not None:
lib_path = lib_path00 + "." + name + "." + attr
c = libcell(lib_path)
setattr(ctx, attr, c)
else:
c = core_cell(node["language"])
c._sovereign = True
setattr(ctx, attr, c)
if "mount" in node and attr in node["mount"]:
c.mount(**node["mount"][attr])
c.connect(getattr(ctx.rc, attr))
code = node.get(attr)
if code is None:
code = node.get("cached_" + attr)
try_set(c, code)
namespace[node["path"] + (attr, ), True] = c, node
namespace[node["path"] + (attr, ), False] = c, node
for pinname, pin in node["pins"].items():
target = getattr(ctx.rc, pinname)
iomode = pin["io"]
if iomode == "input":
io.connect_outchannel((pinname, ), target)
elif iomode == "edit":
io.connect_editchannel((pinname, ), target)
elif iomode == "output":
io.connect_inchannel(target, (pinname, ))
temp = node.get("TEMP")
if temp is None:
temp = {}
for attr in ("code_start", "code_stop", "code_update"):
if attr in temp:
try_set(getattr(ctx, attr), temp[attr])
iohandle = io.handle
for k, v in temp.items():
if k in ("code_start", "code_stop", "code_update"):
continue
setattr(iohandle, k, v)
if not is_lib: #clean up cached state and in_equilibrium, unless a library context
node.pop("cached_state_io", None)
namespace[node["path"], True] = io, node
namespace[node["path"], False] = io, node
node.pop("TEMP", None)
import seamless
from seamless.core import macro_mode_on
from seamless.core import context, cell, transformer, pytransformercell
with macro_mode_on():
ctx = context(toplevel=True)
ctx.cell1 = cell().set(1)
ctx.cell2 = cell().set(2)
ctx.result = cell()
ctx.tf = transformer({"a": "input", "b": "input", "c": "output"})
ctx.cell1.connect(ctx.tf.a)
ctx.cell2.connect(ctx.tf.b)
ctx.code = pytransformercell().set("c = a + b")
ctx.code.connect(ctx.tf.code)
ctx.tf.c.connect(ctx.result)
shell = ctx.tf.shell()
def translate(graph, ctx, from_lib_paths, is_lib):
###import traceback; stack = traceback.extract_stack(); print("TRANSLATE:"); print("".join(traceback.format_list(stack[:3])))
contexts = {con["path"]: con for con in graph if con["type"] == "context"}
for path in sorted(contexts.keys(), key=lambda k: len(k)):
parent = get_path(ctx, path[:-1], None, is_target=False)
name = path[-1]
c = context(context=parent, name=name)
setattr(parent, name, c)
# No need to add it to namespace, as long as the low-level graph structure is imitated
connections = [con for con in graph if con["type"] == "connection"]
connection_paths = [(con["source"], con["target"]) for con in connections]
links = [con for con in graph if con["type"] == "link"]
link_paths = [(node["first"]["path"], node["second"]["path"])
for node in links]
lowlevel_links = {}
#multiple links (A,B), (A,C) are allowed; A becomes the real one
#multiple links (B,A), (C,A) are not allowed
#multiple links (A,B), (A,C), (D,A), (E,D) leads to A,B,C,D => E, E as the real one
simple_links = []
for node in links:
first, second = node["first"], node["second"]
if not first["simple"]:
continue
if not second["simple"]:
continue
first, second = first["path"], second["path"]
simple_links.append((first, second))
assert second not in lowlevel_links
lowlevel_links[second] = first
change = True
while change:
change = False
simple_links0 = simple_links
simple_links = []
for first, second in simple_links0:
while first in lowlevel_links:
first = lowlevel_links[first]
change = True
lowlevel_links[second] = first
simple_links.append((first, second))
link_target_paths = set(lowlevel_links.values())
link_targets = {
} #maps "first" paths of a link (aka link target paths, aka "real cells") to their translated cells
namespace = {}
for node in graph:
t = node["type"]
if t in ("context", "connection", "link"):
continue
path = node["path"]
lib_path = get_lib_path(path[:-1], from_lib_paths)
if t == "cell" and path in link_target_paths:
assert node[
"celltype"] != "structured" #low-level links are between simple cells!
inchannels, outchannels = find_channels(path, connection_paths)
editchannels = find_editchannels(path, link_paths)
translated_cell = translate_cell(node, ctx, namespace, inchannels,
outchannels, editchannels,
lib_path, is_lib)
link_targets[path] = translated_cell
#print("LOW-LEVEL LINKS", lowlevel_links)
#print("LOW-LEVEL LINK TARGETS", link_targets)
for node in graph:
t = node["type"]
if t in ("context", "connection", "link"):
continue
path = node["path"]
lib_path = get_lib_path(path[:-1], from_lib_paths)
if t == "transformer":
inchannels, outchannels = find_channels(node["path"],
connection_paths)
if node["compiled"]:
from .translate_compiled_transformer import translate_compiled_transformer
translate_compiled_transformer(node, ctx, namespace,
inchannels, outchannels,
lib_path, is_lib)
elif node["language"] in ("python", "ipython"):
translate_py_transformer(node, ctx, namespace, inchannels,
outchannels, lib_path, is_lib)
elif node["language"] == "bash":
translate_bash_transformer(node, ctx, namespace, inchannels,
outchannels, lib_path, is_lib)
else:
raise NotImplementedError(node["language"])
elif t == "reactor":
if node["language"] not in ("python", "ipython"):
raise NotImplementedError(node["language"])
inchannels, outchannels = find_channels(node["path"],
connection_paths)
editchannels = find_editchannels(node["path"], link_paths)
translate_py_reactor(node, ctx, namespace, inchannels, outchannels,
editchannels, lib_path, is_lib)
elif t == "cell":
if path in link_target_paths:
continue #done already before
inchannels, outchannels = find_channels(path, connection_paths)
editchannels = find_editchannels(path, link_paths)
link_target = None
if path in lowlevel_links:
link_target = link_targets[lowlevel_links[path]]
translate_cell(node,
ctx,
namespace,
inchannels,
outchannels,
editchannels,
lib_path,
is_lib,
link_target=link_target)
else:
raise TypeError(t)
namespace2 = OrderedDict()
for k in sorted(namespace.keys(), key=lambda k: -len(k)):
namespace2[k] = namespace[k]
for node in links:
translate_link(node, namespace2, ctx)
for node in connections:
translate_connection(node, namespace2, ctx)
def create(ctx, mount, state=None):
with macro_mode_on():
ctx.hub_struc = context(name="hub_struc",context=ctx)
ctx.hub_struc.storage = cell("text")
ctx.hub_struc.form = cell("json")
ctx.hub_struc.data = cell("mixed",
form_cell = ctx.hub_struc.form,
storage_cell = ctx.hub_struc.storage,
)
ctx.hub_struc.schema = cell("json")
ctx.hub_struc.buffer_storage = cell("text")
ctx.hub_struc.buffer_form = cell("json")
ctx.hub_struc.buffer_data = cell("mixed",
form_cell = ctx.hub_struc.buffer_form,
storage_cell = ctx.hub_struc.buffer_storage,
)
bufferwrapper = BufferWrapper(
ctx.hub_struc.buffer_data,
ctx.hub_struc.buffer_storage,
ctx.hub_struc.buffer_form
)
ctx.hub = StructuredCell(
"hub",
ctx.hub_struc.data,
storage = ctx.hub_struc.storage,
form = ctx.hub_struc.form,
schema = ctx.hub_struc.schema,
buffer = bufferwrapper,
inchannels = [("a", "factor1"), ("a", "factor2"), ("b",)],
outchannels = [("a",), ("b",), ("c",)],
state = state,
)
ctx.code = cell("transformer").set("d = a.factor1 * b + a.factor2 * c + a.constant")
ctx.mixer = transformer({
"a": ("input", "ref", "silk"),
"b": ("input", "ref", "object"),
"c": ("input", "ref", "object"),
"d": ("output", "ref", "json"),
})
ctx.a_factor1 = cell("json")
ctx.a_factor2 = cell("json")
ctx.b = cell("json")
ctx.code.connect(ctx.mixer.code)
ctx.hub.connect_inchannel(ctx.a_factor1, ("a", "factor1"))
ctx.hub.connect_inchannel(ctx.a_factor2, ("a", "factor2"))
ctx.hub.connect_inchannel(ctx.b, ("b",))
ctx.hub.connect_outchannel(("a",), ctx.mixer.a)
ctx.hub.connect_outchannel(("b",), ctx.mixer.b)
ctx.hub.connect_outchannel(("c",), ctx.mixer.c)
ctx.result_struc = context(name="result_struc",context=ctx)
ctx.result_struc.storage = cell("text")
ctx.result_struc.form = cell("json")
ctx.result_struc.data = cell("mixed",
form_cell = ctx.result_struc.form,
storage_cell = ctx.result_struc.storage,
)
ctx.result_struc.schema = cell("json")
ctx.result_struc.buffer_storage = cell("text")
ctx.result_struc.buffer_form = cell("json")
ctx.result_struc.buffer_data = cell("mixed",
form_cell = ctx.result_struc.buffer_form,
storage_cell = ctx.result_struc.buffer_storage,
)
bufferwrapper = BufferWrapper(
ctx.result_struc.buffer_data,
ctx.result_struc.buffer_storage,
ctx.result_struc.buffer_form
)
ctx.result = StructuredCell(
"result",
ctx.result_struc.data,
storage = ctx.result_struc.storage,
form = ctx.result_struc.form,
schema = ctx.result_struc.schema,
buffer = bufferwrapper,
inchannels = [()],
outchannels = []
)
ctx.result.connect_inchannel(ctx.mixer.d, ())
if mount:
ctx.mount("/tmp/mount-test")
raise NotImplementedError ###
import seamless
from seamless.core import macro_mode_on
from seamless.core import context, cell, transformer, macro, libcell, StructuredCell
from seamless.core.structured_cell import BufferWrapper, StructuredCellState
from seamless.core import library
with macro_mode_on():
ctx = context(toplevel=True)
def create(ctx, with_buffer, with_schema, inchannels):
with macro_mode_on():
ctx.struc = context(name="struc",context=ctx)
ctx.struc.storage = cell("text")
ctx.struc.form = cell("json")
ctx.struc.data = cell("mixed",
form_cell = ctx.struc.form,
storage_cell = ctx.struc.storage,
)
schema = None
if with_schema:
ctx.struc.schema = cell("json")
schema = ctx.struc.schema
bufferwrapper = None
if with_buffer:
ctx.struc.buffer_storage = cell("text")
ctx.struc.buffer_form = cell("json")
ctx.struc.buffer_data = cell("mixed",
form_cell = ctx.struc.buffer_form,
storage_cell = ctx.struc.buffer_storage,
