def load(ctx): ctx.readme = libcell(".readme") def recreate(ctx, name0, with_buffer, with_schema, inchannels): name = "." + name0 ctx.struc = context(name="struc",context=ctx) ctx.struc.storage = libcell(name+".storage") ctx.struc.form = libcell(name+".form") ctx.struc.data = libmixedcell(name+".data", form_cell = ctx.struc.form, storage_cell = ctx.struc.storage, ) schema = None if with_schema: ctx.struc.schema = libcell(name+".schema") schema = ctx.struc.schema bufferwrapper = None if with_buffer: ctx.struc.buffer_storage = libcell(name+".buffer_storage") ctx.struc.buffer_form = libcell(name+".buffer_form") ctx.struc.buffer_data = libmixedcell(name+".buffer_data", form_cell = ctx.struc.buffer_form, storage_cell = ctx.struc.buffer_storage, ) bufferwrapper = BufferWrapper( ctx.struc.buffer_data, ctx.struc.buffer_storage, ctx.struc.buffer_form ) ctx.hub = StructuredCell( "hub", ctx.struc.data, storage = ctx.struc.storage, form = ctx.struc.form, schema = schema, buffer = bufferwrapper, inchannels = inchannels, outchannels = [], ) ctx.auth_json = context(name="auth_json", context=ctx) recreate(ctx.auth_json, "auth_json.struc", with_buffer=False, with_schema=False, inchannels=[]) ctx.auth = context(name="auth", context=ctx) recreate(ctx.auth, "auth.struc", with_buffer=False, with_schema=True, inchannels=[]) ctx.err = context(name="err", context=ctx) recreate(ctx.err, "err.struc", with_buffer=True, with_schema=True, inchannels=[]) ctx.nauth = context(name="nauth", context=ctx) try: recreate(ctx.nauth, "nauth.struc", with_buffer=False, with_schema=False, inchannels=[("z",)]) except: import traceback; traceback.print_exc()
def translate_module(node, root, namespace, inchannels, outchannels): module_type = node["module_type"] language = node["language"] dependencies = node["dependencies"] path = node["path"] parent = get_path(root, path[:-1], None, None) name = path[-1] for c in inchannels + outchannels: assert not len(c) #should have been checked by highlevel subcontext = context(toplevel=False) setattr(parent, name, subcontext) codecell = core_cell("plain") subcontext.code = codecell if node.get("fingertip_no_recompute"): codecell._fingertip_recompute = False if node.get("fingertip_no_remote"): codecell._fingertip_remote = False pathstr = "." + ".".join(path) checksum = node.get("checksum") if checksum is not None: codecell._set_checksum(checksum, initial=True) if "mount" in node: codecell2 = core_cell("text") subcontext.code2 = codecell2 codecell2.mount(**node["mount"]) mode = node["mount"].get("mode", "rw") if mode == "rw": codecell2.bilink(codecell) elif mode == "r": codecell2.connect(codecell) elif mode == "w": codecell.connect(codecell2) subcontext.module_cell = core_cell("plain") subcontext.gen_module_cell = transformer({ "module_type": ("input", "str"), "language": ("input", "str"), "module_code": ("input", "plain"), "dependencies": ("input", "plain"), "result": ("output", "plain") }) subcontext.gen_module_cell.code.cell().set( inspect.getsource(gen_module_cell)) subcontext.gen_module_cell.module_type.cell().set(module_type) subcontext.gen_module_cell.language.cell().set(language) subcontext.gen_module_cell.dependencies.cell().set(dependencies) codecell.connect(subcontext.gen_module_cell.module_code) subcontext.gen_module_cell.result.connect(subcontext.module_cell) namespace[path, "source"] = subcontext.module_cell, node namespace[path, "target"] = codecell, node return subcontext
def main(): global ctx, compute ctx = context(toplevel=True) ctx.select_compute = libcell("compute.select") ctx.compute = macro(select_params, lib="compute") ctx.select_compute.connect(ctx.compute.code) ctx.compute.which.cell().set("pi") compute = ctx.compute.gen_context ctx.iterations = cell().set(10000) ctx.iterations.connect(compute.iterations) ctx.result = cell() compute.result.connect(ctx.result)
async def main(): ctx = context(toplevel=True) ctx.cell1 = cell("int").set(1) ctx.cell2 = cell("int").set(2) #ctx.code = cell("transformer") #ctx.code = cell("transformer").set("c = 'test'") #ctx.code = cell("transformer").set("raise Exception") #ctx.code = cell("transformer").set("import time; time.sleep(2); c = a + b") ctx.code = cell("transformer").set("a + b") ctx.result = cell("int") ctx.tf = transformer({ "a": "input", "b": "input", "c": "output" }) ctx.cell1_unilink = unilink(ctx.cell1) ctx.cell1_unilink.connect(ctx.tf.a) ctx.cell2.connect(ctx.tf.b) ctx.code_copy = cell("transformer") ctx.code.connect(ctx.code_copy) ctx.code_copy.connect(ctx.tf.code) ctx.result_unilink = unilink(ctx.result) ctx.tf.c.connect(ctx.result_unilink) ctx.result_copy = cell("int") ctx.result.connect(ctx.result_copy) await ctx.computation(1) print("STOP") print(ctx.cell1.value, ctx.cell1, ctx.cell1.status) print(ctx.cell2.value, ctx.cell2, ctx.cell2.status) print(ctx.code.value, ctx.code, ctx.code.status) print(ctx.code_copy.value, ctx.code_copy, ctx.code_copy.status) print(ctx.result.value, ctx.result, ctx.result.status) print(ctx.result_copy.value, ctx.result_copy, ctx.result_copy.status) print(ctx.tf.value, ctx.tf, ctx.tf.status) print(ctx.status) print(ctx.tf.exception) await ctx.computation() ctx.cell1.set(10) await ctx.computation() print(ctx.result.value, ctx.status) ctx.code.set("c = a + b + 1000") await ctx.computation() print(ctx.result.value, ctx.status) print("Introduce delay...") ctx.code.set("import time; time.sleep(2); c = -(a + b)") await ctx.computation(1.0) print("after 1.0 sec...") print(ctx.result.value, ctx.status) print("...") await ctx.computation() print(ctx.result.value, ctx.status)
def build_structured_cell(ctx, name, inchannels, outchannels, *, fingertip_no_remote, fingertip_no_recompute, mount=None, return_context=False, hash_pattern=None): #print("build_structured_cell", name) name2 = name + STRUC_ID c = context(toplevel=False) setattr(ctx, name2, c) if mount is not None: mount.pop("as_directory", None) c.mount(**mount) c.data = core_cell("mixed") c.data._hash_pattern = hash_pattern c.auth = core_cell("mixed") c.auth._hash_pattern = hash_pattern c.schema = core_cell("plain") c.buffer = core_cell("mixed") c.buffer._hash_pattern = hash_pattern sc = StructuredCell(data=c.data, auth=c.auth, schema=c.schema, buffer=c.buffer, inchannels=inchannels, outchannels=outchannels, hash_pattern=hash_pattern) c.example_data = core_cell("mixed") c.example_buffer = core_cell("mixed") c.example = StructuredCell(c.example_data, buffer=c.example_buffer, schema=c.schema) for cc in (c.data, c.buffer, c.schema, c.auth, c.example_data, c.example_buffer): if fingertip_no_recompute: cc._fingertip_recompute = False if fingertip_no_remote: cc._fingertip_remote = False if return_context: return sc, c else: return sc
def define_ctx(): with macro_mode_on(): ctx = context(toplevel=True) ctx.cell1 = cell("json").set(1) ctx.cell2 = cell("json").set(2) ctx.result = cell("json") ctx.tf = transformer({"a": "input", "b": "input", "c": "output"}) ctx.cell1_link = link(ctx.cell1) ctx.cell1_link.connect(ctx.tf.a) ctx.cell2.connect(ctx.tf.b) ctx.code = pytransformercell().set("c = a + b") ctx.code.connect(ctx.tf.code) ctx.result_link = link(ctx.result) ctx.tf.c.connect(ctx.result_link) return ctx
def main(): global ctx, compute ctx = context(toplevel=True) ctx.select_compute = libcell("compute.select") ctx.compute = macro(select_params, lib="compute") ctx.select_compute.connect(ctx.compute.code) ###ctx.compute.which.cell().set("pi") # TODO: malfunctioning ### KLUDGE ctx.which_cell = cell().set("pi") ctx.which_cell.connect(ctx.compute.which) ### /KLUDGE compute = ctx.compute.ctx ctx.iterations = cell().set(10000) ctx.iterations.connect(compute.iterations) ctx.result = cell() compute.result.connect(ctx.result)
def reset_backend(share_schemas=True, with_hash_pattern=True): hp = hash_pattern if with_hash_pattern else None global ctx, s, s2, s3 if ctx is not None: ctx.compute() # makes no difference, but could be easier debugging ctx.destroy() ctx = context(toplevel=True) ctx.data = cell("mixed", hash_pattern=hp) ctx.buffer = cell("mixed", hash_pattern=hp) ctx.schema = cell("plain") ctx.sc = StructuredCell( buffer=ctx.buffer, data=ctx.data, schema=ctx.schema, hash_pattern=hp ) s = ctx.sc.handle ctx.data2 = cell("mixed", hash_pattern=hp) ctx.buffer2 = cell("mixed", hash_pattern=hp) if share_schemas: schema2 = ctx.schema else: ctx.schema2 = cell("plain") schema2 = ctx.schema2 ctx.sc2 = StructuredCell( buffer=ctx.buffer2, data=ctx.data2, schema=schema2, hash_pattern=hp ) s2 = ctx.sc2.handle hp3 = None # never use hash pattern for this one ctx.data3 = cell("mixed", hash_pattern=hp3) ctx.buffer3 = cell("mixed", hash_pattern=hp3) if share_schemas: schema3 = ctx.schema else: ctx.schema3 = cell("plain") schema3 = ctx.schema3 ctx.sc3 = StructuredCell( buffer=ctx.buffer3, data=ctx.data3, schema=schema3, hash_pattern=hp3 ) s3 = ctx.sc3.handle
def define_ctx(): with macro_mode_on(): ctx = context(toplevel=True) ctx.cell1 = cell().set(1) ctx.cell2 = cell().set(2) ctx.result = cell() ctx.tf = transformer({ "a": "input", "b": "input", "c": "output" }) ctx.cell1_link = link(ctx.cell1) ctx.cell1_link.connect(ctx.tf.a) ctx.cell2.connect(ctx.tf.b) ctx.code = pytransformercell().set("c = a + b") ctx.code.connect(ctx.tf.code) ctx.result_link = link(ctx.result) ctx.tf.c.connect(ctx.result_link) return ctx
def transform_job(rqdata): x = decode(rqdata, as_cells=True) transformer_params, output_signature, cells, _, _ = x inputs = cells.keys() with macro_mode_on(): ctx = context(toplevel=True) tf = ctx.TRANSFORMER = transformer(transformer_params) for k in inputs: setattr(ctx, k, cells[k]) cells[k].connect(getattr(tf, k)) outputpin = getattr(tf, tf._output_name) for n, outp in enumerate(output_signature): name = "RESULT" + str(n + 1) if outp == "mixed": c2 = cell("text") setattr(ctx, name + "_storage", c2) c3 = cell("json") setattr(ctx, name + "_form", c3) c = cell("mixed", storage_cell=c2, form_cell=c3) else: c = cell(outp) setattr(ctx, name, c) outputpin.connect(c) while tf.status() not in ("OK", "ERROR"): ctx.equilibrate(1) result = {} if tf.status() != "OK": result["ERROR"] = ctx.tf.transformer.EXCEPTION else: for n, outp in enumerate(output_signature): name = "RESULT" + str(n + 1) data = getattr(ctx, name) if outp == "mixed": storage = getattr(ctx, name + "_storage") result["STORAGE"] = storage form = getattr(ctx, name + "_form") result["FORM"] = form result[outp] = data for key, c in list(result.items()): val = c.serialize_buffer() result[key] = val return result
def macro_code(ctx, param): ctx.sub = context() ctx.a = cell().set(1000 + param) ctx.b = cell().set(2000 + param) ctx.result = cell() ctx.tf = transformer({ "a": "input", "b": "input", "c": "output" }) ctx.a.connect(ctx.tf.a) ctx.b.connect(ctx.tf.b) ctx.code = cell("transformer").set("print('TRANSFORM'); import time; time.sleep(2); c = a + b") ctx.code.connect(ctx.tf.code) ctx.tf.c.connect(ctx.result) assert param != 999 # on purpose if param > 1: ctx.d = cell().set(42) # raise Exception("on purpose") #causes the macro reconstruction to fail pass # For some reason, comments at the end are not captured with inspect.get_source?
def create(ctx, with_buffer, with_schema, inchannels): with macro_mode_on(): ctx.struc = context(name="struc", context=ctx) ctx.struc.storage = cell("text") ctx.struc.form = cell("json") ctx.struc.data = cell( "mixed", form_cell=ctx.struc.form, storage_cell=ctx.struc.storage, ) schema = None if with_schema: ctx.struc.schema = cell("json") schema = ctx.struc.schema bufferwrapper = None if with_buffer: ctx.struc.buffer_storage = cell("text") ctx.struc.buffer_form = cell("json") ctx.struc.buffer_data = cell( "mixed", form_cell=ctx.struc.buffer_form, storage_cell=ctx.struc.buffer_storage, ) bufferwrapper = BufferWrapper(ctx.struc.buffer_data, ctx.struc.buffer_storage, ctx.struc.buffer_form) ctx.hub = StructuredCell( "hub", ctx.struc.data, storage=ctx.struc.storage, form=ctx.struc.form, schema=schema, buffer=bufferwrapper, inchannels=inchannels, outchannels=[], )
def recreate(ctx, name0, with_buffer, with_schema, inchannels): name = "." + name0 ctx.struc = context(name="struc",context=ctx) ctx.struc.storage = libcell(name+".storage") ctx.struc.form = libcell(name+".form") ctx.struc.data = libmixedcell(name+".data", form_cell = ctx.struc.form, storage_cell = ctx.struc.storage, ) schema = None if with_schema: ctx.struc.schema = libcell(name+".schema") schema = ctx.struc.schema bufferwrapper = None if with_buffer: ctx.struc.buffer_storage = libcell(name+".buffer_storage") ctx.struc.buffer_form = libcell(name+".buffer_form") ctx.struc.buffer_data = libmixedcell(name+".buffer_data", form_cell = ctx.struc.buffer_form, storage_cell = ctx.struc.buffer_storage, ) bufferwrapper = BufferWrapper( ctx.struc.buffer_data, ctx.struc.buffer_storage, ctx.struc.buffer_form ) ctx.hub = StructuredCell( "hub", ctx.struc.data, storage = ctx.struc.storage, form = ctx.struc.form, schema = schema, buffer = bufferwrapper, inchannels = inchannels, outchannels = [], )
def create(ctx, with_buffer, with_schema, inchannels): with macro_mode_on(): ctx.struc = context(name="struc",context=ctx) ctx.struc.storage = cell("text") ctx.struc.form = cell("json") ctx.struc.data = cell("mixed", form_cell = ctx.struc.form, storage_cell = ctx.struc.storage, ) schema = None if with_schema: ctx.struc.schema = cell("json") schema = ctx.struc.schema bufferwrapper = None if with_buffer: ctx.struc.buffer_storage = cell("text") ctx.struc.buffer_form = cell("json") ctx.struc.buffer_data = cell("mixed", form_cell = ctx.struc.buffer_form, storage_cell = ctx.struc.buffer_storage, ) bufferwrapper = BufferWrapper( ctx.struc.buffer_data, ctx.struc.buffer_storage, ctx.struc.buffer_form ) ctx.hub = StructuredCell( "hub", ctx.struc.data, storage = ctx.struc.storage, form = ctx.struc.form, schema = schema, buffer = bufferwrapper, inchannels = inchannels, outchannels = [], )
def recreate(ctx, name0, with_buffer, with_schema, inchannels): name = "." + name0 ctx.struc = context(name="struc", context=ctx) ctx.struc.storage = libcell(name + ".storage") ctx.struc.form = libcell(name + ".form") ctx.struc.data = libmixedcell( name + ".data", form_cell=ctx.struc.form, storage_cell=ctx.struc.storage, ) schema = None if with_schema: ctx.struc.schema = libcell(name + ".schema") schema = ctx.struc.schema bufferwrapper = None if with_buffer: ctx.struc.buffer_storage = libcell(name + ".buffer_storage") ctx.struc.buffer_form = libcell(name + ".buffer_form") ctx.struc.buffer_data = libmixedcell( name + ".buffer_data", form_cell=ctx.struc.buffer_form, storage_cell=ctx.struc.buffer_storage, ) bufferwrapper = BufferWrapper(ctx.struc.buffer_data, ctx.struc.buffer_storage, ctx.struc.buffer_form) ctx.hub = StructuredCell( "hub", ctx.struc.data, storage=ctx.struc.storage, form=ctx.struc.form, schema=schema, buffer=bufferwrapper, inchannels=inchannels, outchannels=[], )
import seamless from seamless.core import macro_mode_on from seamless.core import context, cell, transformer, StructuredCell import numpy as np with macro_mode_on(): ctx = context(toplevel=True) ctx.inp_struc = context(name="inp_struc",context=ctx) ctx.inp_struc.storage = cell("text") ctx.inp_struc.form = cell("json") ctx.inp_struc.data = cell("mixed", form_cell = ctx.inp_struc.form, storage_cell = ctx.inp_struc.storage, ) ctx.inp_struc.schema = cell("json") ctx.inp = StructuredCell( "inp", ctx.inp_struc.data, storage = ctx.inp_struc.storage, form = ctx.inp_struc.form, schema = ctx.inp_struc.schema, buffer = None, inchannels = [("a",)], outchannels = [()] ) ctx.a = cell("json") ctx.inp.connect_inchannel(ctx.a, ("a",)) ctx.tf = transformer({ "inp": ("input", "copy", "silk"), "c": "output",
raise NotImplementedError import seamless from seamless.core import macro_mode_on from seamless.core import context, cell, transformer, StructuredCell from seamless.core.structured_cell import BufferWrapper, StructuredCellState import numpy as np with macro_mode_on(): ctx = context(toplevel=True) def create(ctx, mount, state=None): with macro_mode_on(): ctx.hub_struc = context(name="hub_struc",context=ctx) ctx.hub_struc.storage = cell("text") ctx.hub_struc.form = cell("json") ctx.hub_struc.data = cell("mixed", form_cell = ctx.hub_struc.form, storage_cell = ctx.hub_struc.storage, ) ctx.hub_struc.schema = cell("json") ctx.hub_struc.buffer_storage = cell("text") ctx.hub_struc.buffer_form = cell("json") ctx.hub_struc.buffer_data = cell("mixed", form_cell = ctx.hub_struc.buffer_form, storage_cell = ctx.hub_struc.buffer_storage, ) bufferwrapper = BufferWrapper( ctx.hub_struc.buffer_data, ctx.hub_struc.buffer_storage, ctx.hub_struc.buffer_form
def translate_compiled_transformer(node, root, namespace, inchannels, outchannels, lib_path00, is_lib): #TODO: still a lot of common code with translate_py_transformer, put in functions inchannels = [ic for ic in inchannels if ic[0] != "code"] main_module_inchannels = [("objects", ) + ic[1:] for ic in inchannels if ic[0] == "_main_module"] inchannels = [ic for ic in inchannels if ic[0] != "_main_module"] parent = get_path(root, node["path"][:-1], None, None) name = node["path"][-1] lib_path0 = lib_path00 + "." + name if lib_path00 is not None else None ctx = context(context=parent, name=name) setattr(parent, name, ctx) input_name = node["INPUT"] result_name = node["RESULT"] if len(inchannels): lib_path0 = None #partial authority or no authority; no library update in either case for c in inchannels: assert (not len( c)) or c[0] != result_name #should have been checked by highlevel with_result = node["with_result"] assert with_result #compiled transformers must have with_result buffered = node["buffered"] mount = node.get("mount", {}) plain = node["plain"] input_state = node.get("stored_state_input", None) if input_state is None: input_state = node.get("cached_state_input", None) inp, inp_ctx = build_structured_cell(ctx, input_name, True, plain, buffered, inchannels, [()], input_state, lib_path0, return_context=True) setattr(ctx, input_name, inp) if "input_schema" in mount: inp_ctx.schema.mount(**mount["input_schema"]) for inchannel in inchannels: path = node["path"] + inchannel namespace[path, True] = inp.inchannels[inchannel], node assert result_name not in node[ "pins"] #should have been checked by highlevel assert "translator_result_" not in node[ "pins"] #should have been checked by highlevel all_pins = {} inputpins = [] for pinname, pin in node["pins"].items(): p = {"io": "input"} p.update(pin) if p["io"] == "input": inputpins.append(pinname) all_pins[pinname] = p all_pins[result_name] = "output" if node["SCHEMA"]: assert with_result all_pins[node["SCHEMA"]] = { "io": "input", "transfer_mode": "json", "access_mode": "json", "content_type": "json" } in_equilibrium = node.get("in_equilibrium", False) temp = node.get("TEMP") if temp is None: temp = {} # Compiler ctx.language = cell("text").set(node["language"]) main_module_state = node.get("stored_state_main_module", None) if main_module_state is None: main_module_state = node.get("cached_state_main_module", None) ctx.main_module = build_structured_cell( ctx, "main_module", False, True, False, main_module_inchannels, [()], main_module_state, lib_path00, ) if "_main_module" in temp and len(temp["_main_module"]): temp_main_module = temp["_main_module"] main_module_handle = ctx.main_module.handle main_module_data = ctx.main_module.data.value if main_module_data is None: ctx.main_module.monitor.set_path((), {"objects": {}}, forced=True) main_module_data = ctx.main_module.data.value elif "objects" not in main_module_data: main_module_handle["objects"] = {} for objname, obj in temp_main_module.items(): for key, value in obj.items(): if objname in main_module_data["objects"] and \ key in main_module_data["objects"][objname]: msg = "WARNING: %s main module object '%s': %s already defined" print(msg % (node["path"], objname, key)) continue if objname not in main_module_data["objects"]: ctx.main_module.monitor.set_path(("objects", objname), {}, forced=True) main_module_handle["objects"][objname][key] = value elif main_module_state is None: ctx.main_module.monitor.set_path((), {}, forced=True) for ic in main_module_inchannels: icpath = node["path"] + ("_main_module", ) + ic[1:] namespace[icpath, True] = ctx.main_module.inchannels[ic], node compiler_verbose = node["main_module"]["compiler_verbose"] ctx.compiler_verbose = cell("json").set(compiler_verbose) target = node["main_module"].get("target") if target is not None: ctx.main_module.monitor.set_path(("target", ), target, forced=True) # Transformer itself ctf = ctx.tf = context(name="tf", context=ctx) debug = node["debug"] _init_from_library(ctf, debug) if lib_path00 is not None: lib_path = lib_path00 + "." + name + ".code" ctx.code = libcell(lib_path) else: ctx.code = cell("text") ctx.code.set_file_extension(node["file_extension"]) if "code" in mount: ctx.code.mount(**mount["code"]) plain_result = node["plain_result"] result_state = node.get("cached_state_result", None) result, result_ctx = build_structured_cell(ctx, result_name, True, plain_result, False, [()], outchannels, result_state, lib_path0, return_context=True) if "result_schema" in mount: result_ctx.schema.mount(**mount["result_schema"]) setattr(ctx, result_name, result) assert not node["SCHEMA"] ctx.pins = cell("json").set(all_pins) ctx.inputpins = cell("json").set(inputpins) c_inp = getattr(ctx, input_name + STRUC_ID) c_result = getattr(ctx, result_name + STRUC_ID) _finalize(ctx, ctf, inp, c_inp, result, c_result, input_name, result_name) if "header" in mount: ctx.header.mount(**mount["header"]) code = node.get("code") if code is None: code = node.get("cached_code") if code is not None: ctx.code.set(code) if "code" in temp: ctx.code.set(temp["code"]) inphandle = inp.handle for k, v in temp.items(): if k in ("code", "_main_module"): continue setattr(inphandle, k, v) namespace[node["path"] + ("code", ), True] = ctx.code, node namespace[node["path"] + ("code", ), False] = ctx.code, node if not is_lib: #clean up cached state and in_equilibrium, unless a library context node.pop("cached_state_input", None) if not in_equilibrium: node.pop("cached_state_result", None) node.pop("in_equilibrium", None) namespace[node["path"], True] = inp, node namespace[node["path"], False] = result, node node.pop("TEMP", None)
def map_dict_chunk(ctx, chunksize, graph, inp, keyorder, has_uniform, elision, lib_module_dict): #print("map_dict_chunk", inp) from seamless.core import Cell as CoreCell from seamless.core import cell, context, macro, path, transformer from seamless.core.structured_cell import StructuredCell from seamless.core.HighLevelContext import HighLevelContext from seamless.core.unbound_context import UnboundContext import math pseudo_connections = [] ctx.sc_data = cell("mixed") ctx.sc_buffer = cell("mixed") inpkeys = keyorder nchunks = math.ceil(len(inpkeys) / chunksize) ctx.sc = StructuredCell( data=ctx.sc_data, buffer=ctx.sc_buffer, inchannels=[(n + 1, ) for n in range(nchunks)], outchannels=[()], ) if has_uniform: ctx.uniform = cell("mixed") first = True for n in range(nchunks): pos = chunksize * n hc = HighLevelContext(graph) subctx = "subctx_" + str(n + 1) setattr(ctx, subctx, hc) if first: if not hasattr(hc, "inp"): raise TypeError( "map_dict_chunk context must have a cell called 'inp'") hci = hc.inp if has_uniform: if first: if not hasattr(hc, "uniform"): raise TypeError( "map_dict_chunk context must have a cell called 'uniform'" ) if isinstance(hc.uniform, StructuredCell): raise TypeError( "map_dict_chunk context has a cell called 'uniform', but its celltype must be mixed, not structured" ) if not isinstance(hc.uniform, CoreCell): raise TypeError( "map_dict_chunk context must have an attribute 'uniform' that is a cell, not a {}" .format(type(hc.uniform))) ctx.uniform.connect(hc.uniform) con = ["..uniform"], ["ctx", subctx, "uniform"] pseudo_connections.append(con) if first: if isinstance(hci, StructuredCell): raise TypeError( "map_dict_chunk context has a cell called 'inp', but its celltype must be mixed, not structured" ) if not isinstance(hci, CoreCell): raise TypeError( "map_dict_chunk context must have an attribute 'inp' that is a cell, not a {}" .format(type(hci))) if hci.celltype != "mixed": raise TypeError( "map_dict_chunk context has a cell called 'inp', but its celltype must be mixed, not {}" .format(hci.celltype)) con = ["..inp"], ["ctx", subctx, "inp"] pseudo_connections.append(con) inputchunk = {k: inp[k] for k in inpkeys[pos:pos + chunksize]} #print("CHUNK", list(inputchunk.keys())) chunk_ctx = context() setattr(ctx, "chunk_%d" % (n + 1), chunk_ctx) chunk_ctx.inputchunk_deep = cell("mixed", hash_pattern={"*": "#"}) chunk_ctx.inputchunk_deep.set(inputchunk) chunk_ctx.inputchunk_deep2 = cell("plain") chunk_ctx.inputchunk_deep.connect(chunk_ctx.inputchunk_deep2) chunk_ctx.inputchunk_checksum = cell("checksum") chunk_ctx.inputchunk_deep2.connect(chunk_ctx.inputchunk_checksum) # chunk_ctx.inputchunk_checksum has the correct checksum, but there is no valid conversion # (because it is unsafe). # Use a macro to do it chunk_ctx.get_inputchunk = macro( {"inputchunk_checksum": { "io": "input", "celltype": "checksum" }}) get_inputchunk = lib_module_dict["helper"]["get_inputchunk_dict"] chunk_ctx.get_inputchunk.code.cell().set(get_inputchunk) chunk_ctx.inputchunk_checksum.connect( chunk_ctx.get_inputchunk.inputchunk_checksum) p = path(chunk_ctx.get_inputchunk.ctx).inputchunk chunk_ctx.inputchunk = cell("mixed", hash_pattern={"*": "#"}) p.connect(chunk_ctx.inputchunk) if first: if not hasattr(hc, "result"): raise TypeError( "map_dict_chunk context must have a cell called 'result'") if isinstance(hc.result, StructuredCell): raise TypeError( "map_dict_chunk context has a cell called 'result', but its celltype must be mixed, not structured" ) if not isinstance(hc.result, CoreCell): raise TypeError( "map_dict_chunk context must have an attribute 'result' that is a cell, not a {}" .format(type(hc.result))) chunk_ctx.inputchunk.connect(hci) chunk_ctx.result = cell("mixed", hash_pattern={"*": "#"}) chunk_ctx.result_deep = cell("checksum") hc.result.connect(chunk_ctx.result) chunk_ctx.result.connect(chunk_ctx.result_deep) chunk_ctx.result_deep.connect(ctx.sc.inchannels[(n + 1, )]) con = ["ctx", subctx, "result"], ["..result"] pseudo_connections.append(con) first = False ctx.subresults = cell("plain") ctx.sc.outchannels[()].connect(ctx.subresults) merge_subresults = lib_module_dict["helper"]["merge_subresults_chunk"] ctx.merge_subresults = transformer({ "subresults": { "io": "input", "celltype": "plain" }, "result": { "io": "output", "celltype": "plain" } }) ctx.merge_subresults.code.cell().set(merge_subresults) ctx.subresults.connect(ctx.merge_subresults.subresults) ctx.result_deep = cell("plain") ctx.merge_subresults.result.connect(ctx.result_deep) # ctx.result_deep has the correct checksum, but there is no valid conversion # (because it is unsafe). # Use a macro to do it ctx.get_result = macro( {"result_checksum": { "io": "input", "celltype": "checksum" }}) get_result = lib_module_dict["helper"]["get_result_dict"] ctx.get_result.code.cell().set(get_result) ctx.result_deep.connect(ctx.get_result.result_checksum) p = path(ctx.get_result.ctx).result ctx.result = cell("mixed", hash_pattern={"*": "#"}) p.connect(ctx.result) if not elision: ctx._pseudo_connections = pseudo_connections
def translate_compiled_transformer(node, root, namespace, inchannels, outchannels, *, has_meta_connection): from .translate import set_structured_cell_from_checksum from ..highlevel.Environment import Environment env0 = Environment(None) env0._load(node.get("environment")) env = env0._to_lowlevel() inchannels = [ic for ic in inchannels if ic[0] != "code"] main_module_inchannels = [ ("objects", ) + ic[1:] for ic in inchannels if ic[0] == "_main_module" ] + [("link_options", )] inchannels = [ic for ic in inchannels if ic[0] != "_main_module"] parent = get_path(root, node["path"][:-1], None, None) name = node["path"][-1] ctx = context(toplevel=False) setattr(parent, name, ctx) result_name = node["RESULT"] input_name = node["INPUT"] inputpins = [] all_inchannels = set(inchannels) pin_cells = {} for pin in list(node["pins"].keys()): pin_cell_name = pin + "_PIN" assert pin_cell_name not in all_inchannels assert pin_cell_name not in node["pins"] pin_cell = cell("mixed") cell_setattr(node, ctx, pin_cell_name, pin_cell) pin_cells[pin] = pin_cell mount = node.get("mount", {}) inp, inp_ctx = build_structured_cell( ctx, input_name, inchannels, [()], fingertip_no_remote=node.get("fingertip_no_remote", False), fingertip_no_recompute=node.get("fingertip_no_recompute", False), hash_pattern=node.get("hash_pattern"), return_context=True) setattr(ctx, input_name, inp) namespace[node["path"] + ("SCHEMA", ), "source"] = inp.schema, node if "input_schema" in mount: inp_ctx.schema.mount(**mount["input_schema"]) for inchannel in inchannels: path = node["path"] + inchannel namespace[path, "target"] = inp.inchannels[inchannel], node assert result_name not in node[ "pins"] #should have been checked by highlevel assert "executor_result_" not in node[ "pins"] #should have been checked by highlevel all_pins = {} inputpins = [] for pinname, pin in node["pins"].items(): p = {"io": "input"} p.update(pin) all_pins[pinname] = p inputpins.append(pinname) all_pins[result_name] = {"io": "output"} if node["SCHEMA"]: all_pins[node["SCHEMA"]] = {"io": "input", "celltype": "mixed"} # Compiler ctx.language = cell("str").set(node["language"]) ctx.main_module = build_structured_cell( ctx, "main_module", main_module_inchannels, [()], fingertip_no_remote=node.get("fingertip_no_remote", False), fingertip_no_recompute=node.get("fingertip_no_recompute", False), ) for ic in main_module_inchannels: if ic == "link_options": continue icpath = node["path"] + ("_main_module", ) + ic[1:] namespace[icpath, "target"] = ctx.main_module.inchannels[ic], node # Transformer itself ctf = ctx.tf = context() debug = node["debug"] or node.get("compiled_debug") if debug is None: debug = False _init_from_graph(ctf, debug) ctx.code = cell("text") ctx.code.set_file_extension(node["file_extension"]) if "code" in mount: ctx.code.mount(**mount["code"]) checksum = node.get("checksum", {}) if "code" in checksum: ctx.code._set_checksum(checksum["code"], initial=True) main_module_checksum = checksum.get( "main_module", 'd0a1b2af1705c1b8495b00145082ef7470384e62ac1c4d9b9cdbbe0476c28f8c' # {} ) set_structured_cell_from_checksum(ctx.main_module, {"auth": main_module_checksum}) inp_checksum = convert_checksum_dict(checksum, "input") set_structured_cell_from_checksum(inp, inp_checksum) namespace[node["path"] + ("code", ), "target"] = ctx.code, node namespace[node["path"] + ("code", ), "source"] = ctx.code, node if has_meta_connection: ctx.meta = cell("plain") ctx.meta.connect(ctf.executor.META) namespace[node["path"] + ("meta", ), "target"] = ctx.meta, node else: meta = node.get("meta") if meta is not None: ctf.executor.meta = meta result, result_ctx = build_structured_cell( ctx, result_name, [()], outchannels, fingertip_no_remote=node.get("fingertip_no_remote", False), fingertip_no_recompute=node.get("fingertip_no_recompute", False), return_context=True) namespace[node["path"] + ("RESULTSCHEMA", ), "source"] = result.schema, node if "result_schema" in mount: result_ctx.schema.mount(**mount["result_schema"]) setattr(ctx, result_name, result) assert not node["SCHEMA"] result_checksum = {} for k in checksum: if not k.startswith("result"): continue k2 = "value" if k == "result" else k[len("result_"):] result_checksum[k2] = checksum[k] set_structured_cell_from_checksum(result, result_checksum) ctx.pins = cell("plain").set(all_pins) ctx.inputpins = cell("plain").set(inputpins) c_inp = getattr(ctx, input_name + STRUC_ID) c_result = getattr(ctx, result_name + STRUC_ID) _finalize(ctx, ctf, inp, c_inp, result, c_result, input_name, result_name, inchannels, node) if "header" in mount: ctx.header.mount(**mount["header"]) namespace[node["path"] + ("header", ), "source"] = ctx.header, node if env is not None: ctf.executor.env = env namespace[node["path"], "target"] = inp, node namespace[node["path"], "source"] = result, node
import seamless from seamless.core import macro_mode_on from seamless.core import context, cell, transformer, macro, libcell, StructuredCell from seamless.core.structured_cell import BufferWrapper, StructuredCellState from seamless.core import library with macro_mode_on(): ctx = context(toplevel=True) def create(ctx, with_buffer, with_schema, inchannels): with macro_mode_on(): ctx.struc = context(name="struc", context=ctx) ctx.struc.storage = cell("text") ctx.struc.form = cell("json") ctx.struc.data = cell( "mixed", form_cell=ctx.struc.form, storage_cell=ctx.struc.storage, ) schema = None if with_schema: ctx.struc.schema = cell("json") schema = ctx.struc.schema bufferwrapper = None if with_buffer: ctx.struc.buffer_storage = cell("text") ctx.struc.buffer_form = cell("json") ctx.struc.buffer_data = cell( "mixed", form_cell=ctx.struc.buffer_form,
import seamless from seamless.core import macro_mode_on from seamless.core import context, cell, transformer, StructuredCell import numpy as np with macro_mode_on(): ctx = context(toplevel=True) ctx.mount("/tmp/mount-test", persistent=None) #directory remains, but empty ctx.inp_struc = context(name="inp_struc", context=ctx) ctx.inp_struc.storage = cell("text") ctx.inp_struc.form = cell("json") ctx.inp_struc.data = cell( "mixed", form_cell=ctx.inp_struc.form, storage_cell=ctx.inp_struc.storage, ) ctx.inp = StructuredCell("inp", ctx.inp_struc.data, storage=ctx.inp_struc.storage, form=ctx.inp_struc.form, schema=None, buffer=None, inchannels=None, outchannels=[()]) ctx.tf = transformer({ "inp": ("input", "copy", "silk"), "c": "output", }) ctx.inp.connect_outchannel((), ctx.tf.inp)
def translate_macro(node, root, namespace, inchannels, outchannels): from .translate import set_structured_cell_from_checksum parent = get_path(root, node["path"][:-1], None, None) name = node["path"][-1] ctx = context(toplevel=False) setattr(parent, name, ctx) param_name = node["PARAM"] all_inchannels = set(inchannels) param_inchannels = [] interchannels = [] pin_cells = {} pin_mpaths0 = {} for pinname in list(node["pins"].keys()): pin = node["pins"][pinname] if pin["io"] == "parameter": pin_cell_name = pinname + "_PARAM" assert pin_cell_name not in node["pins"] assert pin_cell_name not in all_inchannels pinname2 = as_tuple(pinname) interchannels.append(pinname2) if pinname2 in inchannels: param_inchannels.append(pinname2) elif pin["io"] in ("input", "output", "edit"): pin_cell_name = pinname else: raise ValueError((pin["io"], pinname)) pin_hash_pattern = pin.get("hash_pattern") celltype = pin.get("celltype", "mixed") if celltype == "mixed": pin_cell = cell(celltype, hash_pattern=pin_hash_pattern) else: pin_cell = cell(celltype) cell_setattr(node, ctx, pin_cell_name, pin_cell) pin_cells[pinname] = pin_cell if pin["io"] != "parameter": pin_mpaths0[pinname] = (pin["io"] in ("input", "edit")) mount = node.get("mount", {}) param = None if len(interchannels): param, param_ctx = build_structured_cell(ctx, param_name, param_inchannels, interchannels, return_context=True, fingertip_no_remote=False, fingertip_no_recompute=False, hash_pattern={"*": "#"}) setattr(ctx, param_name, param) namespace[node["path"] + ("SCHEMA", ), "source"] = param.schema, node if "param_schema" in mount: param_ctx.schema.mount(**mount["param_schema"]) param_pins = {} for pinname, pin in node["pins"].items(): if pin["io"] != "parameter": continue p = {"io": "input"} p.update(pin) param_pins[pinname] = p ctx.macro = macro(param_pins) if node.get("elision"): ctx.macro.allow_elision = True elision = {"macro": ctx.macro, "input_cells": {}, "output_cells": {}} for pinname in pin_mpaths0: is_input = pin_mpaths0[pinname] pin_mpath = getattr(core_path(ctx.macro.ctx), pinname) pin_cell = pin_cells[pinname] if is_input: if node["pins"][pinname]["io"] == "edit": pin_cell.bilink(pin_mpath) else: pin_cell.connect(pin_mpath) elision["input_cells"][pin_cell] = pin_mpath else: pin_mpath.connect(pin_cell) elision["output_cells"][pin_cell] = pin_mpath ctx._get_manager().set_elision(**elision) ctx.code = cell("macro") if "code" in mount: ctx.code.mount(**mount["code"]) ctx.code.connect(ctx.macro.code) checksum = node.get("checksum", {}) if "code" in checksum: ctx.code._set_checksum(checksum["code"], initial=True) if param is not None: param_checksum = convert_checksum_dict(checksum, "param") set_structured_cell_from_checksum(param, param_checksum) namespace[node["path"] + ("code", ), "target"] = ctx.code, node namespace[node["path"] + ("code", ), "source"] = ctx.code, node for pinname in node["pins"]: path = node["path"] + as_tuple(pinname) pin = node["pins"][pinname] if pin["io"] == "parameter": pinname2 = as_tuple(pinname) if pinname2 in inchannels: namespace[path, "target"] = param.inchannels[pinname], node target = getattr(ctx.macro, pinname) assert target is not None, pinname pin_cell = pin_cells[pinname] param.outchannels[pinname2].connect(pin_cell) pin_cell.connect(target) elif pin["io"] == "edit": namespace[path, "edit"] = pin_cells[pinname], node else: cmode = "target" if pin["io"] == "input" else "source" namespace[path, cmode] = pin_cells[pinname], node
def load(ctx): ctx.readme = libcell(".readme") def recreate(ctx, name0, with_buffer, with_schema, inchannels): name = "." + name0 ctx.struc = context(name="struc", context=ctx) ctx.struc.storage = libcell(name + ".storage") ctx.struc.form = libcell(name + ".form") ctx.struc.data = libmixedcell( name + ".data", form_cell=ctx.struc.form, storage_cell=ctx.struc.storage, ) schema = None if with_schema: ctx.struc.schema = libcell(name + ".schema") schema = ctx.struc.schema bufferwrapper = None if with_buffer: ctx.struc.buffer_storage = libcell(name + ".buffer_storage") ctx.struc.buffer_form = libcell(name + ".buffer_form") ctx.struc.buffer_data = libmixedcell( name + ".buffer_data", form_cell=ctx.struc.buffer_form, storage_cell=ctx.struc.buffer_storage, ) bufferwrapper = BufferWrapper(ctx.struc.buffer_data, ctx.struc.buffer_storage, ctx.struc.buffer_form) ctx.hub = StructuredCell( "hub", ctx.struc.data, storage=ctx.struc.storage, form=ctx.struc.form, schema=schema, buffer=bufferwrapper, inchannels=inchannels, outchannels=[], ) ctx.auth_json = context(name="auth_json", context=ctx) recreate(ctx.auth_json, "auth_json.struc", with_buffer=False, with_schema=False, inchannels=[]) ctx.auth = context(name="auth", context=ctx) recreate(ctx.auth, "auth.struc", with_buffer=False, with_schema=True, inchannels=[]) ctx.err = context(name="err", context=ctx) recreate(ctx.err, "err.struc", with_buffer=True, with_schema=True, inchannels=[]) ctx.nauth = context(name="nauth", context=ctx) try: recreate(ctx.nauth, "nauth.struc", with_buffer=False, with_schema=False, inchannels=[("z", )]) except: import traceback traceback.print_exc()
assert which in ("pi", "e") ctx.readme = libcell(".readme") ctx.loader = macro({}) if which == "pi": ctx.load = libcell(".pi.load") else: ctx.load = libcell(".e.load") ctx.load.connect(ctx.loader.code) compute = ctx.loader.gen_context ctx.iterations = cell() ctx.iterations.connect(compute.iterations) ctx.result = cell() compute.result.connect(ctx.result) lctx = context(toplevel=True) lctx.readme = cell("text").set("Compute pi or e iteratively") lctx.pi = context(context=lctx, name="pi") lctx.pi.code = cell("python").set(compute_pi) lctx.pi.load = cell("macro").set(load_pi) lctx.e = context(context=lctx, name="e") lctx.e.code = cell("python").set(compute_e) lctx.e.load = cell("macro").set(load_e) lctx.select = cell("macro").set(select) lctx.equilibrate() lib = library.build(lctx) library.register("compute", lib) select_params = { "which": ("ref", "text"), }
def translate_bash_transformer(node, root, namespace, inchannels, outchannels, lib_path00, is_lib): #TODO: simple translation, without a structured cell #TODO: there is a lot of common code with py transformer # Just to register the "bash_transformer" lib from seamless.lib.bash_transformer import bash_transformer as _ inchannels = [ic for ic in inchannels if ic[0] != "code"] parent = get_path(root, node["path"][:-1], None, None) name = node["path"][-1] lib_path0 = lib_path00 + "." + name if lib_path00 is not None else None ctx = context(context=parent, name=name) setattr(parent, name, ctx) result_name = node["RESULT"] input_name = node["INPUT"] if len(inchannels): lib_path0 = None #partial authority or no authority; no library update in either case for c in inchannels: assert (not len(c)) or c[0] != result_name #should have been checked by highlevel with_result = node["with_result"] buffered = node["buffered"] pins = node["pins"].copy() for extrapin in ("bashcode", "pins"): assert extrapin not in node["pins"], extrapin pins[extrapin] = { "transfer_mode": "ref", "access_mode": "default", "content_type": None, } ctx.pins = core_cell("json").set(list(pins.keys())) interchannels = [as_tuple(pin) for pin in pins] plain = node["plain"] input_state = node.get("stored_state_input", None) mount = node.get("mount", {}) if input_state is None: input_state = node.get("cached_state_input", None) inp, inp_ctx = build_structured_cell( ctx, input_name, True, plain, buffered, inchannels, interchannels, input_state, lib_path0, return_context=True ) setattr(ctx, input_name, inp) if "input_schema" in mount: inp_ctx.schema.mount(**mount["input_schema"]) for inchannel in inchannels: path = node["path"] + inchannel namespace[path, True] = inp.inchannels[inchannel], node assert result_name not in pins #should have been checked by highlevel all_pins = {} for pinname, pin in pins.items(): p = {"io": "input"} p.update(pin) all_pins[pinname] = p all_pins[result_name] = {"io": "output", "transfer_mode": "copy"} if node["SCHEMA"]: assert with_result all_pins[node["SCHEMA"]] = { "io": "input", "transfer_mode": "json", "access_mode": "json", "content_type": "json" } in_equilibrium = node.get("in_equilibrium", False) ctx.tf = transformer(all_pins, in_equilibrium=in_equilibrium) if node["debug"]: ctx.tf.debug = True if lib_path00 is not None: lib_path = lib_path00 + "." + name + ".code" ctx.code = libcell(lib_path) else: ctx.code = core_cell("json") if "code" in mount: ctx.code.mount(**mount["code"]) ctx.code._sovereign = True ctx.pins.connect(ctx.tf.pins) ctx.code.connect(ctx.tf.bashcode) code = node.get("code") if code is None: code = node.get("cached_code") ctx.code.set(code) temp = node.get("TEMP") if temp is None: temp = {} if "code" in temp: ctx.code.set(temp["code"]) with library.bind("bash_transformer"): ctx.executor_code = libcell(".executor_code") ctx.executor_code.connect(ctx.tf.code) inphandle = inp.handle for k,v in temp.items(): if k == "code": continue setattr(inphandle, k, v) namespace[node["path"] + ("code",), True] = ctx.code, node namespace[node["path"] + ("code",), False] = ctx.code, node for pin in list(node["pins"].keys()): target = getattr(ctx.tf, pin) inp.connect_outchannel( (pin,) , target ) if with_result: plain_result = node["plain_result"] result_state = node.get("cached_state_result", None) result, result_ctx = build_structured_cell( ctx, result_name, True, plain_result, False, [()], outchannels, result_state, lib_path0, return_context=True ) if "result_schema" in mount: result_ctx.schema.mount(**mount["result_schema"]) setattr(ctx, result_name, result) result_pin = getattr(ctx.tf, result_name) result.connect_inchannel(result_pin, ()) if node["SCHEMA"]: schema_pin = getattr(ctx.tf, node["SCHEMA"]) result.schema.connect(schema_pin) else: for c in outchannels: assert len(c) == 0 #should have been checked by highlevel result = getattr(ctx.tf, result_name) namespace[node["path"] + (result_name,), False] = result, node if not is_lib: #clean up cached state and in_equilibrium, unless a library context node.pop("cached_state_input", None) if not in_equilibrium: node.pop("cached_state_result", None) node.pop("in_equilibrium", None) namespace[node["path"], True] = inp, node namespace[node["path"], False] = result, node node.pop("TEMP", None)
def build_structured_cell(ctx, name, silk, plain, buffered, inchannels, outchannels, state, lib_path0, *, editchannels=[], mount=None, return_context=False): #print("build_structured_cell", name, lib_path) name2 = name + STRUC_ID c = context(name=name2, context=ctx) setattr(ctx, name2, c) if mount is not None: c.mount(**mount) lib_path = lib_path0 + "." + name2 if lib_path0 is not None else None sovereign = True if lib_path: path = lib_path + ".form" cc = libcell(path) else: cc = core_cell("json") cc._sovereign = sovereign c.form = cc if plain: if lib_path: path = lib_path + ".data" cc = libcell(path) else: cc = core_cell("json") cc._sovereign = sovereign c.data = cc storage = None else: if lib_path: path = lib_path + ".storage" storage = libcell(path) else: storage = core_cell("text") storage._sovereign = sovereign c.storage = storage if lib_path: path = lib_path + ".data" c.data = libmixedcell(path, form_cell=c.form, storage_cell=c.storage) else: c.data = core_cell("mixed", form_cell=c.form, storage_cell=c.storage) c.data._sovereign = sovereign if silk: if lib_path: path = lib_path + ".schema" schema = libcell(path) else: schema = core_cell("json") c.schema = schema else: schema = None if buffered: if lib_path: path = lib_path + ".buffer_form" cc = libcell(path) else: cc = core_cell("json") cc._sovereign = sovereign c.buffer_form = cc if plain: if lib_path: path = lib_path + ".buffer_data" cc = libcell(path) else: cc = core_cell("json") cc._sovereign = sovereign c.buffer_data = cc buffer_storage = None else: if lib_path: path = lib_path + ".buffer_storage" buffer_storage = libcell(path) else: buffer_storage = core_cell("text") buffer_storage._sovereign = sovereign c.buffer_storage = buffer_storage if lib_path: path = lib_path + ".buffer_data" c.buffer_data = libmixedcell( path, form_cell=c.buffer_form, storage_cell=c.buffer_storage, ) else: c.buffer_data = core_cell( "mixed", form_cell=c.buffer_form, storage_cell=c.buffer_storage, ) c.buffer_data._sovereign = sovereign bufferwrapper = BufferWrapper(c.buffer_data, buffer_storage, c.buffer_form) else: bufferwrapper = None sc = StructuredCell(name, c.data, storage=storage, form=c.form, schema=schema, buffer=bufferwrapper, inchannels=inchannels, outchannels=outchannels, state=state, editchannels=editchannels) if return_context: return sc, c else: return sc
import seamless from seamless.core import macro_mode_on from seamless.core import context, textcell, transformer, pytransformercell with macro_mode_on(): ctx = context(toplevel=True) ctx.cell1 = textcell().set(1) ctx.cell2 = textcell().set(2) ctx.result = textcell() ctx.tf = transformer({"a": "input", "b": "input", "c": "output"}) ctx.cell1.connect(ctx.tf.a) ctx.cell2.connect(ctx.tf.b) ctx.code = pytransformercell().set("c = float(a) + float(b)") ctx.code.connect(ctx.tf.code) ctx.tf.c.connect(ctx.result) ctx.result.mount("/tmp/mount-test/myresult", persistent=True) ctx.mount("/tmp/mount-test") ctx.sub = context(toplevel=False, context=ctx, name="sub") ctx.sub.mycell = textcell().set("This is my cell\nend") ctx.equilibrate() print(ctx.result.value) ctx.cell1.set(10) ctx.equilibrate() print(ctx.result.value) print(ctx.result.value) ctx.code.set("c = float(a) + float(b) + 1000") ctx.equilibrate() print(ctx.result.value) print(ctx.status())
def select(ctx, which): assert which in ("pi", "e") ctx.readme = libcell(".readme") ctx.loader = macro({}) if which == "pi": ctx.load = libcell(".pi.load") else: ctx.load = libcell(".e.load") ctx.load.connect(ctx.loader.code) compute = ctx.loader.ctx ctx.iterations = cell() ctx.iterations.connect(compute.iterations) ctx.result = cell() compute.result.connect(ctx.result) lctx = context(toplevel=True) lctx.readme = cell("text").set("Compute pi or e iteratively") lctx.pi = context() lctx.pi.code = cell("python").set(compute_pi) lctx.pi.load = cell("macro").set(load_pi) lctx.e = context() lctx.e.code = cell("python").set(compute_e) lctx.e.load = cell("macro").set(load_e) lctx.select = cell("macro").set(select) lctx.equilibrate() lib = library.build(lctx) library.register("compute", lib) select_params = { "which": ("ref", "text"),
import seamless from seamless.core import macro_mode_on from seamless.core import context, cell, transformer, StructuredCell from seamless.core.structured_cell import BufferWrapper import numpy as np with macro_mode_on(): ctx = context(toplevel=True) ctx.hub_struc = context(name="hub_struc", context=ctx) ctx.hub_struc.storage = cell("text") ctx.hub_struc.form = cell("json") ctx.hub_struc.data = cell( "mixed", form_cell=ctx.hub_struc.form, storage_cell=ctx.hub_struc.storage, ) ctx.hub_struc.schema = cell("json") ctx.hub_struc.buffer_storage = cell("text") ctx.hub_struc.buffer_form = cell("json") ctx.hub_struc.buffer_data = cell( "mixed", form_cell=ctx.hub_struc.buffer_form, storage_cell=ctx.hub_struc.buffer_storage, ) bufferwrapper = BufferWrapper(ctx.hub_struc.buffer_data, ctx.hub_struc.buffer_storage, ctx.hub_struc.buffer_form) ctx.hub = StructuredCell("hub", ctx.hub_struc.data, storage=ctx.hub_struc.storage, form=ctx.hub_struc.form,
from math import pi from seamless.core import context, cell ctx0 = context(toplevel=True) ctx0.cell = cell("mixed").set(pi) ctx0.cell2 = cell("python").set("lambda a: 2 * a") ctx0.cell.set({"a": pi}) import json graph = json.load(open("twopi.seamless")) from seamless.highlevel import load_graph ctx = load_graph(graph, cache_ctx=ctx0) ctx.translate(force=True) print(ctx.pi.checksum) print(ctx.pi.value) print("equilibrate") ctx.equilibrate() print(ctx.twopi.value) graph2 = json.load(open("twopi-result.seamless")) ctx2 = load_graph(graph2, cache_ctx=ctx0) print(ctx2.pi.checksum) print(ctx2.pi.value) print("equilibrate") ctx2.equilibrate() print(ctx2.twopi.value)
def translate_bash_transformer(node, root, namespace, inchannels, outchannels, lib_path00, is_lib): #TODO: simple translation, without a structured cell #TODO: there is a lot of common code with py transformer assert not "code" in node ### node["code"] is an outdated attribute # Just to register the "bash_transformer" lib from seamless.core.macro_mode import get_macro_mode, curr_macro from seamless.lib.bash_transformer import bash_transformer as _ inchannels = [ic for ic in inchannels if ic[0] != "code"] parent = get_path(root, node["path"][:-1], None, None) name = node["path"][-1] lib_path0 = lib_path00 + "." + name if lib_path00 is not None else None ctx = context(toplevel=False) setattr(parent, name, ctx) result_name = node["RESULT"] input_name = node["INPUT"] for c in inchannels: assert (not len(c)) or c[0] != result_name #should have been checked by highlevel with_result = node["with_result"] buffered = node["buffered"] pins = node["pins"].copy() for extrapin in ("bashcode", "pins"): assert extrapin not in node["pins"], extrapin pins[extrapin] = { "transfer_mode": "ref", "access_mode": "plain", "content_type": None, } ctx.pins = core_cell("plain").set(list(pins.keys())) interchannels = [as_tuple(pin) for pin in pins] plain = node["plain"] mount = node.get("mount", {}) inp, inp_ctx = build_structured_cell( ctx, input_name, True, plain, buffered, inchannels, interchannels, lib_path0, return_context=True ) setattr(ctx, input_name, inp) if "input_schema" in mount: inp_ctx.schema.mount(**mount["input_schema"]) for inchannel in inchannels: path = node["path"] + inchannel namespace[path, True] = inp.inchannels[inchannel], node assert result_name not in pins #should have been checked by highlevel all_pins = {} for pinname, pin in pins.items(): p = {"io": "input"} p.update(pin) all_pins[pinname] = p all_pins[result_name] = {"io": "output", "transfer_mode": "copy"} if node["SCHEMA"]: assert with_result all_pins[node["SCHEMA"]] = { "io": "input", "transfer_mode": "json", "access_mode": "json", "content_type": "json" } ctx.tf = transformer(all_pins) if node["debug"]: ctx.tf.debug = True if lib_path00 is not None: lib_path = lib_path00 + "." + name + ".code" ctx.code = libcell(lib_path) else: ctx.code = core_cell("text") if "code" in mount: ctx.code.mount(**mount["code"]) ctx.code._sovereign = True ctx.pins.connect(ctx.tf.pins) ctx.code.connect(ctx.tf.bashcode) checksum = node.get("checksum", {}) if "code" in checksum: ctx.code.set_checksum(checksum["code"]) if "input" in checksum: inp.set_checksum(checksum["input"]) with library.bind("bash_transformer"): ctx.executor_code = libcell(".executor_code") ctx.executor_code.connect(ctx.tf.code) namespace[node["path"] + ("code",), True] = ctx.code, node namespace[node["path"] + ("code",), False] = ctx.code, node for pin in list(node["pins"].keys()): target = getattr(ctx.tf, pin) inp.outchannels[(pin,)].connect(target) if with_result: plain_result = node["plain_result"] result, result_ctx = build_structured_cell( ctx, result_name, True, plain_result, False, [()], outchannels, lib_path0, return_context=True ) if "result_schema" in mount: result_ctx.schema.mount(**mount["result_schema"]) setattr(ctx, result_name, result) result_pin = getattr(ctx.tf, result_name) result_pin.connect(result.inchannels[()]) if node["SCHEMA"]: schema_pin = getattr(ctx.tf, node["SCHEMA"]) result.schema.connect(schema_pin) if "result" in checksum: result.set_checksum(checksum["result"]) if "schema" in checksum: result.schema.set_checksum(checksum["schema"]) else: for c in outchannels: assert len(c) == 0 #should have been checked by highlevel result = getattr(ctx.tf, result_name) namespace[node["path"] + (result_name,), False] = result, node namespace[node["path"], True] = inp, node namespace[node["path"], False] = result, node
import seamless from seamless.core import macro_mode_on from seamless.core import context,textcell, cell, transformer, pytransformercell with macro_mode_on(): ctx = context(toplevel=True) ctx.cell1 = cell().set(1) ctx.cell2 = cell().set(2) ctx.result = cell() ctx.tf = transformer({ "a": "input", "b": "input", "c": "output" }) ctx.cell1.connect(ctx.tf.a) ctx.cell2.connect(ctx.tf.b) ctx.code = pytransformercell().set("c = a + b") ctx.code.connect(ctx.tf.code) ctx.tf.c.connect(ctx.result) ctx.result.mount("/tmp/mount-test/myresult", persistent=True) ctx.mount("/tmp/mount-test") ctx.sub = context(toplevel=False) ctx.sub.mycell = textcell().set("This is my cell\nend") ctx.equilibrate() print(ctx.result.value) ctx.cell1.set(10) ctx.equilibrate() print(ctx.result.value) print(ctx.result.value) ctx.code.set("c = float(a) + float(b) + 1000")
def translate_bash_transformer(node, root, namespace, inchannels, outchannels, *, has_meta_connection): from .translate import set_structured_cell_from_checksum from ..highlevel.Environment import Environment from ..core.environment import (validate_capabilities, validate_conda_environment, validate_docker) env0 = Environment(None) env0._load(node.get("environment")) env = env0._to_lowlevel() is_docker_transformer = False if env is not None and env.get("docker") is not None: ok1 = validate_capabilities(env)[0] ok2 = validate_conda_environment(env)[0] ok3 = validate_docker(env)[0] if not (ok1 or ok2 or ok3): is_docker_transformer = True if is_docker_transformer: from .translate_bashdocker_transformer import translate_bashdocker_transformer docker = env.pop("docker") docker_image = docker["name"] docker_options = docker["options"] # TODO: pass on version and checksum as well? if "powers" not in env: env["powers"] = [] env["powers"].append("docker") return translate_bashdocker_transformer( node, root, namespace, inchannels, outchannels, has_meta_connection=has_meta_connection, env=env, docker_image=docker_image, docker_options=docker_options) inchannels = [ic for ic in inchannels if ic[0] != "code"] parent = get_path(root, node["path"][:-1], None, None) name = node["path"][-1] ctx = context(toplevel=False) setattr(parent, name, ctx) result_name = node["RESULT"] input_name = node["INPUT"] result_cell_name = result_name + "_CELL" forbidden = [result_name, result_cell_name, "bashcode", "pins_"] pin_intermediate = {} for pin in node["pins"].keys(): pin_intermediate[pin] = input_name + "_PIN_" + pin forbidden.append(pin_intermediate[pin]) for c in inchannels: assert ( not len(c) ) or c[0] not in forbidden #should have been checked by highlevel pins = node["pins"].copy() pins["bashcode"] = {"celltype": "text"} pins["pins_"] = {"celltype": "plain"} ctx.pins = cell("plain").set(list(pins.keys())) interchannels = [as_tuple(pin) for pin in pins] mount = node.get("mount", {}) inp, inp_ctx = build_structured_cell( ctx, input_name, inchannels, interchannels, fingertip_no_remote=node.get("fingertip_no_remote", False), fingertip_no_recompute=node.get("fingertip_no_recompute", False), hash_pattern=node.get("hash_pattern"), return_context=True) setattr(ctx, input_name, inp) namespace[node["path"] + ("SCHEMA", ), "source"] = inp.schema, node if "input_schema" in mount: inp_ctx.schema.mount(**mount["input_schema"]) for inchannel in inchannels: path = node["path"] + inchannel namespace[path, "target"] = inp.inchannels[inchannel], node assert result_name not in pins #should have been checked by highlevel all_pins = {} for pinname, pin in pins.items(): p = {"io": "input"} p.update(pin) all_pins[pinname] = p all_pins[result_name] = {"io": "output"} if node["SCHEMA"]: raise NotImplementedError all_pins[node["SCHEMA"]] = { "io": "input", "transfer_mode": "json", "access_mode": "json", "content_type": "json" } ctx.tf = transformer(all_pins) if node["debug"]: ctx.tf.debug = True ctx.code = cell("text") if "code" in mount: ctx.code.mount(**mount["code"]) ctx.pins.connect(ctx.tf.pins_) ctx.code.connect(ctx.tf.bashcode) checksum = node.get("checksum", {}) if "code" in checksum: ctx.code._set_checksum(checksum["code"], initial=True) inp_checksum = convert_checksum_dict(checksum, "input") set_structured_cell_from_checksum(inp, inp_checksum) ctx.executor_code = sctx.executor_code.cell() ctx.executor_code.connect(ctx.tf.code) namespace[node["path"] + ("code", ), "target"] = ctx.code, node namespace[node["path"] + ("code", ), "source"] = ctx.code, node for pinname, pin in node["pins"].items(): target = ctx.tf.get_pin(pinname) celltype = pin.get("celltype", "mixed") if celltype == "code": celltype = "text" intermediate_cell = cell(celltype) cell_setattr(node, ctx, pin_intermediate[pinname], intermediate_cell) inp.outchannels[(pinname, )].connect(intermediate_cell) intermediate_cell.connect(target) meta = deepcopy(node.get("meta", {})) meta["transformer_type"] = "bash" ctx.tf.meta = meta if has_meta_connection: ctx.meta = cell("plain") ctx.meta.connect(ctx.tf.META) namespace[node["path"] + ("meta", ), "target"] = ctx.meta, node result, result_ctx = build_structured_cell( ctx, result_name, [()], outchannels, fingertip_no_remote=node.get("fingertip_no_remote", False), fingertip_no_recompute=node.get("fingertip_no_recompute", False), return_context=True) namespace[node["path"] + ("RESULTSCHEMA", ), "source"] = result.schema, node if "result_schema" in mount: result_ctx.schema.mount(**mount["result_schema"]) setattr(ctx, result_name, result) result_pin = ctx.tf.get_pin(result_name) result_cell = cell("mixed") cell_setattr(node, ctx, result_cell_name, result_cell) result_pin.connect(result_cell) result_cell.connect(result.inchannels[()]) if node["SCHEMA"]: schema_pin = ctx.tf.get_pin(node["SCHEMA"]) result.schema.connect(schema_pin) result_checksum = {} for k in checksum: if not k.startswith("result"): continue k2 = "value" if k == "result" else k[len("result_"):] result_checksum[k2] = checksum[k] set_structured_cell_from_checksum(result, result_checksum) if env is not None: ctx.tf.env = env namespace[node["path"], "target"] = inp, node namespace[node["path"], "source"] = result, node
def build_structured_cell( ctx, name, silk, plain, buffered, inchannels, outchannels, lib_path0, *, editchannels=[], mount=None, return_context=False ): #print("build_structured_cell", name, lib_path) name2 = name + STRUC_ID c = context(toplevel=False) setattr(ctx, name2, c) if mount is not None: c.mount(**mount) lib_path = lib_path0 + "." + name2 if lib_path0 is not None else None sovereign = True if plain: if lib_path: path = lib_path + ".data" cc = libcell(path) else: cc = core_cell("mixed") cc._sovereign = sovereign c.data = cc storage = None else: if lib_path: path = lib_path + ".data" c.data = libcell(path, "mixed") else: c.data = core_cell("mixed") c.data._sovereign = sovereign if silk: if lib_path: path = lib_path + ".schema" schema = libcell(path) else: schema = core_cell("plain") c.schema = schema else: schema = None if buffered: if plain: if lib_path: path = lib_path + ".buffer" cc = libcell(path) else: cc = core_cell("mixed") cc._sovereign = sovereign c.buffer = cc storage = None else: if lib_path: path = lib_path + ".buffer" c.buffer = libcell(path, "mixed") else: c.buffer = core_cell("mixed") c.buffer._sovereign = sovereign buffer = c.buffer else: buffer = None sc = StructuredCell( name, c.data, schema=schema, buffer=buffer, plain=plain, inchannels=inchannels, outchannels=outchannels, editchannels=editchannels ) if return_context: return sc, c else: return sc
raise NotImplementedError # Silk access is not working; the modified code below kludges around it import seamless from seamless.core import macro_mode_on from seamless.core import context, cell, transformer, StructuredCell import numpy as np with macro_mode_on(): ctx = context(toplevel=True) ctx.mount("/tmp/mount-test", persistent=None) #directory remains, but empty ctx.inp_struc = context(toplevel=False) ctx.inp_struc.data = cell("mixed") ctx.inp = StructuredCell( "inp", ctx.inp_struc.data, plain = False, schema = None, buffer = None, inchannels = None, outchannels = [()] ) ctx.tf = transformer({ ###"inp": ("input", "copy", "silk"), "inp": ("input", "copy", "mixed"), "c": "output", }) ctx.inp.outchannels[()].connect(ctx.tf.inp) ###ctx.tf.code.cell().set("c = inp.a * inp.dat + inp.b") ctx.tf.code.cell().set("c = inp['a'] * inp['dat'] + inp['b']")
def create(ctx, mount, state=None): with macro_mode_on(): ctx.hub_struc = context(name="hub_struc", context=ctx) ctx.hub_struc.storage = cell("text") ctx.hub_struc.form = cell("json") ctx.hub_struc.data = cell( "mixed", form_cell=ctx.hub_struc.form, storage_cell=ctx.hub_struc.storage, ) ctx.hub_struc.schema = cell("json") ctx.hub_struc.buffer_storage = cell("text") ctx.hub_struc.buffer_form = cell("json") ctx.hub_struc.buffer_data = cell( "mixed", form_cell=ctx.hub_struc.buffer_form, storage_cell=ctx.hub_struc.buffer_storage, ) bufferwrapper = BufferWrapper(ctx.hub_struc.buffer_data, ctx.hub_struc.buffer_storage, ctx.hub_struc.buffer_form) ctx.hub = StructuredCell( "hub", ctx.hub_struc.data, storage=ctx.hub_struc.storage, form=ctx.hub_struc.form, schema=ctx.hub_struc.schema, buffer=bufferwrapper, inchannels=[("a", "factor1"), ("a", "factor2"), ("b", )], outchannels=[("a", ), ("b", ), ("c", )], state=state, ) ctx.code = cell("transformer").set( "d = a.factor1 * b + a.factor2 * c + a.constant") ctx.mixer = transformer({ "a": ("input", "ref", "silk"), "b": ("input", "ref", "object"), "c": ("input", "ref", "object"), "d": ("output", "ref", "json"), }) ctx.a_factor1 = cell("json") ctx.a_factor2 = cell("json") ctx.b = cell("json") ctx.code.connect(ctx.mixer.code) ctx.hub.connect_inchannel(ctx.a_factor1, ("a", "factor1")) ctx.hub.connect_inchannel(ctx.a_factor2, ("a", "factor2")) ctx.hub.connect_inchannel(ctx.b, ("b", )) ctx.hub.connect_outchannel(("a", ), ctx.mixer.a) ctx.hub.connect_outchannel(("b", ), ctx.mixer.b) ctx.hub.connect_outchannel(("c", ), ctx.mixer.c) ctx.result_struc = context(name="result_struc", context=ctx) ctx.result_struc.storage = cell("text") ctx.result_struc.form = cell("json") ctx.result_struc.data = cell( "mixed", form_cell=ctx.result_struc.form, storage_cell=ctx.result_struc.storage, ) ctx.result_struc.schema = cell("json") ctx.result_struc.buffer_storage = cell("text") ctx.result_struc.buffer_form = cell("json") ctx.result_struc.buffer_data = cell( "mixed", form_cell=ctx.result_struc.buffer_form, storage_cell=ctx.result_struc.buffer_storage, ) bufferwrapper = BufferWrapper(ctx.result_struc.buffer_data, ctx.result_struc.buffer_storage, ctx.result_struc.buffer_form) ctx.result = StructuredCell("result", ctx.result_struc.data, storage=ctx.result_struc.storage, form=ctx.result_struc.form, schema=ctx.result_struc.schema, buffer=bufferwrapper, inchannels=[()], outchannels=[]) ctx.result.connect_inchannel(ctx.mixer.d, ()) if mount: ctx.mount("/tmp/mount-test")
import seamless from seamless.core import macro_mode_on from seamless.core import context, cell, transformer with macro_mode_on(): topctx = context(toplevel=True) ctx = topctx.sub = context(toplevel=False) assert topctx.sub is ctx ctx.cell1 = cell().set(1) ctx.cell2 = cell().set(2) ctx.result = cell() ctx.tf = transformer({ "a": "input", "b": "input", "c": "output" }) ctx.cell1.connect(ctx.tf.a) ctx.cell2.connect(ctx.tf.b) ctx.code = cell("transformer").set("c = a + b") ctx.code.connect(ctx.tf.code) ctx.tf.c.connect(ctx.result) topctx.compute() print(ctx.result.value) ctx.cell1.set(10) topctx.compute() print(ctx.result.value) ctx.code.set("c = a + b + 1000") topctx.compute() print(ctx.result.value) print(ctx.status)
""" Final test for compiled transformers, using stdlib.compiled_transformer """ from seamless.core import context, cell, transformer, macro, libcell, macro_mode_on from seamless.core import StructuredCell from seamless.core import library from copy import deepcopy # 1: set up example data with macro_mode_on(): ctx = context(toplevel=True) # 1a. Setup of StructuredCells ctx.inp_struc = context(name="inp_struc",context=ctx) ctx.inp_struc.storage = cell("text") ctx.inp_struc.form = cell("json") ctx.inp_struc.data = cell("mixed", form_cell = ctx.inp_struc.form, storage_cell = ctx.inp_struc.storage, ) ctx.inp_struc.schema = cell("json") ctx.inp = StructuredCell( "inp", ctx.inp_struc.data, storage = ctx.inp_struc.storage, form = ctx.inp_struc.form, schema = ctx.inp_struc.schema, buffer = None, inchannels = [], outchannels = [()]
def translate(graph, ctx, environment): from ..core.macro_mode import curr_macro if curr_macro() is None: print_info("*" * 30 + "TRANSLATE" + "*" * 30) #import traceback; stack = traceback.extract_stack(); print("TRANSLATE:"); print("".join(traceback.format_list(stack[:3]))) nodes, connections = graph["nodes"], graph["connections"] contexts = {con["path"]: con for con in nodes if con["type"] == "context"} for path in sorted(contexts.keys(), key=lambda k: len(k)): parent = get_path(ctx, path[:-1], None, is_target=False) name = path[-1] c = context() setattr(parent, name, c) # No need to add it to namespace, as long as the low-level graph structure is imitated connection_paths = [(con["source"], con["target"]) for con in connections if con["type"] == "connection"] namespace = {} for node in nodes: t = node["type"] if t in ("context", "link"): continue path = node["path"] for node in nodes: t = node["type"] if t in ("context", "link"): continue path = node["path"] if t == "transformer": inchannels, outchannels = find_channels(node["path"], connection_paths) try: inchannels.remove(("meta", )) has_meta_connection = True except ValueError: has_meta_connection = False language = node["language"] if node["compiled"]: from .translate_compiled_transformer import translate_compiled_transformer translate_compiled_transformer( node, ctx, namespace, inchannels, outchannels, has_meta_connection=has_meta_connection) elif language == "bash": translate_bash_transformer( node, ctx, namespace, inchannels, outchannels, has_meta_connection=has_meta_connection) else: ipy_template = None py_bridge = None if language not in ("python", "ipython"): ok = False if environment is not None: try: ipy_template = environment.get_ipy_template( language) ok = True except KeyError: pass try: py_bridge = environment.get_py_bridge(language) ok = True except KeyError: pass if ipy_template is not None and py_bridge is not None: msg = "Language '{}' has an IPython template AND a Python bridge" raise ValueError(msg.format(language)) if not ok: raise NotImplementedError(language) translate_py_transformer( node, ctx, namespace, inchannels, outchannels, ipy_template=ipy_template, py_bridge=py_bridge, has_meta_connection=has_meta_connection) elif t == "macro": if node["language"] != "python": raise NotImplementedError(node["language"]) inchannels, outchannels = find_channels(node["path"], connection_paths) translate_macro(node, ctx, namespace, inchannels, outchannels) elif t == "cell": inchannels, outchannels = find_channels(path, connection_paths) translate_cell(node, ctx, namespace, inchannels, outchannels) elif t == "module": inchannels, outchannels = find_channels(path, connection_paths) translate_module(node, ctx, namespace, inchannels, outchannels) elif t == "libinstance": msg = "Libinstance '%s' was not removed during pre-translation" raise TypeError(msg % str(path)) else: raise TypeError(t) node.pop("UNTRANSLATED", None) node.pop("UNSHARE", None) namespace2 = OrderedDict() for k in sorted(namespace.keys(), key=lambda k: -len(k)): namespace2[k] = namespace[k] for connection in connections: if connection["type"] == "connection": translate_connection(connection, namespace2, ctx) elif connection["type"] == "link": translate_link(connection, namespace2, ctx) elif connection["type"] == "virtual": pass else: raise TypeError(connection["type"])
icellname = "input_param_" + param icell = cell("mixed") setattr(c, icellname, icell) outchannel = c.input.outchannels[(param, )] pin = getattr(c.tf, param) outchannel.connect(icell) icell.connect(pin) c.example_data = cell("mixed") c.example_buffer = cell("mixed") c.example = StructuredCell(data=c.example_data, buffer=c.example_buffer, schema=c.input_schema) with macro_mode_on(): ctx = context(toplevel=True) tf_names = [("tf1", ), ("tf2", ), ("tf3", ), ("tf4", )] channel_names = tf_names # TODO (long term): try numeric path ctx.params_struc = context() ctx.params_struc.data = cell("mixed") ctx.params_struc.auth = cell("mixed") ctx.params_struc.buffer = cell("mixed") ctx.params_struc.schema = cell("plain") ctx.params_struc.example_buffer = cell("mixed") ctx.params_struc.example_data = cell("mixed") ctx.params = StructuredCell( ctx.params_struc.data, auth=ctx.params_struc.auth, buffer=ctx.params_struc.buffer, schema=ctx.params_struc.schema,
"language": "python", "transformer": True, "checksum": 'b0480c66eb4dbb31f5311e09e89b9414c880360842b9d5ef7b6621fc31a5ab99', }], "connections": [{ "source": ("pi",), "target": ("doubleit", "a"), }, { "source": ("doubleit",), "target": ("twopi",), }, { "source": ("code",), "target": ("doubleit", "code"), }], } ctx0 = context(toplevel=True) ctx0.pi = cell("mixed").set(math.pi) assert ctx0.pi.checksum == '9809b7dfcfe29dd194c71c7d2da94af3aeef98f079eeff8e1d9e5099acef737c' ctx0.code = cell("python").set("result = a * 2") assert ctx0.code.checksum == 'b0480c66eb4dbb31f5311e09e89b9414c880360842b9d5ef7b6621fc31a5ab99' with macro_mode_on(): ctx = context(toplevel=True, manager=ctx0._get_manager()) translate(graph, ctx, [], False) ctx.equilibrate() print(ctx.twopi.value)
import seamless from seamless.core import macro_mode_on from seamless.core import context, cell, transformer, StructuredCell from seamless.core.structured_cell import BufferWrapper import numpy as np with macro_mode_on(): ctx = context(toplevel=True) ctx.hub_struc = context(name="hub_struc",context=ctx) ctx.hub_struc.storage = cell("text") ctx.hub_struc.form = cell("json") ctx.hub_struc.data = cell("mixed", form_cell = ctx.hub_struc.form, storage_cell = ctx.hub_struc.storage, ) ctx.hub_struc.schema = cell("json") ctx.hub_struc.buffer_storage = cell("text") ctx.hub_struc.buffer_form = cell("json") ctx.hub_struc.buffer_data = cell("mixed", form_cell = ctx.hub_struc.buffer_form, storage_cell = ctx.hub_struc.buffer_storage, ) bufferwrapper = BufferWrapper( ctx.hub_struc.buffer_data, ctx.hub_struc.buffer_storage, ctx.hub_struc.buffer_form ) ctx.hub = StructuredCell( "hub", ctx.hub_struc.data, storage = ctx.hub_struc.storage,
def translate_py_transformer(node, root, namespace, inchannels, outchannels, lib_path00, is_lib): #TODO: simple translation, without a structured cell assert not "code" in node ### node["code"] is an outdated attribute inchannels = [ic for ic in inchannels if ic[0] != "code"] parent = get_path(root, node["path"][:-1], None, None) name = node["path"][-1] lib_path0 = lib_path00 + "." + name if lib_path00 is not None else None ctx = context(toplevel=False) setattr(parent, name, ctx) result_name = node["RESULT"] if node["language"] == "ipython": assert result_name == "result" input_name = node["INPUT"] for c in inchannels: assert (not len(c)) or c[0] != result_name #should have been checked by highlevel with_result = node["with_result"] buffered = node["buffered"] interchannels = [as_tuple(pin) for pin in node["pins"]] plain = node["plain"] mount = node.get("mount", {}) silk = (buffered or not plain) inp, inp_ctx = build_structured_cell( ctx, input_name, silk, plain, buffered, inchannels, interchannels, lib_path0, return_context=True ) setattr(ctx, input_name, inp) if "input_schema" in mount: inp_ctx.schema.mount(**mount["input_schema"]) for inchannel in inchannels: path = node["path"] + inchannel namespace[path, True] = inp.inchannels[inchannel], node assert result_name not in node["pins"] #should have been checked by highlevel all_pins = {} for pinname, pin in node["pins"].items(): p = {"io": "input"} p.update(pin) all_pins[pinname] = p all_pins[result_name] = {"io": "output", "transfer_mode": "copy"} if node["SCHEMA"]: assert with_result all_pins[node["SCHEMA"]] = { "io": "input", "transfer_mode": "json", "access_mode": "json", "content_type": "json" } ctx.tf = transformer(all_pins) if node["debug"]: ctx.tf.debug = True if lib_path00 is not None: lib_path = lib_path00 + "." + name + ".code" ctx.code = libcell(lib_path) else: if node["language"] == "ipython": ctx.code = core_cell("ipython") else: ctx.code = core_cell("transformer") if "code" in mount: ctx.code.mount(**mount["code"]) ctx.code._sovereign = True ctx.code.connect(ctx.tf.code) checksum = node.get("checksum", {}) if "code" in checksum: ctx.code.set_checksum(checksum["code"]) if "input" in checksum: inp.set_checksum(checksum["input"]) namespace[node["path"] + ("code",), True] = ctx.code, node namespace[node["path"] + ("code",), False] = ctx.code, node for pin in list(node["pins"].keys()): target = getattr(ctx.tf, pin) inp.outchannels[(pin,)].connect(target) if with_result: plain_result = node["plain_result"] result, result_ctx = build_structured_cell( ctx, result_name, True, plain_result, False, [()], outchannels, lib_path0, return_context=True ) if "result_schema" in mount: result_ctx.schema.mount(**mount["result_schema"]) setattr(ctx, result_name, result) result_pin = getattr(ctx.tf, result_name) result_pin.connect(result.inchannels[()]) if node["SCHEMA"]: schema_pin = getattr(ctx.tf, node["SCHEMA"]) result.schema.connect(schema_pin) if "result" in checksum: result.set_checksum(checksum["result"]) if "schema" in checksum: result.schema.set_checksum(checksum["schema"]) else: for c in outchannels: assert len(c) == 0 #should have been checked by highlevel result = getattr(ctx.tf, result_name) namespace[node["path"] + (result_name,), False] = result, node namespace[node["path"], True] = inp, node namespace[node["path"], False] = result, node
def translate_py_reactor(node, root, namespace, inchannels, outchannels, editchannels, lib_path00, is_lib): #TODO: simple-mode translation, without a structured cell skip_channels = ("code_start", "code_update", "code_stop") inchannels = [ic for ic in inchannels if ic[0] not in skip_channels] editchannels = [ec for ec in editchannels if ec[0] not in skip_channels] parent = get_path(root, node["path"][:-1], None, None) name = node["path"][-1] lib_path0 = lib_path00 + "." + name if lib_path00 is not None else None ctx = context(context=parent, name=name) setattr(parent, name, ctx) io_name = node["IO"] if len(inchannels): lib_path0 = None #partial authority or no authority; no library update in either case buffered = node["buffered"] interchannels_in = [ as_tuple(p) for p, pin in node["pins"].items() if pin["io"] == "output" ] interchannels_out = [ as_tuple(p) for p, pin in node["pins"].items() if pin["io"] == "input" ] interchannels_edit = [ as_tuple(p) for p, pin in node["pins"].items() if pin["io"] == "edit" ] all_inchannels = interchannels_in + inchannels #highlevel must check that there are no duplicates all_outchannels = interchannels_out + [ p for p in outchannels if p not in interchannels_out ] all_editchannels = interchannels_edit + [ p for p in editchannels if p not in interchannels_edit ] plain = node["plain"] io_state = node.get("stored_state_io", None) if io_state is None: io_state = node.get("cached_state_io", None) io = build_structured_cell(ctx, io_name, True, plain, buffered, all_inchannels, all_outchannels, io_state, lib_path0, editchannels=all_editchannels) setattr(ctx, io_name, io) for inchannel in inchannels: path = node["path"] + inchannel namespace[path, True] = io.inchannels[inchannel], node for outchannel in outchannels: path = node["path"] + outchannel namespace[path, False] = io.outchannels[outchannel], node for channel in editchannels: path = node["path"] + channel namespace[path, True] = io.editchannels[channel], node namespace[path, False] = io.editchannels[channel], node ctx.rc = reactor(node["pins"]) for attr in ("code_start", "code_stop", "code_update"): if lib_path00 is not None: lib_path = lib_path00 + "." + name + "." + attr c = libcell(lib_path) setattr(ctx, attr, c) else: c = core_cell(node["language"]) c._sovereign = True setattr(ctx, attr, c) if "mount" in node and attr in node["mount"]: c.mount(**node["mount"][attr]) c.connect(getattr(ctx.rc, attr)) code = node.get(attr) if code is None: code = node.get("cached_" + attr) try_set(c, code) namespace[node["path"] + (attr, ), True] = c, node namespace[node["path"] + (attr, ), False] = c, node for pinname, pin in node["pins"].items(): target = getattr(ctx.rc, pinname) iomode = pin["io"] if iomode == "input": io.connect_outchannel((pinname, ), target) elif iomode == "edit": io.connect_editchannel((pinname, ), target) elif iomode == "output": io.connect_inchannel(target, (pinname, )) temp = node.get("TEMP") if temp is None: temp = {} for attr in ("code_start", "code_stop", "code_update"): if attr in temp: try_set(getattr(ctx, attr), temp[attr]) iohandle = io.handle for k, v in temp.items(): if k in ("code_start", "code_stop", "code_update"): continue setattr(iohandle, k, v) if not is_lib: #clean up cached state and in_equilibrium, unless a library context node.pop("cached_state_io", None) namespace[node["path"], True] = io, node namespace[node["path"], False] = io, node node.pop("TEMP", None)
import seamless from seamless.core import macro_mode_on from seamless.core import context, cell, transformer, pytransformercell with macro_mode_on(): ctx = context(toplevel=True) ctx.cell1 = cell().set(1) ctx.cell2 = cell().set(2) ctx.result = cell() ctx.tf = transformer({"a": "input", "b": "input", "c": "output"}) ctx.cell1.connect(ctx.tf.a) ctx.cell2.connect(ctx.tf.b) ctx.code = pytransformercell().set("c = a + b") ctx.code.connect(ctx.tf.code) ctx.tf.c.connect(ctx.result) shell = ctx.tf.shell()
def translate(graph, ctx, from_lib_paths, is_lib): ###import traceback; stack = traceback.extract_stack(); print("TRANSLATE:"); print("".join(traceback.format_list(stack[:3]))) contexts = {con["path"]: con for con in graph if con["type"] == "context"} for path in sorted(contexts.keys(), key=lambda k: len(k)): parent = get_path(ctx, path[:-1], None, is_target=False) name = path[-1] c = context(context=parent, name=name) setattr(parent, name, c) # No need to add it to namespace, as long as the low-level graph structure is imitated connections = [con for con in graph if con["type"] == "connection"] connection_paths = [(con["source"], con["target"]) for con in connections] links = [con for con in graph if con["type"] == "link"] link_paths = [(node["first"]["path"], node["second"]["path"]) for node in links] lowlevel_links = {} #multiple links (A,B), (A,C) are allowed; A becomes the real one #multiple links (B,A), (C,A) are not allowed #multiple links (A,B), (A,C), (D,A), (E,D) leads to A,B,C,D => E, E as the real one simple_links = [] for node in links: first, second = node["first"], node["second"] if not first["simple"]: continue if not second["simple"]: continue first, second = first["path"], second["path"] simple_links.append((first, second)) assert second not in lowlevel_links lowlevel_links[second] = first change = True while change: change = False simple_links0 = simple_links simple_links = [] for first, second in simple_links0: while first in lowlevel_links: first = lowlevel_links[first] change = True lowlevel_links[second] = first simple_links.append((first, second)) link_target_paths = set(lowlevel_links.values()) link_targets = { } #maps "first" paths of a link (aka link target paths, aka "real cells") to their translated cells namespace = {} for node in graph: t = node["type"] if t in ("context", "connection", "link"): continue path = node["path"] lib_path = get_lib_path(path[:-1], from_lib_paths) if t == "cell" and path in link_target_paths: assert node[ "celltype"] != "structured" #low-level links are between simple cells! inchannels, outchannels = find_channels(path, connection_paths) editchannels = find_editchannels(path, link_paths) translated_cell = translate_cell(node, ctx, namespace, inchannels, outchannels, editchannels, lib_path, is_lib) link_targets[path] = translated_cell #print("LOW-LEVEL LINKS", lowlevel_links) #print("LOW-LEVEL LINK TARGETS", link_targets) for node in graph: t = node["type"] if t in ("context", "connection", "link"): continue path = node["path"] lib_path = get_lib_path(path[:-1], from_lib_paths) if t == "transformer": inchannels, outchannels = find_channels(node["path"], connection_paths) if node["compiled"]: from .translate_compiled_transformer import translate_compiled_transformer translate_compiled_transformer(node, ctx, namespace, inchannels, outchannels, lib_path, is_lib) elif node["language"] in ("python", "ipython"): translate_py_transformer(node, ctx, namespace, inchannels, outchannels, lib_path, is_lib) elif node["language"] == "bash": translate_bash_transformer(node, ctx, namespace, inchannels, outchannels, lib_path, is_lib) else: raise NotImplementedError(node["language"]) elif t == "reactor": if node["language"] not in ("python", "ipython"): raise NotImplementedError(node["language"]) inchannels, outchannels = find_channels(node["path"], connection_paths) editchannels = find_editchannels(node["path"], link_paths) translate_py_reactor(node, ctx, namespace, inchannels, outchannels, editchannels, lib_path, is_lib) elif t == "cell": if path in link_target_paths: continue #done already before inchannels, outchannels = find_channels(path, connection_paths) editchannels = find_editchannels(path, link_paths) link_target = None if path in lowlevel_links: link_target = link_targets[lowlevel_links[path]] translate_cell(node, ctx, namespace, inchannels, outchannels, editchannels, lib_path, is_lib, link_target=link_target) else: raise TypeError(t) namespace2 = OrderedDict() for k in sorted(namespace.keys(), key=lambda k: -len(k)): namespace2[k] = namespace[k] for node in links: translate_link(node, namespace2, ctx) for node in connections: translate_connection(node, namespace2, ctx)
def create(ctx, mount, state=None): with macro_mode_on(): ctx.hub_struc = context(name="hub_struc",context=ctx) ctx.hub_struc.storage = cell("text") ctx.hub_struc.form = cell("json") ctx.hub_struc.data = cell("mixed", form_cell = ctx.hub_struc.form, storage_cell = ctx.hub_struc.storage, ) ctx.hub_struc.schema = cell("json") ctx.hub_struc.buffer_storage = cell("text") ctx.hub_struc.buffer_form = cell("json") ctx.hub_struc.buffer_data = cell("mixed", form_cell = ctx.hub_struc.buffer_form, storage_cell = ctx.hub_struc.buffer_storage, ) bufferwrapper = BufferWrapper( ctx.hub_struc.buffer_data, ctx.hub_struc.buffer_storage, ctx.hub_struc.buffer_form ) ctx.hub = StructuredCell( "hub", ctx.hub_struc.data, storage = ctx.hub_struc.storage, form = ctx.hub_struc.form, schema = ctx.hub_struc.schema, buffer = bufferwrapper, inchannels = [("a", "factor1"), ("a", "factor2"), ("b",)], outchannels = [("a",), ("b",), ("c",)], state = state, ) ctx.code = cell("transformer").set("d = a.factor1 * b + a.factor2 * c + a.constant") ctx.mixer = transformer({ "a": ("input", "ref", "silk"), "b": ("input", "ref", "object"), "c": ("input", "ref", "object"), "d": ("output", "ref", "json"), }) ctx.a_factor1 = cell("json") ctx.a_factor2 = cell("json") ctx.b = cell("json") ctx.code.connect(ctx.mixer.code) ctx.hub.connect_inchannel(ctx.a_factor1, ("a", "factor1")) ctx.hub.connect_inchannel(ctx.a_factor2, ("a", "factor2")) ctx.hub.connect_inchannel(ctx.b, ("b",)) ctx.hub.connect_outchannel(("a",), ctx.mixer.a) ctx.hub.connect_outchannel(("b",), ctx.mixer.b) ctx.hub.connect_outchannel(("c",), ctx.mixer.c) ctx.result_struc = context(name="result_struc",context=ctx) ctx.result_struc.storage = cell("text") ctx.result_struc.form = cell("json") ctx.result_struc.data = cell("mixed", form_cell = ctx.result_struc.form, storage_cell = ctx.result_struc.storage, ) ctx.result_struc.schema = cell("json") ctx.result_struc.buffer_storage = cell("text") ctx.result_struc.buffer_form = cell("json") ctx.result_struc.buffer_data = cell("mixed", form_cell = ctx.result_struc.buffer_form, storage_cell = ctx.result_struc.buffer_storage, ) bufferwrapper = BufferWrapper( ctx.result_struc.buffer_data, ctx.result_struc.buffer_storage, ctx.result_struc.buffer_form ) ctx.result = StructuredCell( "result", ctx.result_struc.data, storage = ctx.result_struc.storage, form = ctx.result_struc.form, schema = ctx.result_struc.schema, buffer = bufferwrapper, inchannels = [()], outchannels = [] ) ctx.result.connect_inchannel(ctx.mixer.d, ()) if mount: ctx.mount("/tmp/mount-test")
raise NotImplementedError ### import seamless from seamless.core import macro_mode_on from seamless.core import context, cell, transformer, macro, libcell, StructuredCell from seamless.core.structured_cell import BufferWrapper, StructuredCellState from seamless.core import library with macro_mode_on(): ctx = context(toplevel=True) def create(ctx, with_buffer, with_schema, inchannels): with macro_mode_on(): ctx.struc = context(name="struc",context=ctx) ctx.struc.storage = cell("text") ctx.struc.form = cell("json") ctx.struc.data = cell("mixed", form_cell = ctx.struc.form, storage_cell = ctx.struc.storage, ) schema = None if with_schema: ctx.struc.schema = cell("json") schema = ctx.struc.schema bufferwrapper = None if with_buffer: ctx.struc.buffer_storage = cell("text") ctx.struc.buffer_form = cell("json") ctx.struc.buffer_data = cell("mixed", form_cell = ctx.struc.buffer_form, storage_cell = ctx.struc.buffer_storage,