def test_map_dict_chunk(mylib): from seamless.highlevel import Context, Cell ctx = Context() ctx.include(mylib.map_dict_chunk) ctx.mul = Context() ctx.mul.inp = Cell("mixed") def mul(a, factor): print("MUL", a) result = {} for key in a: result[key] = a[key] * factor return result ctx.mul.tf = mul ctx.mul.tf.debug = True ctx.mul.tf.a = ctx.mul.inp ctx.mul.tf.factor = 3 ctx.mul.result = ctx.mul.tf ctx.mul.result.celltype = "mixed" ctx.compute() ctx.inp = {"key1": 10, "key2": 220, "key3": 30, "key4": 40} ctx.inp.hash_pattern = {"*": "#"} ctx.result = Cell() ctx.keyorder = Cell("plain") ctx.mapping = ctx.lib.map_dict_chunk(context_graph=ctx.mul, inp=ctx.inp, chunksize=2, keyorder0=[], keyorder=ctx.keyorder, result=ctx.result, elision=True, elision_chunksize=3) ctx.compute() print(ctx.mapping.ctx.status) print(ctx.mapping.ctx.m.ctx.top.exception) print(ctx.result.value) ctx.mapping.keyorder0 = ctx.keyorder.value ctx.compute() print(ctx.result.value) print("UP") inp = ctx.inp.value inp.update({ "a": 80, "b": 30, "c": 999, "d": -1, }) ctx.inp = inp ctx.compute() print(ctx.result.value) print(ctx.keyorder.value)
def test_map_list(mylib): from seamless.highlevel import Context, Cell ctx = Context() ctx.include(mylib.map_list) ctx.add = Context() ctx.add.inp = Cell("mixed") def add(a, b): print("ADD", a, b) return a + b ctx.add.tf = add ctx.add.tf.debug = True ctx.add.tf.a = ctx.add.inp ctx.add.tf.b = 1000 ctx.add.result = ctx.add.tf ctx.add.result.celltype = "int" ctx.compute() ctx.inp = [10, 20, 30, 40] ctx.inp.hash_pattern = {"!": "#"} ctx.result = Cell() ctx.mapping = ctx.lib.map_list(context_graph=ctx.add, inp=ctx.inp, result=ctx.result, elision=True, elision_chunksize=2) ctx.compute() print(ctx.mapping.ctx.m.exception) print(ctx.result.value) ctx.inp += [80, 12, 1, 1, 10, 20, 30, 40] ctx.compute() print(ctx.result.value)
def test_map_dict(mylib): from seamless.highlevel import Context, Cell ctx = Context() ctx.include(mylib.map_dict) ctx.add = Context() ctx.add.inp = Cell("mixed") def add(a, b): print("ADD", a, b) return a + b ctx.add.tf = add ctx.add.tf.debug = True ctx.add.tf.a = ctx.add.inp ctx.add.tf.b = 1000 ctx.add.result = ctx.add.tf ctx.add.result.celltype = "int" ctx.compute() ctx.inp = {"key1": 10, "key2": 220, "key3": 30, "key4": 40} ctx.inp.hash_pattern = {"*": "#"} ctx.result = Cell() ctx.keyorder = Cell("plain") ctx.mapping = ctx.lib.map_dict(context_graph=ctx.add, inp=ctx.inp, keyorder0=[], keyorder=ctx.keyorder, result=ctx.result, elision=True, elision_chunksize=2) ctx.compute() print(ctx.mapping.ctx.status) print(ctx.mapping.ctx.m.ctx.top.exception) print(ctx.result.value) ctx.mapping.keyorder0 = ctx.keyorder.value ctx.compute() print(ctx.result.value) inp = ctx.inp.value inp.update({ "a": 80, "b": 30, "c": 999, "d": -1, }) ctx.inp = inp ctx.compute() print(ctx.result.value) print(ctx.keyorder.value)
def test_map_list_N_uniform(mylib): from seamless.highlevel import Context, Cell ctx = Context() ctx.include(mylib.map_list_N) ctx.add = Context() ctx.add.uniform = Cell("mixed") ctx.add.inp = Context() ctx.add.inp.a = Cell("mixed") ctx.add.inp.b = Cell("mixed") def add(a, b, c): print("ADD", a, b, c) return a + b + c ctx.add.tf = add ctx.add.tf.debug = True ctx.add.tf.a = ctx.add.inp.a ctx.add.tf.b = ctx.add.inp.b ctx.add.tf.c = ctx.add.uniform ctx.add.result = ctx.add.tf ctx.add.result.celltype = "int" ctx.compute() ctx.a = [110, 120, 130, 140] ctx.a.hash_pattern = {"!": "#"} ctx.b = [2, 4, 8, 12] ctx.b.hash_pattern = {"!": "#"} ctx.c = 7000 ctx.result = Cell() ctx.mapping = ctx.lib.map_list_N(context_graph=ctx.add, inp={ "a": ctx.a, "b": ctx.b, }, uniform=ctx.c, result=ctx.result, elision=True, elision_chunksize=2) ctx.compute() print(ctx.result.value) ctx.c = 8000 ctx.compute() print(ctx.result.value)
ctx = Context() ctx.include(mylib.channel) def filter_code(key, value): import os return os.path.splitext(key)[1] == "" ctx.filter_code = Cell("code").set(filter_code) ctx.inst = (ctx.lib.channel().fromPath("./*", is_text=True).filter(filter_code)) ctx.result = ctx.inst.result ctx.result.celltype = "plain" ctx.compute() print(ctx.result.value) ctx.inst.first(lambda k, v: k == "./b") ctx.compute() print(ctx.result.value) print(ctx.inst.ctx.step2_first.tf.status) print(ctx.inst.ctx.step2_first.tf.exception) """ if not hasattr(self, "kw"): self.kw = {} n = 0 while 1: n += 1
ctx = Context() def func(a, b, c, x): return a + b + 10 * c - x ctx.tf = func ctx.a = 10 ctx.b = 20 ctx.tf.a = ctx.a ctx.tf.b = ctx.b ctx.tf.c = 80 ctx.tf.x = 123 ctx.compute() def func2(b, c, d): return b + 10 * c - d ctx.tf = func2 ctx.tf.d = 5 ctx.compute() print(ctx.tf.status) print(ctx.tf.exception) print(ctx.tf.inp.value) print(ctx.tf.result.value) graph = ctx.get_graph()
from seamless.highlevel import Context, Cell from seamless.highlevel.library import LibraryContainer from pprint import pprint lib = LibraryContainer("lib") subctx = Context() subctx.x = 20 subctx.y = 5 subctx.minus = lambda x, y: x - y subctx.minus.x = subctx.x subctx.minus.y = subctx.y subctx.result = subctx.minus subctx.compute() print(subctx.result.value) # Simplified version of lib.instantiate lib.instantiate0 = Context() def constructor(ctx, libctx, context_graph, copies): for n in range(copies): name = "copy{}".format(n + 1) subctx = Context() subctx.set_graph(context_graph) setattr(ctx, name, subctx) lib.instantiate0.constructor = constructor lib.instantiate0.params = {"copies": "value", "context_graph": "context"}
ctx2.vis_status.share(readonly=True) c = ctx2.html = Cell() c.set(open("status-visualization.html").read()) c.celltype = "text" c.mimetype = "text/html" c.share(path="index.html") import seamless, os seamless_dir = os.path.dirname(seamless.__file__) c = ctx2.seamless_client_js = Cell() c.celltype = "text" c.set(open(seamless_dir + "/js/seamless-client.js").read()) c.mimetype = "text/javascript" c.share(path="seamless-client.js") c = ctx2.status_visualization_js = Cell() c.celltype = "text" c.set(open("status-visualization.js").read()) c.mimetype = "text/javascript" c.share(path="status-visualization.js") c = ctx2.css = Cell() c.celltype = "text" c.set(open("status-visualization.css").read()) c.mimetype = "text/css" c.share(path="status-visualization.css") ctx2.compute() ctx2.save_graph("../status-visualization.seamless") ctx2.save_zip("../status-visualization.zip")
def constructor(ctx, libctx, language, code, result, scatter, inputpins, celltypes): ctx.code = Cell("text") ctx.code.set(code) ctx.result = Cell() if scatter not in inputpins: raise AttributeError( "Pin '{}' to scatter does not exist".format(scatter)) scattered_input = inputpins[scatter] if scattered_input[0] == "value": # Simple case (scattered input as value) scattered_value = scattered_input[1] if isinstance(scattered_value, list): keys = range(1, len(scattered_value) + 1) values = scattered_value elif isinstance(scattered_value, dict): for k in scattered_value: if not isinstance(k, str): raise TypeError( "Pin '{}' to scatter is a dict with non-string key '{}'" .format(scatter, k)) keys = scattered_value.keys() values = scattered_value.values() else: raise TypeError( "Pin '{}' to scatter must be a list or dict, not '{}'".format( scatter, type(scattered_value))) for pin_name in inputpins: if pin_name == scatter: continue pin_type, pin_cell = inputpins[pin_name] if pin_type != "cell": continue ctx[pin_name] = Cell(pin_cell.celltype) pin_cell.connect(ctx[pin_name]) for key, value in zip(keys, values): tf_name = "TRANSFORMER_" + str(key) tf = ctx[tf_name] = Transformer() tf.language = language tf.code = ctx.code tf[scatter] = value for pin_name in inputpins: if pin_name in celltypes: getattr(tf.pins, pin_name).celltype = celltypes[pin_name] if pin_name == scatter: continue pin_type, pin_content = inputpins[pin_name] if pin_type == "value": tf[pin_name] = pin_content else: tf[pin_name] = ctx[pin_name] tf_result_name = "TRANSFORMER_RESULT_" + str(key) ctx[tf_result_name] = tf.result ctx.result[key] = ctx[tf_result_name] elif scattered_input[0] == "cell": # Complex case (scattered input as cell) scattered_cell = scattered_input[1] tf_ctx = Context() tf_ctx[scatter] = Cell(celltype=celltypes.get(scatter, "mixed")) tf = tf_ctx.tf = Transformer() tf[scatter] = tf_ctx[scatter] getattr(tf.pins, scatter).celltype = celltypes.get(scatter, "mixed") tf.code = code for pin_name in inputpins: if pin_name == scatter: continue pin_type, pin_value = inputpins[pin_name] pin_name2 = "PIN_" + pin_name if pin_type == "value": tf[pin_name] = pin_value elif pin_type == "cell": tf_ctx[pin_name2] = Cell( celltype=celltypes.get(pin_name, "mixed")) tf[pin_name] = tf_ctx[pin_name2] getattr(tf.pins, pin_name).celltype = celltypes.get(pin_name, "mixed") tf_ctx.result = Cell(celltype=celltypes.get("result", "mixed")) tf_ctx.result = tf.result tf_ctx.compute() tf_graph = tf_ctx.get_graph() ctx.m = Macro() ctx.m.code = libctx.macro_code.value ctx.m.tf_graph = tf_graph ctx.scattered_input = Cell(scattered_cell.celltype) ctx.m.scattered_input = ctx.scattered_input scattered_cell.connect(ctx.scattered_input) ctx.m.scattered_input_name = scatter ctx.m.celltypes = celltypes for pin_name in inputpins: if pin_name == scatter: continue pin_type, pin_cell = inputpins[pin_name] pin_name2 = "PIN_" + pin_name if pin_type == "cell": ctx[pin_name2] = Cell( celltype=celltypes.get(pin_name, "mixed")) pin_cell.connect(ctx[pin_name2]) setattr(ctx.m, pin_name2, ctx[pin_name2]) ctx.m.pins.result = {"io": "output", "celltype": "mixed"} ctx.result = ctx.m.result else: raise TypeError(scattered_input[0]) result.connect_from(ctx.result)
sctx.b = Cell("str") sctx.a = sctx.inp2.a sctx.b = sctx.inp2.b def add(a, b): print("ADD", a[:10], b[:10]) return a + b sctx.add = add sctx.add.a = sctx.a sctx.add.b = sctx.b sctx.result = sctx.add sctx.result.celltype = "str" sctx.compute() ctx = Context() graph = sctx.get_graph(runtime=True) ctx.graph = Cell("plain").set(graph) ctx.data = Cell() ctx.data.hash_pattern = {"!": "#"} ctx.compute() #ctx.data.schema.storage = "pure-plain" # bad idea... validation forces full value construction repeat = int(10e6) #for n in range(1000): # 2x10 GB for n in range(100): # 2x1 GB a = "A:%d:" % n + str(n % 10) * repeat b = "B:%d:" % n + str(n % 10) * repeat """
from seamless.core.manager.tasks.evaluate_expression import SerializeToBufferTask old_run = SerializeToBufferTask._run async def _run(self): print("SERIALIZE TO BUFFER", self.value) return await old_run(self) SerializeToBufferTask._run = _run ctx = Context() ctx.a = Cell("int").set(2) ctx.cell = Cell() ctx.compute() ctx.cell.a = ctx.a print("START") ctx.compute() print(ctx.cell.value) cs = bytes.fromhex(ctx.cell.checksum) print("RE-TRANSLATE 1") ctx.translate(force=True) ctx.compute() # "SERIALIZE TO BUFFER" must NOT be printed print(ctx.cell.value) print("DONE") manager = ctx._manager manager.cachemanager.join_cache.clear() print("RE-TRANSLATE 2") ctx.translate(force=True) ctx.compute() # "SERIALIZE TO BUFFER" must be printed
ctx_switch.constructor_params = { "celltype": "value", "input": { "type": "cell", "io": "input" }, "selected": { "type": "cell", "io": "input" }, "outputs": { "type": "celldict", "io": "output" }, } ctx_switch.compute() ctx_join.join_code = Cell("code") ctx_join.join_code = join_func ctx_join.constructor_code = Cell("code") ctx_join.constructor_code = constructor_join ctx_join.constructor_params = { "celltype": "value", "output": { "type": "cell", "io": "output" }, "selected": { "type": "cell", "io": "input" },
import os, json ctx = Context() ctx.include(stdlib.merge) ctx.initial_graph = Cell("plain").mount("initial-graph.seamless", "r") ctx.seamless2webform = Cell("code").mount("../seamless2webform.py", "r") ctx.gen_webform = Transformer() ctx.gen_webform.graph = ctx.initial_graph ctx.gen_webform.pins.graph.celltype = "plain" ctx.gen_webform.code = ctx.seamless2webform ctx.initial_webform = ctx.gen_webform ctx.initial_webform.celltype = "plain" ctx.initial_webform.mount("initial-webform.json", "w") ctx.initial_webform0 = Cell("text") ctx.initial_webform0 = ctx.initial_webform ctx.compute() ctx.webform = Cell("plain").mount("webform.json") ctx.webform0 = Cell("text") ctx.link(ctx.webform, ctx.webform0) ctx.webform_CONFLICT = Cell("text").mount("webform-CONFLICT.txt") ctx.webform_STATE = Cell("str") ctx.webform_DUMMY = Cell("text") ctx.compute() ctx.merge_webform = ctx.lib.merge(upstream=ctx.initial_webform0, modified=ctx.webform0, conflict=ctx.webform_CONFLICT, merged=ctx.webform_DUMMY, state=ctx.webform_STATE)
from seamless.highlevel import Context code = """ sleep 3 head -$lines testdata > RESULT """ ctx = Context() ctx.code = code ctx.code.celltype = "text" ctx.tf = lambda lines, testdata: None ctx.tf.language = "bash" ctx.tf.docker_image = "ubuntu" ctx.tf.testdata = "a \nb \nc \nd \ne \nf \n" ctx.tf.lines = 3 ctx.tf.code = ctx.code ctx.compute(1) ctx.tf.lines = 2 # This will cancel the old transformer, and hopefully the Docker image ctx.compute() print(ctx.tf.status) print(ctx.tf.exception) print(ctx.tf.result.value)
mylib.map_list_N.constructor = constructors.map_list_N.constructor mylib.map_list_N.params = constructors.map_list_N.constructor_params mylib.map_dict = ctx mylib.map_dict.constructor = constructors.map_dict.constructor mylib.map_dict.params = constructors.map_dict.constructor_params mylib.map_dict_chunk = ctx mylib.map_dict_chunk.constructor = constructors.map_dict_chunk.constructor mylib.map_dict_chunk.params = constructors.map_dict_chunk.constructor_params from testing import test test(mylib) libctx = Context() for attr in ("map_list", "map_list_N", "map_dict", "map_dict_chunk"): setattr(libctx, attr, Context()) l = getattr(libctx, attr) l.static = ctx l.constructor_code = Cell("code").set( getattr(constructors, attr).constructor) l.constructor_params = getattr(constructors, attr).constructor_params libctx.compute() graph = libctx.get_graph() import os, json currdir = os.path.dirname(os.path.abspath(__file__)) graph_filename = os.path.join(currdir, "../lib-map.seamless") zip_filename = os.path.join(currdir, "../lib-map.zip") libctx.save_graph(graph_filename) libctx.save_zip(zip_filename)
"b": "input", "c": "output", } ctx.tf = transformer(pins) ctx.a = cell("int") ctx.a.connect(ctx.tf.a) ctx.tf.b.cell().set(b) ctx.tf.code.cell().set("c = a * b") ctx.c = cell("int") ctx.tf.c.connect(ctx.c) return m.code = run_macro ctx.result = ctx.m.c ctx.compute() print(ctx.result.value) print(m.status, m.exception) print("re-translate") m.elision = True ctx.translate(force=True) ctx.compute() # Must NOT print RUN MACRO, because of elision print(ctx.result.value) print(m.status, m.exception) print("change b to 10") ctx.m.b = 10 ctx.compute() # Must print RUN MACRO 10 print(ctx.result.value) # 100 print(m.status, m.exception) print("change b back to 20") ctx.m.b = 20
import json import seamless from seamless.highlevel import load_graph, Context graph = json.load(open("twopi-result.seamless")) zipfile = "twopi-result.zip" """ # Does not work well... ctx = load_graph(graph) ctx.add_zip(zipfile) ctx.translate(force=True) print(ctx.pi.value) """ ctx = Context() ctx.add_zip(zipfile) # for now, should be before set_graph to avoid glitches ctx.set_graph(graph) ctx.translate() print(ctx.pi.value.unsilk ) # For now, None; could be defined immediately in future print(ctx.twopi.value.unsilk) # set to None, because of independence ctx.compute() # re-runs the computation; # in the future, the graph will be loaded more smartly # so that this is either not needed, or runs instantly (cache hit) print(ctx.twopi.value.unsilk) print()
from seamless.highlevel import Context, Cell ctx = Context() ctx.v = "test" ctx.v_schema = Cell() ctx.v_schema.celltype = "plain" ###ctx.mount("/tmp/mount-test") ctx.translate() ctx.link(ctx.v.schema, ctx.v_schema) ctx.translate() ctx.v_schema.set({'type': 'integer'}) ctx.compute() print(ctx.v.schema) print("*" * 50) print(ctx.v.exception) print("*" * 50) ctx.v.schema.set({}) ctx.compute() # this is needed, else the 1.2 below might take effect first, # and then be overwritten by this. Seamless is async!! print(ctx.v.schema) print(ctx.v_schema.value) ctx.v.example.set(1.2) ctx.compute() print("value:", ctx.v.value) print("data:", ctx.v.data) print("buffered:", ctx.v.buffered) print(ctx.v_schema.value) print("*" * 50) print(ctx.v.exception) print("*" * 50) ctx.v_schema.set({"type": "string"})
sctx.b = Cell("mixed") sctx.a = sctx.inp.a sctx.b = sctx.inp.b def add(a, b): print("ADD", a[:10]) return a + b sctx.add = add sctx.add.a = sctx.a sctx.add.b = sctx.b sctx.result = sctx.add sctx.result.celltype = "mixed" ctx.compute() ctx.data_a = Cell() ctx.data_a.hash_pattern = {"!": "#"} ctx.data_b = Cell() ctx.data_b.hash_pattern = {"!": "#"} ctx.compute() # Next section is 14.5 secs (if the database is filled), but can be elided to ~0.5s by setting checksum directly (if in flatfile cache). # Not having a DB at all is also 13 secs, so DB request communication (without upload) doesn't cost much. repeat = int(10e6) #repeat = int(5) #for n in range(1000): # 2x10 GB #for n in range(100): # 2x1 GB for n in range(1000):
class ForkedPdb(pdb.Pdb): """A Pdb subclass that may be used from a forked multiprocessing child """ def interaction(self, *args, **kwargs): _stdin = sys.stdin try: sys.stdin = open('/dev/stdin') super().interaction(*args, **kwargs) finally: sys.stdin = _stdin #from pdb_clone.pdb import set_trace #from pdb import set_trace #from ipdb import set_trace #set_trace = ForkedPdb().set_trace from seamless.pdb import set_trace set_trace() return 3 * a ctx.transform = triple_it ctx.transform.debug = True ctx.code = ctx.transform.code.pull() ctx.code.mount("triple_it.py") ctx.transform.a = ctx.a ctx.myresult = ctx.transform ctx.compute(report=None) print(ctx.myresult.value)
sctx.a = Cell("int") sctx.b = Cell("int") sctx.a = sctx.inp.a sctx.b = sctx.inp.b def add(a, b): return a + b sctx.add = add sctx.add.a = sctx.a sctx.add.b = sctx.b sctx.result = sctx.add sctx.result.celltype = "int" ctx.compute() data = [ { "a": 5, "b": 6, }, { "a": -2, "b": 8, }, { "a": 3, "b": 14, }, {