def test_map_dict_chunk(mylib):
from seamless.highlevel import Context, Cell
ctx = Context()
ctx.include(mylib.map_dict_chunk)
ctx.mul = Context()
ctx.mul.inp = Cell("mixed")
def mul(a, factor):
print("MUL", a)
result = {}
for key in a:
result[key] = a[key] * factor
return result
ctx.mul.tf = mul
ctx.mul.tf.debug = True
ctx.mul.tf.a = ctx.mul.inp
ctx.mul.tf.factor = 3
ctx.mul.result = ctx.mul.tf
ctx.mul.result.celltype = "mixed"
ctx.compute()
ctx.inp = {"key1": 10, "key2": 220, "key3": 30, "key4": 40}
ctx.inp.hash_pattern = {"*": "#"}
ctx.result = Cell()
ctx.keyorder = Cell("plain")
ctx.mapping = ctx.lib.map_dict_chunk(context_graph=ctx.mul,
inp=ctx.inp,
chunksize=2,
keyorder0=[],
keyorder=ctx.keyorder,
result=ctx.result,
elision=True,
elision_chunksize=3)
ctx.compute()
print(ctx.mapping.ctx.status)
print(ctx.mapping.ctx.m.ctx.top.exception)
print(ctx.result.value)
ctx.mapping.keyorder0 = ctx.keyorder.value
ctx.compute()
print(ctx.result.value)
print("UP")
inp = ctx.inp.value
inp.update({
"a": 80,
"b": 30,
"c": 999,
"d": -1,
})
ctx.inp = inp
ctx.compute()
print(ctx.result.value)
print(ctx.keyorder.value)
def test_map_list(mylib):
from seamless.highlevel import Context, Cell
ctx = Context()
ctx.include(mylib.map_list)
ctx.add = Context()
ctx.add.inp = Cell("mixed")
def add(a, b):
print("ADD", a, b)
return a + b
ctx.add.tf = add
ctx.add.tf.debug = True
ctx.add.tf.a = ctx.add.inp
ctx.add.tf.b = 1000
ctx.add.result = ctx.add.tf
ctx.add.result.celltype = "int"
ctx.compute()
ctx.inp = [10, 20, 30, 40]
ctx.inp.hash_pattern = {"!": "#"}
ctx.result = Cell()
ctx.mapping = ctx.lib.map_list(context_graph=ctx.add,
inp=ctx.inp,
result=ctx.result,
elision=True,
elision_chunksize=2)
ctx.compute()
print(ctx.mapping.ctx.m.exception)
print(ctx.result.value)
ctx.inp += [80, 12, 1, 1, 10, 20, 30, 40]
ctx.compute()
print(ctx.result.value)
def test_map_dict(mylib):
from seamless.highlevel import Context, Cell
ctx = Context()
ctx.include(mylib.map_dict)
ctx.add = Context()
ctx.add.inp = Cell("mixed")
def add(a, b):
print("ADD", a, b)
return a + b
ctx.add.tf = add
ctx.add.tf.debug = True
ctx.add.tf.a = ctx.add.inp
ctx.add.tf.b = 1000
ctx.add.result = ctx.add.tf
ctx.add.result.celltype = "int"
ctx.compute()
ctx.inp = {"key1": 10, "key2": 220, "key3": 30, "key4": 40}
ctx.inp.hash_pattern = {"*": "#"}
ctx.result = Cell()
ctx.keyorder = Cell("plain")
ctx.mapping = ctx.lib.map_dict(context_graph=ctx.add,
inp=ctx.inp,
keyorder0=[],
keyorder=ctx.keyorder,
result=ctx.result,
elision=True,
elision_chunksize=2)
ctx.compute()
print(ctx.mapping.ctx.status)
print(ctx.mapping.ctx.m.ctx.top.exception)
print(ctx.result.value)
ctx.mapping.keyorder0 = ctx.keyorder.value
ctx.compute()
print(ctx.result.value)
inp = ctx.inp.value
inp.update({
"a": 80,
"b": 30,
"c": 999,
"d": -1,
})
ctx.inp = inp
ctx.compute()
print(ctx.result.value)
print(ctx.keyorder.value)
def test_map_list_N_uniform(mylib):
from seamless.highlevel import Context, Cell
ctx = Context()
ctx.include(mylib.map_list_N)
ctx.add = Context()
ctx.add.uniform = Cell("mixed")
ctx.add.inp = Context()
ctx.add.inp.a = Cell("mixed")
ctx.add.inp.b = Cell("mixed")
def add(a, b, c):
print("ADD", a, b, c)
return a + b + c
ctx.add.tf = add
ctx.add.tf.debug = True
ctx.add.tf.a = ctx.add.inp.a
ctx.add.tf.b = ctx.add.inp.b
ctx.add.tf.c = ctx.add.uniform
ctx.add.result = ctx.add.tf
ctx.add.result.celltype = "int"
ctx.compute()
ctx.a = [110, 120, 130, 140]
ctx.a.hash_pattern = {"!": "#"}
ctx.b = [2, 4, 8, 12]
ctx.b.hash_pattern = {"!": "#"}
ctx.c = 7000
ctx.result = Cell()
ctx.mapping = ctx.lib.map_list_N(context_graph=ctx.add,
inp={
"a": ctx.a,
"b": ctx.b,
},
uniform=ctx.c,
result=ctx.result,
elision=True,
elision_chunksize=2)
ctx.compute()
print(ctx.result.value)
ctx.c = 8000
ctx.compute()
print(ctx.result.value)
ctx = Context()
ctx.include(mylib.channel)
def filter_code(key, value):
import os
return os.path.splitext(key)[1] == ""
ctx.filter_code = Cell("code").set(filter_code)
ctx.inst = (ctx.lib.channel().fromPath("./*",
is_text=True).filter(filter_code))
ctx.result = ctx.inst.result
ctx.result.celltype = "plain"
ctx.compute()
print(ctx.result.value)
ctx.inst.first(lambda k, v: k == "./b")
ctx.compute()
print(ctx.result.value)
print(ctx.inst.ctx.step2_first.tf.status)
print(ctx.inst.ctx.step2_first.tf.exception)
"""
if not hasattr(self, "kw"):
self.kw = {}
n = 0
while 1:
n += 1
ctx = Context()
def func(a, b, c, x):
return a + b + 10 * c - x
ctx.tf = func
ctx.a = 10
ctx.b = 20
ctx.tf.a = ctx.a
ctx.tf.b = ctx.b
ctx.tf.c = 80
ctx.tf.x = 123
ctx.compute()
def func2(b, c, d):
return b + 10 * c - d
ctx.tf = func2
ctx.tf.d = 5
ctx.compute()
print(ctx.tf.status)
print(ctx.tf.exception)
print(ctx.tf.inp.value)
print(ctx.tf.result.value)
graph = ctx.get_graph()
from seamless.highlevel import Context, Cell
from seamless.highlevel.library import LibraryContainer
from pprint import pprint
lib = LibraryContainer("lib")
subctx = Context()
subctx.x = 20
subctx.y = 5
subctx.minus = lambda x, y: x - y
subctx.minus.x = subctx.x
subctx.minus.y = subctx.y
subctx.result = subctx.minus
subctx.compute()
print(subctx.result.value)
# Simplified version of lib.instantiate
lib.instantiate0 = Context()
def constructor(ctx, libctx, context_graph, copies):
for n in range(copies):
name = "copy{}".format(n + 1)
subctx = Context()
subctx.set_graph(context_graph)
setattr(ctx, name, subctx)
lib.instantiate0.constructor = constructor
lib.instantiate0.params = {"copies": "value", "context_graph": "context"}
ctx2.vis_status.share(readonly=True)
c = ctx2.html = Cell()
c.set(open("status-visualization.html").read())
c.celltype = "text"
c.mimetype = "text/html"
c.share(path="index.html")
import seamless, os
seamless_dir = os.path.dirname(seamless.__file__)
c = ctx2.seamless_client_js = Cell()
c.celltype = "text"
c.set(open(seamless_dir + "/js/seamless-client.js").read())
c.mimetype = "text/javascript"
c.share(path="seamless-client.js")
c = ctx2.status_visualization_js = Cell()
c.celltype = "text"
c.set(open("status-visualization.js").read())
c.mimetype = "text/javascript"
c.share(path="status-visualization.js")
c = ctx2.css = Cell()
c.celltype = "text"
c.set(open("status-visualization.css").read())
c.mimetype = "text/css"
c.share(path="status-visualization.css")
ctx2.compute()
ctx2.save_graph("../status-visualization.seamless")
ctx2.save_zip("../status-visualization.zip")
def constructor(ctx, libctx, language, code, result, scatter, inputpins,
celltypes):
ctx.code = Cell("text")
ctx.code.set(code)
ctx.result = Cell()
if scatter not in inputpins:
raise AttributeError(
"Pin '{}' to scatter does not exist".format(scatter))
scattered_input = inputpins[scatter]
if scattered_input[0] == "value":
# Simple case (scattered input as value)
scattered_value = scattered_input[1]
if isinstance(scattered_value, list):
keys = range(1, len(scattered_value) + 1)
values = scattered_value
elif isinstance(scattered_value, dict):
for k in scattered_value:
if not isinstance(k, str):
raise TypeError(
"Pin '{}' to scatter is a dict with non-string key '{}'"
.format(scatter, k))
keys = scattered_value.keys()
values = scattered_value.values()
else:
raise TypeError(
"Pin '{}' to scatter must be a list or dict, not '{}'".format(
scatter, type(scattered_value)))
for pin_name in inputpins:
if pin_name == scatter:
continue
pin_type, pin_cell = inputpins[pin_name]
if pin_type != "cell":
continue
ctx[pin_name] = Cell(pin_cell.celltype)
pin_cell.connect(ctx[pin_name])
for key, value in zip(keys, values):
tf_name = "TRANSFORMER_" + str(key)
tf = ctx[tf_name] = Transformer()
tf.language = language
tf.code = ctx.code
tf[scatter] = value
for pin_name in inputpins:
if pin_name in celltypes:
getattr(tf.pins, pin_name).celltype = celltypes[pin_name]
if pin_name == scatter:
continue
pin_type, pin_content = inputpins[pin_name]
if pin_type == "value":
tf[pin_name] = pin_content
else:
tf[pin_name] = ctx[pin_name]
tf_result_name = "TRANSFORMER_RESULT_" + str(key)
ctx[tf_result_name] = tf.result
ctx.result[key] = ctx[tf_result_name]
elif scattered_input[0] == "cell":
# Complex case (scattered input as cell)
scattered_cell = scattered_input[1]
tf_ctx = Context()
tf_ctx[scatter] = Cell(celltype=celltypes.get(scatter, "mixed"))
tf = tf_ctx.tf = Transformer()
tf[scatter] = tf_ctx[scatter]
getattr(tf.pins, scatter).celltype = celltypes.get(scatter, "mixed")
tf.code = code
for pin_name in inputpins:
if pin_name == scatter:
continue
pin_type, pin_value = inputpins[pin_name]
pin_name2 = "PIN_" + pin_name
if pin_type == "value":
tf[pin_name] = pin_value
elif pin_type == "cell":
tf_ctx[pin_name2] = Cell(
celltype=celltypes.get(pin_name, "mixed"))
tf[pin_name] = tf_ctx[pin_name2]
getattr(tf.pins,
pin_name).celltype = celltypes.get(pin_name, "mixed")
tf_ctx.result = Cell(celltype=celltypes.get("result", "mixed"))
tf_ctx.result = tf.result
tf_ctx.compute()
tf_graph = tf_ctx.get_graph()
ctx.m = Macro()
ctx.m.code = libctx.macro_code.value
ctx.m.tf_graph = tf_graph
ctx.scattered_input = Cell(scattered_cell.celltype)
ctx.m.scattered_input = ctx.scattered_input
scattered_cell.connect(ctx.scattered_input)
ctx.m.scattered_input_name = scatter
ctx.m.celltypes = celltypes
for pin_name in inputpins:
if pin_name == scatter:
continue
pin_type, pin_cell = inputpins[pin_name]
pin_name2 = "PIN_" + pin_name
if pin_type == "cell":
ctx[pin_name2] = Cell(
celltype=celltypes.get(pin_name, "mixed"))
pin_cell.connect(ctx[pin_name2])
setattr(ctx.m, pin_name2, ctx[pin_name2])
ctx.m.pins.result = {"io": "output", "celltype": "mixed"}
ctx.result = ctx.m.result
else:
raise TypeError(scattered_input[0])
result.connect_from(ctx.result)
sctx.b = Cell("str")
sctx.a = sctx.inp2.a
sctx.b = sctx.inp2.b
def add(a, b):
print("ADD", a[:10], b[:10])
return a + b
sctx.add = add
sctx.add.a = sctx.a
sctx.add.b = sctx.b
sctx.result = sctx.add
sctx.result.celltype = "str"
sctx.compute()
ctx = Context()
graph = sctx.get_graph(runtime=True)
ctx.graph = Cell("plain").set(graph)
ctx.data = Cell()
ctx.data.hash_pattern = {"!": "#"}
ctx.compute()
#ctx.data.schema.storage = "pure-plain" # bad idea... validation forces full value construction
repeat = int(10e6)
#for n in range(1000): # 2x10 GB
for n in range(100): # 2x1 GB
a = "A:%d:" % n + str(n % 10) * repeat
b = "B:%d:" % n + str(n % 10) * repeat
"""
from seamless.core.manager.tasks.evaluate_expression import SerializeToBufferTask
old_run = SerializeToBufferTask._run
async def _run(self):
print("SERIALIZE TO BUFFER", self.value)
return await old_run(self)
SerializeToBufferTask._run = _run
ctx = Context()
ctx.a = Cell("int").set(2)
ctx.cell = Cell()
ctx.compute()
ctx.cell.a = ctx.a
print("START")
ctx.compute()
print(ctx.cell.value)
cs = bytes.fromhex(ctx.cell.checksum)
print("RE-TRANSLATE 1")
ctx.translate(force=True)
ctx.compute() # "SERIALIZE TO BUFFER" must NOT be printed
print(ctx.cell.value)
print("DONE")
manager = ctx._manager
manager.cachemanager.join_cache.clear()
print("RE-TRANSLATE 2")
ctx.translate(force=True)
ctx.compute() # "SERIALIZE TO BUFFER" must be printed
ctx_switch.constructor_params = {
"celltype": "value",
"input": {
"type": "cell",
"io": "input"
},
"selected": {
"type": "cell",
"io": "input"
},
"outputs": {
"type": "celldict",
"io": "output"
},
}
ctx_switch.compute()
ctx_join.join_code = Cell("code")
ctx_join.join_code = join_func
ctx_join.constructor_code = Cell("code")
ctx_join.constructor_code = constructor_join
ctx_join.constructor_params = {
"celltype": "value",
"output": {
"type": "cell",
"io": "output"
},
"selected": {
"type": "cell",
"io": "input"
},
import os, json
ctx = Context()
ctx.include(stdlib.merge)
ctx.initial_graph = Cell("plain").mount("initial-graph.seamless", "r")
ctx.seamless2webform = Cell("code").mount("../seamless2webform.py", "r")
ctx.gen_webform = Transformer()
ctx.gen_webform.graph = ctx.initial_graph
ctx.gen_webform.pins.graph.celltype = "plain"
ctx.gen_webform.code = ctx.seamless2webform
ctx.initial_webform = ctx.gen_webform
ctx.initial_webform.celltype = "plain"
ctx.initial_webform.mount("initial-webform.json", "w")
ctx.initial_webform0 = Cell("text")
ctx.initial_webform0 = ctx.initial_webform
ctx.compute()
ctx.webform = Cell("plain").mount("webform.json")
ctx.webform0 = Cell("text")
ctx.link(ctx.webform, ctx.webform0)
ctx.webform_CONFLICT = Cell("text").mount("webform-CONFLICT.txt")
ctx.webform_STATE = Cell("str")
ctx.webform_DUMMY = Cell("text")
ctx.compute()
ctx.merge_webform = ctx.lib.merge(upstream=ctx.initial_webform0,
modified=ctx.webform0,
conflict=ctx.webform_CONFLICT,
merged=ctx.webform_DUMMY,
state=ctx.webform_STATE)
from seamless.highlevel import Context code = """ sleep 3 head -$lines testdata > RESULT """ ctx = Context() ctx.code = code ctx.code.celltype = "text" ctx.tf = lambda lines, testdata: None ctx.tf.language = "bash" ctx.tf.docker_image = "ubuntu" ctx.tf.testdata = "a \nb \nc \nd \ne \nf \n" ctx.tf.lines = 3 ctx.tf.code = ctx.code ctx.compute(1) ctx.tf.lines = 2 # This will cancel the old transformer, and hopefully the Docker image ctx.compute() print(ctx.tf.status) print(ctx.tf.exception) print(ctx.tf.result.value)
mylib.map_list_N.constructor = constructors.map_list_N.constructor
mylib.map_list_N.params = constructors.map_list_N.constructor_params
mylib.map_dict = ctx
mylib.map_dict.constructor = constructors.map_dict.constructor
mylib.map_dict.params = constructors.map_dict.constructor_params
mylib.map_dict_chunk = ctx
mylib.map_dict_chunk.constructor = constructors.map_dict_chunk.constructor
mylib.map_dict_chunk.params = constructors.map_dict_chunk.constructor_params
from testing import test
test(mylib)
libctx = Context()
for attr in ("map_list", "map_list_N", "map_dict", "map_dict_chunk"):
setattr(libctx, attr, Context())
l = getattr(libctx, attr)
l.static = ctx
l.constructor_code = Cell("code").set(
getattr(constructors, attr).constructor)
l.constructor_params = getattr(constructors, attr).constructor_params
libctx.compute()
graph = libctx.get_graph()
import os, json
currdir = os.path.dirname(os.path.abspath(__file__))
graph_filename = os.path.join(currdir, "../lib-map.seamless")
zip_filename = os.path.join(currdir, "../lib-map.zip")
libctx.save_graph(graph_filename)
libctx.save_zip(zip_filename)
"b": "input",
"c": "output",
}
ctx.tf = transformer(pins)
ctx.a = cell("int")
ctx.a.connect(ctx.tf.a)
ctx.tf.b.cell().set(b)
ctx.tf.code.cell().set("c = a * b")
ctx.c = cell("int")
ctx.tf.c.connect(ctx.c)
return
m.code = run_macro
ctx.result = ctx.m.c
ctx.compute()
print(ctx.result.value)
print(m.status, m.exception)
print("re-translate")
m.elision = True
ctx.translate(force=True)
ctx.compute() # Must NOT print RUN MACRO, because of elision
print(ctx.result.value)
print(m.status, m.exception)
print("change b to 10")
ctx.m.b = 10
ctx.compute() # Must print RUN MACRO 10
print(ctx.result.value) # 100
print(m.status, m.exception)
print("change b back to 20")
ctx.m.b = 20
import json
import seamless
from seamless.highlevel import load_graph, Context
graph = json.load(open("twopi-result.seamless"))
zipfile = "twopi-result.zip"
""" # Does not work well...
ctx = load_graph(graph)
ctx.add_zip(zipfile)
ctx.translate(force=True)
print(ctx.pi.value)
"""
ctx = Context()
ctx.add_zip(zipfile) # for now, should be before set_graph to avoid glitches
ctx.set_graph(graph)
ctx.translate()
print(ctx.pi.value.unsilk
) # For now, None; could be defined immediately in future
print(ctx.twopi.value.unsilk) # set to None, because of independence
ctx.compute() # re-runs the computation;
# in the future, the graph will be loaded more smartly
# so that this is either not needed, or runs instantly (cache hit)
print(ctx.twopi.value.unsilk)
print()
from seamless.highlevel import Context, Cell
ctx = Context()
ctx.v = "test"
ctx.v_schema = Cell()
ctx.v_schema.celltype = "plain"
###ctx.mount("/tmp/mount-test")
ctx.translate()
ctx.link(ctx.v.schema, ctx.v_schema)
ctx.translate()
ctx.v_schema.set({'type': 'integer'})
ctx.compute()
print(ctx.v.schema)
print("*" * 50)
print(ctx.v.exception)
print("*" * 50)
ctx.v.schema.set({})
ctx.compute() # this is needed, else the 1.2 below might take effect first,
# and then be overwritten by this. Seamless is async!!
print(ctx.v.schema)
print(ctx.v_schema.value)
ctx.v.example.set(1.2)
ctx.compute()
print("value:", ctx.v.value)
print("data:", ctx.v.data)
print("buffered:", ctx.v.buffered)
print(ctx.v_schema.value)
print("*" * 50)
print(ctx.v.exception)
print("*" * 50)
ctx.v_schema.set({"type": "string"})
sctx.b = Cell("mixed")
sctx.a = sctx.inp.a
sctx.b = sctx.inp.b
def add(a, b):
print("ADD", a[:10])
return a + b
sctx.add = add
sctx.add.a = sctx.a
sctx.add.b = sctx.b
sctx.result = sctx.add
sctx.result.celltype = "mixed"
ctx.compute()
ctx.data_a = Cell()
ctx.data_a.hash_pattern = {"!": "#"}
ctx.data_b = Cell()
ctx.data_b.hash_pattern = {"!": "#"}
ctx.compute()
# Next section is 14.5 secs (if the database is filled), but can be elided to ~0.5s by setting checksum directly (if in flatfile cache).
# Not having a DB at all is also 13 secs, so DB request communication (without upload) doesn't cost much.
repeat = int(10e6)
#repeat = int(5)
#for n in range(1000): # 2x10 GB
#for n in range(100): # 2x1 GB
for n in range(1000):
class ForkedPdb(pdb.Pdb):
"""A Pdb subclass that may be used
from a forked multiprocessing child
"""
def interaction(self, *args, **kwargs):
_stdin = sys.stdin
try:
sys.stdin = open('/dev/stdin')
super().interaction(*args, **kwargs)
finally:
sys.stdin = _stdin
#from pdb_clone.pdb import set_trace
#from pdb import set_trace
#from ipdb import set_trace
#set_trace = ForkedPdb().set_trace
from seamless.pdb import set_trace
set_trace()
return 3 * a
ctx.transform = triple_it
ctx.transform.debug = True
ctx.code = ctx.transform.code.pull()
ctx.code.mount("triple_it.py")
ctx.transform.a = ctx.a
ctx.myresult = ctx.transform
ctx.compute(report=None)
print(ctx.myresult.value)
sctx.a = Cell("int")
sctx.b = Cell("int")
sctx.a = sctx.inp.a
sctx.b = sctx.inp.b
def add(a, b):
return a + b
sctx.add = add
sctx.add.a = sctx.a
sctx.add.b = sctx.b
sctx.result = sctx.add
sctx.result.celltype = "int"
ctx.compute()
data = [
{
"a": 5,
"b": 6,
},
{
"a": -2,
"b": 8,
},
{
"a": 3,
"b": 14,
},
{
