def constructor(ctx, libctx, context_graph, inp, result, elision,
elision_chunksize):
m = ctx.m = Macro()
m.elision = elision
m.graph = context_graph
m.pins.graph.celltype = "plain"
m.pins.result = {
"io": "output",
"celltype": "mixed",
"hash_pattern": {
"!": "#"
}
}
ctx.inp = Context()
ctx.cs_inp = Context()
inp_prefix = "INPUT_"
m.inp_prefix = inp_prefix
m.pins.inp_prefix.celltype = "str"
m.elision_chunksize = elision_chunksize
m.pins.elision_chunksize.celltype = "int"
for key in inp:
c = Cell()
ctx.inp[key] = c
c.hash_pattern = {"!": "#"}
inp[key].connect(c)
ctx.cs_inp[key] = Cell("checksum")
ctx.cs_inp[key] = ctx.inp[key]
setattr(m, inp_prefix + key, ctx.cs_inp[key])
macro_code_lib_code = libctx.map_list_N.value + "\n\n" + libctx.map_list_N_nested.value
macro_code_lib = {
"type": "interpreted",
"language": "python",
"code": macro_code_lib_code
}
ctx.macro_code_lib = Cell("plain").set(macro_code_lib)
ctx.macro_code_lib_code = Cell("code").set(macro_code_lib_code)
m.macro_code_lib = ctx.macro_code_lib
m.pins.macro_code_lib.celltype = "plain"
m.pins.macro_code_lib.subcelltype = "module"
m.macro_code_lib_code = ctx.macro_code_lib_code
m.pins.macro_code_lib_code.celltype = "plain"
m.map_list_N_nested_code = libctx.map_list_N_nested.value
###m.pins.map_list_N_nested_code.celltype = "python"
m.code = libctx.main.value
ctx.result = Cell()
ctx.result.hash_pattern = {"!": "#"}
ctx.result = m.result
result.connect_from(ctx.result)
def constructor(ctx, libctx, template, pattern, ncopies, imports, exports,
entries, exits):
def verify_path(path):
if isinstance(path, (str, int)):
return (path, )
assert isinstance(path, (list, tuple)), type(path)
for attr in path:
assert isinstance(attr, (str, int)), path
return path
for k, path in list(entries.items()):
assert k in imports, k
entries[k] = verify_path(path)
for k, path in list(exits.items()):
assert k in exports, k
exits[k] = verify_path(path)
for n in range(ncopies):
name = "{}{}".format(pattern, n + 1)
instance = Context()
instance.set_graph(template)
setattr(ctx, name, instance)
instance = getattr(ctx, name) # SubContext
for k, path in entries.items():
import_cell = imports[k]
subinstance = instance
for subpathnr, subpath in enumerate(path):
if subpath not in subinstance.get_children():
curr_path = path[:subpathnr + 1]
raise AttributeError("Invalid path {} ({})" %
(path, curr_path))
subinstance = getattr(subinstance, subpath)
if not isinstance(subinstance, Cell):
raise TypeError("Invalid path {} is {} instead of Cell" %
(path, type(subinstance)))
import_cell.connect(subinstance, source_path=name)
for k, path in exits.items():
export_cell = exports[k]
subinstance = instance
for subpathnr, subpath in enumerate(path):
if subpath not in subinstance.get_children():
curr_path = path[:subpathnr + 1]
raise AttributeError("Invalid path {} ({})" %
(path, curr_path))
subinstance = getattr(subinstance, subpath)
if not isinstance(subinstance, Cell):
raise TypeError("Invalid path {} is {} instead of Cell" %
(path, type(subinstance)))
export_cell.connect_from(subinstance, target_path=name)
def bind_status_graph(ctx, status_graph, *, zips=None, mounts=False, shares=True):
""""Creates context that will monitor the status of ctx
The context is loaded from status_graph, which must be a graph in JSON format.
It uses the same manager as ctx.
The status graph's underlying buffers must be available already
(from add_zip or via Seamless database)
The status graph must have a cell called "graph",
and normally, also a cell shared as "index.html"
The status graph will receive the share namespace "status"
mounts and shares have the same meaning as in from_graph
Additional zips can be provided.
They will be passed to ctx.add_zip before the graph is loaded
"""
from seamless.highlevel import Context
ctx2 = Context()
if zips is not None:
for zipf in zips:
ctx2.add_zip(zipf)
ctx2.share_namespace="status"
ctx2.set_graph(
status_graph,
mounts=mounts,
shares=shares
)
assert "graph" in ctx2.get_children()
observe_graph_bound = partial(
observe_graph, ctx, ctx2
)
ctx2.translate()
params = {"runtime": True}
ctx.observe(("get_graph",), observe_graph_bound, OBSERVE_GRAPH_DELAY, params=params)
def observe2(graph):
try:
graph_rt = ctx2.graph_rt
except AttributeError:
graph_rt = None
if not isinstance(graph_rt, Cell):
return
ctx2.graph.set(deepcopy(graph))
ctx.observe(("get_graph",), observe2, OBSERVE_GRAPH_DELAY)
return ctx2
async def load():
from seamless.metalevel.bind_status_graph import bind_status_graph_async
import json
global ctx, webctx, save
graph = json.load(open("graph/" + PROJNAME + ".seamless"))
for f in ("web/index.html", "web/index.js", "web/index-CONFLICT.html",
"web/index-CONFLICT.js", "web/webform.json",
"web/webform-CONFLICT.txt"):
if os.path.exists(f):
dest = f + "-BAK"
if os.path.exists(dest):
os.remove(dest)
shutil.move(f, dest)
ctx = Context()
ctx.load_vault("vault")
ctx.set_graph(graph, mounts=True, shares=True)
await ctx.translation(force=True)
status_graph = json.load(open("graph/" + PROJNAME + "-webctx.seamless"))
webctx = await bind_status_graph_async(ctx,
status_graph,
mounts=True,
shares=True)
def save():
import os, itertools, shutil
def backup(filename):
if not os.path.exists(filename):
return filename
for n in itertools.count():
n2 = n if n else ""
new_filename = "{}.bak{}".format(filename, n2)
if not os.path.exists(new_filename):
break
shutil.move(filename, new_filename)
return filename
ctx.save_graph(backup("graph/" + PROJNAME + ".seamless"))
webctx.save_graph(backup("graph/" + PROJNAME + "-monitoring.seamless"))
ctx.save_vault("vault")
webctx.save_vault("vault")
print("""Project loaded.
Main context is "ctx"
Web/status context is "webctx"
Open http://localhost:<REST server port> to see the web page
Open http://localhost:<REST server port>/status/status.html to see the status
Run save() to save the project
""")
async def load():
from seamless.metalevel.bind_status_graph import bind_status_graph_async
import json
global ctx, ctx2, save
graph = json.load(open("graph/" + PROJNAME + ".seamless"))
ctx = Context()
ctx.add_zip("graph/" + PROJNAME + ".zip")
ctx.set_graph(graph, mounts=True, shares=True)
await ctx.translation(force=True)
status_graph = json.load(open("graph/" + PROJNAME +
"-monitoring.seamless"))
ctx2 = await bind_status_graph_async(
ctx,
status_graph,
mounts=True,
shares=True,
zips=["graph/" + PROJNAME + "-monitoring.zip"],
)
def save():
import os, itertools, shutil
def backup(filename):
if not os.path.exists(filename):
return filename
for n in itertools.count():
n2 = n if n else ""
new_filename = "{}.bak{}".format(filename, n2)
if not os.path.exists(new_filename):
break
shutil.move(filename, new_filename)
return filename
ctx.save_graph(backup("graph/" + PROJNAME + ".seamless"))
ctx2.save_graph(backup("graph/" + PROJNAME + "-monitoring.seamless"))
ctx.save_zip(backup("graph/" + PROJNAME + ".zip"))
ctx2.save_zip(backup("graph/" + PROJNAME + "-monitoring.zip"))
print("""Project loaded.
Main context is "ctx"
Status context is "ctx2"
Run save() to save the context
""")
def constructor(ctx, libctx, result, state={}, **kw):
ctx.result = Cell("mixed")
if state is None:
return
startvalue = state.get("startvalue")
if startvalue is not None:
ctx.startvalue = Cell("mixed").set(startvalue)
channel_contents = ctx.startvalue
for step, operator in enumerate(state.get("operators", [])):
opname, op_params = operator
subctxname = "step%d_%s" % (step + 1, opname)
ctx[subctxname] = Context()
subctx = ctx[subctxname]
if opname in ("filter", "first"):
subctx.tf = Transformer()
subctx.tf.code = op_params
subctx.tf.channel_contents = channel_contents
subctx.result = subctx.tf
subctx.result.celltype = "mixed"
channel_contents = subctx.result
else:
raise NotImplementedError(opname)
ctx.result = channel_contents
result.connect_from(ctx.result)
from seamless.highlevel import Context
import time
ctx = Context()
ctx.pdbcodes = ["1AVX", "1ACB"]
#ctx.pdbcodes.celltype = "plain"
ctx.test = ["1BBB"]
#ctx.test.celltype = "plain"
t = ctx.pdbcodes.traitlet()
ctx.compute()
def obs(change):
print("OBS", change)
t.observe(obs)
print("start")
print(t.value)
t.value = ["1ZZZ"]
time.sleep(0.2) #value update takes 0.1 sec
ctx.compute()
print(t.value)
print(ctx.pdbcodes.value)
print("#2")
ctx.pdbcodes = ["1AAA"]
ctx.compute()
print(t.value)
print(ctx.pdbcodes.value)
print("#3")
t.value = ["1QQQ"]
time.sleep(0.2) #value update takes 0.1 sec
ctx.compute()
from seamless.highlevel import Context, Transformer
try:
import seamless
redis_sink = seamless.RedisSink()
import asyncio
asyncio.get_event_loop().run_until_complete(asyncio.sleep(0.5))
redis_sink.connection.info()
except:
print("No Redis found!")
ctx = Context()
ctx.pdb0 = open("1crn.pdb").read()
ctx.pdb0.celltype = "text"
ctx.pdb0.share()
ctx.filter_pdb = Transformer()
ctx.filter_pdb.language = "bash"
ctx.filter_pdb.code = 'grep ATOM pdb0 | awk \'$3 == "CA" || $3 == "C" || $3 == "O" || $3 == "N"\' '
ctx.filter_pdb.pdb0 = ctx.pdb0
ctx.bb_pdb = ctx.filter_pdb
ctx.bb_pdb.celltype = "text"
ctx.bb_pdb.share()
ctx.bb_pdb = ctx.filter_pdb
ctx.bb_pdb.celltype = "text"
ctx.bb_pdb.share()
ctx.fix_pdb = Transformer()
return 3 * a + b
if __name__ == "__main__":
# force the use of processes
import os
os.environ["SEAMLESS_USE_PROCESSES"] = "true"
#set multiprocessing to "spawn", as it would be on Windows
import multiprocessing
multiprocessing.set_start_method("spawn")
from seamless.highlevel import Context
ctx = Context()
#ctx.mount("/tmp/mount-test")
ctx.a = 12
ctx.transform = triple_it
ctx.transform.a = ctx.a
ctx.myresult = ctx.transform
ctx.equilibrate()
print(ctx.myresult.value)
ctx.tfcode >> ctx.transform.code
print(ctx.tfcode._get_hcell())
print(ctx.tfcode.value)
ctx.transform.b = 100
ctx.tfcode = triple_it_b
from seamless.highlevel import Context, Transformer, Cell
from functools import partial
ctx2 = Context()
ctx2.share_namespace = "status"
ctx2.graph = {}
ctx2.graph.celltype = "plain"
ctx2.graph.share()
ctx2.graph_rt = {}
ctx2.graph_rt.celltype = "plain"
ctx2.status_ = {}
ctx2.status_data = ctx2.status_
ctx2.status_data.celltype = "plain"
gvs = ctx2.gen_vis_status = Transformer()
gvs.graph = ctx2.graph_rt
gvs.status_ = ctx2.status_
gvs.code.mount("gen_vis_status.py", authority="file")
ctx2.vis_status = ctx2.gen_vis_status
ctx2.vis_status.celltype = "plain"
ctx2.vis_status.share(readonly=True)
c = ctx2.html = Cell()
c.set(open("status-visualization.html").read())
c.celltype = "text"
c.mimetype = "text/html"
c.share(path="index.html")
import seamless, os
seamless_dir = os.path.dirname(seamless.__file__)
c = ctx2.seamless_client_js = Cell()
from seamless.highlevel import Context, Reactor
ctx = Context()
ctx.test = "<b>This is a test</b>"
print("cell:")
print(ctx.test.mimetype)
ctx.test.datatype = "text"
print(ctx.test.mimetype)
ctx.test.datatype = "html"
print(ctx.test.mimetype)
ctx.test.mimetype = "text/xhtml"
print(ctx.test.mimetype)
print()
ctx.tf = lambda a,b: 10
print("transformer:")
print(ctx.tf.code.mimetype)
print()
ctx.rc = Reactor()
print("reactor:")
print(ctx.rc.code_start.mimetype)
print(ctx.rc.code_update.mimetype)
print(ctx.rc.code_stop.mimetype)
print()
ctx.tf._get_htf()["language"] = "cpp" #hack
print("C++ transformer:")
print(ctx.tf.code.mimetype)
from seamless.highlevel import Context, Cell
import traceback
ctx = Context()
ctx.a = Cell("int").set(20)
ctx.b = ctx.a
ctx.b.celltype = "int"
ctx.compute()
print(ctx.b.value)
try:
ctx.b.mount("/tmp/x")
except Exception:
traceback.print_exc()
try:
ctx.b.share(readonly=False)
except Exception:
traceback.print_exc()
ctx.compute()
import os, tempfile
from seamless.highlevel import Context, Cell
ctx = Context()
ctx.a = 10
ctx.a.celltype = "plain"
ctx.b = 30
ctx.b.celltype = "plain"
def build_transformer():
del ctx.transform
ctx.transform = lambda a, b: a + b
ctx.translate()
ctx.transform.example.a = 0
ctx.transform.example.b = 0
ctx.result = ctx.transform
ctx.result.celltype = "plain"
ctx.transform.a = ctx.a
ctx.transform.b = ctx.b
ctx.transform.language = "cpp"
ctx.transform.main_module.compiler_verbose = False
ctx.code = ctx.transform.code.pull()
ctx.code = """
extern "C" double add(int a, int b);
extern "C" int transform(int a, int b, double *result) {
*result = add(a,b);
from seamless.highlevel import Context
ctx = Context()
ctx.a = {}
ctx.a.b = {}
ctx.a.b.c = {}
ctx.a.b.c.d = 10
print(ctx.a.value)
def report(a,**args):
print("report", a, args)
ctx.report = report
ctx.report.a = ctx.a
ctx.report.b = ctx.a.b
ctx.report.c = ctx.a.b.c
ctx.report.d = ctx.a.b.c.d
ctx.equilibrate()
ctx.a.schema["b"].pop("c")
ctx.a.b.c = None
ctx.equilibrate()
print(ctx.report.status())
ctx.a.b.c = {"d": 10, "dd": 20}
ctx.equilibrate()
from seamless.highlevel import Context, Transformer
ctx = Context()
ctx.a = 12
ipycode = """
%%timeit
def triple_it(a):
return 3 * a
"""
ctx.transform = Transformer()
ctx.transform.language = "ipython"
ctx.transform.code = ipycode
ctx.transform.a = ctx.a
ctx.myresult = ctx.transform
ctx.equilibrate()
print(ctx.myresult.value)
ctx.a = 13
ctx.equilibrate()
print(ctx.myresult.value)
import numpy as np
from seamless.highlevel import Context, Cell, stdlib
from seamless.highlevel import set_resource
gen_header_file = "gen_header.py"
compiler_file = "compiler.py"
translator_file = "translator.py"
ctx = Context()
def gen_header(input_schema, result_schema, input_name, result_name, inputpins):
return None
ctx.gen_header = gen_header
pins = ctx.gen_header._get_htf()["pins"] ###
pins["input_schema"]["access_mode"] = "plain"
pins["result_schema"]["access_mode"] = "plain"
pins["input_name"]["access_mode"] = "text"
pins["result_name"]["access_mode"] = "text"
ctx.gen_header.code = set_resource(gen_header_file)
ctx.compiler = lambda lang, header, compiled_code, main_module, compiler_verbose: None
ctx.compiler.code = set_resource(compiler_file)
pins = ctx.compiler._get_htf()["pins"] ###
pins["lang"]["access_mode"] = "text"
pins["compiled_code"]["access_mode"] = "text"
pins["header"]["access_mode"] = "text"
pins["main_module"]["access_mode"] = "plain"
pins["compiler_verbose"]["access_mode"] = "plain"
def func(binary_module, pins, input_schema, result_schema, input_name, result_name, kwargs):
None
from seamless.highlevel import Context
ctx = Context()
ctx.txt = "not OK"
ctx.txt.celltype = "text"
ctx.txt.mount("mount.txt", authority="file")
ctx.equilibrate()
print(ctx.txt.value)
ctx.mount("mount-test", persistent=False)
ctx.equilibrate()
from seamless.highlevel import Context, mylib
ctx = Context()
ctx.spam = "spam"
mylib.Test = ctx
mylib.Test.set_constructor(
constructor=lambda ctx: ctx,
post_constructor=None,
args=[],
direct_library_access=True
)
ctx = Context()
ctx.test = mylib.Test()
print(ctx.test.spam)
print(ctx.test.spam.value)
import os, tempfile
from seamless.highlevel import Context, Cell
ctx = Context()
ctx.mount(os.path.join(tempfile.gettempdir(), "transformer-compiled"))
ctx.transform = lambda a,b: a + b
ctx.transform.example.a = 0
ctx.transform.example.b = 0
ctx.result = ctx.transform
ctx.result.celltype = "json"
ctx.equilibrate()
print(ctx.result.value)
ctx.transform.language = "cpp"
ctx.code >> ctx.transform.code
ctx.code = """
extern "C" double transform(int a, int b) {
return a + b;
}"""
ctx.transform.result.example = 0.0 #example, just to fill the schema
ctx.equilibrate()
print(ctx.result.value)
ctx.a = 10
ctx.a.celltype = "json"
ctx.transform.a = ctx.a
ctx.b = 30
ctx.b.celltype = "json"
ctx.transform.b = ctx.b
params["port"] = db_port
seamless.database_sink.connect(**params)
seamless.database_cache.connect(**params)
communion_server.configure_master(
transformation_job=True,
transformation_status=True,
)
import asyncio
asyncio.get_event_loop().run_until_complete(asyncio.sleep(2))
import math
from seamless.highlevel import Context, Cell
import json
ctx = Context()
ctx.pi = math.pi
ctx.doubleit = lambda a: 2 * a
ctx.doubleit.a = ctx.pi
ctx.twopi = ctx.doubleit
ctx.translate()
ctx.compute()
print(ctx.pi.value)
print(ctx.twopi.value)
ctx.doubleit.code = lambda a: 42
ctx.compute()
print(ctx.pi.value)
print(ctx.twopi.value)
"""
Version of high-in-low5 that maps over N inputs, zipped
"""
from seamless.highlevel import Context, Cell, Macro
from seamless.highlevel.library import LibraryContainer
mylib = LibraryContainer("mylib")
mylib.map_list_N = Context()
def constructor(ctx, libctx, context_graph, inp, result):
m = ctx.m = Macro()
m.graph = context_graph
m.pins.result = {
"io": "output",
"celltype": "mixed",
"hash_pattern": {
"!": "#"
}
}
ctx.inp = Context()
ctx.cs_inp = Context()
inp_prefix = "INPUT_"
m.inp_prefix = inp_prefix
for key in inp:
c = Cell()
ctx.inp[key] = c
c.hash_pattern = {"!": "#"}
inp[key].connect(c)
from seamless.highlevel import Context, Cell, Link, Reactor ctx = Context() """ ctx.tf = lambda a,b: 42 ctx.x = ctx.tf.code ctx.y = ctx.x ctx.z = ctx.y ctx.translate() print(ctx.tf.code.value) print(ctx.x.value) print(ctx.y.value) print(ctx.z.value) ctx.x = "blah" print(ctx.x._get_hcell()) ctx.x.datatype = "text" print(ctx.tf.code.value) print(ctx.x.value) print(ctx.y.value) print(ctx.z.value) ctx.tf.code = lambda q,r: 10 print(ctx.tf.code.value) print(ctx.x.value) ctx.z = ctx.tf.code ctx.x = "blah2" print(ctx.tf.code.value) print(ctx.x.value)
from seamless.highlevel import Context, Cell, Macro
ctx = Context()
ctx.a = Cell("int")
ctx.b = Cell("int")
def add(a,b):
return a+b
ctx.add = add
ctx.add.a = ctx.a
ctx.add.b = ctx.b
ctx.result = ctx.add
ctx.result.celltype = "int"
ctx.compute()
graph = ctx.get_graph(runtime=True)
ctx = Context()
ctx.graph = Cell("plain").set(graph)
m = ctx.m = Macro()
ctx.par_static = 100
ctx.par_dynamic = 20
m.par_static = ctx.par_static
m.graph = ctx.graph
m.pins.par_dynamic = {"io": "input", "celltype": "int"}
m.pins.graph_result = {"io": "output", "celltype": "int"}
def run_macro(ctx, par_static, graph):
print("RUN MACRO", par_static)
ctx.subctx = HighLevelContext(graph)
ctx.subctx.a.set(par_static)
ctx.par_dynamic = cell("int")
ctx.par_dynamic.connect(ctx.subctx.b)
from seamless.highlevel import Context, Cell ctx = Context() ctx.a = 10 t = ctx.a.traitlet() ctx.a.set(20) print(t.value) t.value = 80 print(ctx.a.value)
from seamless.highlevel import Context
ctx = Context()
ctx.mount("/tmp/mount-test")
ctx.a = 12
def triple_it(a, b):
print("3 * a + b, a = %s, b = %s" % (a, b))
return 3 * a + b
ctx.transform = triple_it
ctx.transform.a = ctx.a
ctx.transform.b = 6
ctx.myresult = ctx.transform
ctx.equilibrate()
print(ctx.myresult.value)
ctx2 = Context()
ctx2.sub = ctx
ctx2.sub2 = ctx
print(ctx2.sub.myresult.value)
ctx2.equilibrate()
print(ctx2.sub.myresult.value)
ctx2.sub.a = 3
ctx2.sub2.a = 5
ctx2.equilibrate()
print(ctx2.sub.myresult.value)
print(ctx2.sub2.myresult.value)
from seamless.highlevel import Context, Cell
ctx = Context()
ctx.a = Cell("int").set(10)
ctx.c = Cell("int").set(30)
ctx.s = Cell()
ctx.translate()
ctx.s.a = ctx.a
ctx.s.c = ctx.c
ctx.ss = ctx.s
ctx.ss.celltype = "plain"
ctx.compute()
print(ctx.s.value)
print(ctx.ss.value)
ctx.s.set("NOT TO BE PRINTED")
ctx.compute()
print(ctx.s.value)
print(ctx.ss.value)
print(ctx.s.exception)
print("")
ctx.s = "NOT TO BE PRINTED 2"
ctx.s.a = ctx.a
ctx.s.c = ctx.c
ctx.compute()
print(ctx.s.value)
print(ctx.ss.value)
print(ctx.s.exception)
print("")
ctx.s.set({})
ctx.compute()
print(ctx.s.value)
print(ctx.ss.value)
NOTE: it is extremely annoying that all debuggers use breakpoints based on file names.
Fortunately, this is easy to spoof for Python.
Internally, ptvsd uses code from PyDev to debug the script. It relies on
inspect.currentframe().f_code.co_filename to find the filename
(although mappings can be defined).
This corresponds *exactly* to the identifier of seamless.core.cached_compile.
TODO: Therefore, a "virtual filename" attribute will be supported, which will be passed
to cached_compile. It is your responsibility that this corresponds to a real file.
TODO: interpreted modules will also have a file path in their tree. You can map
to their write-only, just as for binary modules, to facilitate visual debugging.
File names and paths must be stripped from checksum calculations.
"""
from seamless.highlevel import Context
ctx = Context()
ctx.a = 12
def triple_it(a):
import sys, pdb
class ForkedPdb(pdb.Pdb):
"""A Pdb subclass that may be used
from a forked multiprocessing child
"""
def interaction(self, *args, **kwargs):
_stdin = sys.stdin
try:
sys.stdin = open('/dev/stdin')
super().interaction(*args, **kwargs)
from seamless.highlevel import Context
ctx = Context()
ctx.a = {"x": 123}
z = ctx.get_zip()
print("ZIP size:", len(z))
ctx.b = ctx.a.x
ctx.compute()
print(ctx.b.value, ctx.b.checksum)
z = ctx.get_zip()
print("ZIP size:", len(z))
ctx.b.scratch = True
ctx.compute()
print(ctx.b.value, ctx.b.checksum)
z = ctx.get_zip()
print("ZIP size (should be 448):", len(z))
from seamless.highlevel import Context, Cell, stdlib
from seamless.highlevel import set_resource
executor_file = "executor.py"
ctx = Context()
ctx.executor_code = set_resource(executor_file)
ctx.executor_code._get_hcell()["language"] = "python"
ctx.executor_code._get_hcell()["transformer"] = True
ctx.executor_code.celltype = "code"
ctx.translate()
if __name__ == "__main__":
#ctx.mount("/tmp/seamless-test", persistent=False) #TODO: persistent=False (does not delete atm)
ctx.testdata = "a\nb\nc\nd\ne\nf\n"
ctx.docker_command = "bash -c 'head -$lines testdata'"
ctx.executor = lambda docker_command, docker_image, docker_options, testdata, pins, lines: None
ctx.executor.pins = ["lines", "testdata"]
ctx.executor.code = ctx.executor_code
ctx.executor.docker_command = ctx.docker_command
ctx.executor.docker_image = "ubuntu"
ctx.executor.docker_options = {}
ctx.executor.testdata = ctx.testdata
ctx.executor.lines = 3
ctx.result = ctx.executor
ctx.equilibrate()
print(ctx.result.value)
ctx.executor_code = ctx.executor_code.value + "\npass"
ctx.equilibrate()
print(ctx.result.value)
else:
from seamless.highlevel import Context, Reactor, Cell, Link, stdlib
from seamless.highlevel import set_resource
ctx = Context()
merge = ctx.merge = Reactor()
merge._get_hrc()["plain"] = True ### TODO: direct API from highlevel.Reactor
merge.set_pin("fallback", io="input")
merge.set_pin("upstream", io="input", access_mode="text")
merge.set_pin("upstream_stage", io="edit", access_mode="text", must_be_defined=False)
merge.set_pin("base", io="edit", access_mode="text", must_be_defined=False)
merge.set_pin("modified", io="edit", access_mode="text", must_be_defined=False)
merge.set_pin("conflict", io="edit", access_mode="text", must_be_defined=False)
merge.set_pin("merged", io="output", access_mode="text")
merge.set_pin("mode", io="output")
merge.fallback = "no"
#TODO: add the validator to the schema of the .fallback property, instead of the main schema
# (requires that .add_validator and ._set_property/._set_method become schema methods)
def validate_fallback(self):
assert self.fallback in ("upstream", "modified", "no"), self.fallback
merge.io.handle.add_validator(validate_fallback)
merge.code_start = set_resource("cell-merge-START.py")
merge.code_update = set_resource("cell-merge-UPDATE.py")
merge.code_stop = ""
# Public cells
ctx.upstream = Cell()
from seamless.highlevel import Context, mylib
def constructor(ctx, dup):
for n in range(dup):
cellname = "dup" + str(n+1)
setattr(ctx, cellname, ctx.spam.value)
if n+1 > 2:
getattr(ctx, cellname).set(n+1)
return ctx
ctx = Context()
ctx.spam = "spam"
mylib.Test = ctx
mylib.Test.set_constructor(
constructor=constructor,
post_constructor=None,
args=[
{"name": "dup", "as_cell": False, "auth": True},
],
direct_library_access=False
)
ctx = Context()
ctx.test = mylib.Test(dup=6)
print(ctx.test.spam, ctx.test.spam.value)
print(ctx.test.dup3, ctx.test.dup3.value)
print(ctx.test.dup6, ctx.test.dup6.value)
ctx.test2 = mylib.Test(3)
print(ctx.test2.dup1, ctx.test2.dup1.value)
print(ctx.test2.dup2, ctx.test2.dup2.value)
from seamless.highlevel import Context
ctx = Context()
ctx.pdbcodes = ["1AVX", "1ACB"]
t = ctx.pdbcodes.traitlet()
ctx.translate()
def obs(change):
print("OBS", change)
t.observe(obs)
print("start")
t.value = ["1ZZZ"]
print(t.value)
print(ctx.pdbcodes.value)
print("again") # This will notify once (bug in traitlets?)
ctx.pdbcodes = ["1AAA"]
print(t.value)
print(ctx.pdbcodes.value)
print("again3")
t.receive_update(["1BBB"]) # This will notify twice (bug doesn't trigger here)
from seamless.highlevel import Context, Cell
ctx = Context()
ctx.v = "test"
ctx.v_schema = Cell()
ctx.v_schema.celltype = "plain"
###ctx.mount("/tmp/mount-test")
ctx.translate()
ctx.link(ctx.v.schema, ctx.v_schema)
ctx.translate()
ctx.v_schema.set({'type': 'integer'})
ctx.compute()
print(ctx.v.schema)
print("*" * 50)
print(ctx.v.exception)
print("*" * 50)
ctx.v.schema.set({})
ctx.compute() # this is needed, else the 1.2 below might take effect first,
# and then be overwritten by this. Seamless is async!!
print(ctx.v.schema)
print(ctx.v_schema.value)
ctx.v.example.set(1.2)
ctx.compute()
print("value:", ctx.v.value)
print("data:", ctx.v.data)
print("buffered:", ctx.v.buffered)
print(ctx.v_schema.value)
print("*" * 50)
print(ctx.v.exception)
print("*" * 50)
ctx.v_schema.set({"type": "string"})
import math
from seamless.highlevel import Context, Cell
import json
ctx = Context()
ctx.pi = math.pi
ctx.doubleit = lambda a: 2 * a
ctx.doubleit.a = ctx.pi
ctx.twopi = ctx.doubleit
ctx.translate()
graph = ctx.get_graph()
print(json.dumps( graph, indent=2, sort_keys=True))
json.dump(graph, open("twopi.seamless", "w"), indent=2, sort_keys=True)
ctx.equilibrate()
print(ctx.pi.value)
print(ctx.twopi.value)
ctx.doubleit.code = lambda a: 42
ctx.equilibrate()
print(ctx.pi.value)
print(ctx.twopi.value)
ctx.translate(force=True)
ctx.equilibrate()
print(ctx.pi.value)
print(ctx.twopi.value)
print()
ctx.doubleit.code = lambda a: 2 * a
ctx.equilibrate()
from seamless.highlevel import Context, Cell, Transformer
ctx = Context()
ctx.a = Cell()
ctx.a.celltype = "int"
ctx.compute()
ctx.a.set(1)
ctx.compute()
ctx.a.set("test")
ctx.compute()
print("*" * 80)
print(ctx.a.exception)
print(ctx.a.value)
print("*" * 80)
ctx.a = 12
ctx.compute()
ctx.a.celltype = "str"
ctx.b = ctx.a
ctx.b.celltype = "int"
ctx.compute()
print("*" * 80)
print("a", ctx.a.exception)
print("a", ctx.a.value)
print("*" * 80)
print("b", ctx.b.exception)
print("b", ctx.b.value)
print("*" * 80)
ctx.a = "test2"
ctx.compute()
import os, tempfile
from seamless.highlevel import Context, Cell
ctx = Context()
ctx.a = 10
ctx.a.celltype = "json"
ctx.b = 30
ctx.b.celltype = "json"
def build_transformer():
ctx.transform = lambda a,b: a + b
ctx.transform.example.a = 0
ctx.transform.example.b = 0
ctx.result = ctx.transform
ctx.result.celltype = "json"
ctx.transform.a = ctx.a
ctx.transform.b = ctx.b
ctx.transform.language = "cpp"
ctx.transform.main_module.compiler_verbose = False
ctx.code >> ctx.transform.code
ctx.code = """
extern "C" double add(int a, int b);
extern "C" double transform(int a, int b) {
return add(a,b);
}"""
ctx.transform.result.example = 0.0 #example, just to fill the schema
from seamless.highlevel import Context, Cell
from seamless.highlevel import set_resource
# 1: Setup context
ctx = Context()
def macro_code(ctx, fallback_mode, code_start, code_update):
reactor_params = {
"fallback_mode": {"io": "input", "celltype": "str"},
"upstream": {"io": "input", "celltype": "text"},
"merged": {"io": "output", "celltype": "text"},
"state": {"io": "output", "celltype": "str"},
}
for k in "upstream_stage", "base", "modified", "conflict":
reactor_params[k] = {
"io": "edit",
"celltype": "text",
"must_be_defined": False,
}
merge = ctx.merge = reactor(reactor_params)
ctx.fallback_mode = cell("str").set(fallback_mode)
ctx.fallback_mode.connect(merge.fallback_mode)
ctx.upstream = cell("text")
ctx.upstream.connect(merge.upstream)
ctx.upstream_stage = cell("text")
ctx.upstream_stage.connect(merge.upstream_stage)
ctx.base = cell("text")
ctx.base.connect(merge.base)
ctx.modified = cell("text")
from seamless.highlevel import Context
ctx = Context()
ctx.code = "bash -c 'head -$lines testdata'"
ctx.code.celltype = "text"
ctx.code.mount("/tmp/test.bash")
ctx.tf = lambda lines, testdata: None
ctx.tf.language = "docker"
ctx.tf.docker_image = "ubuntu"
ctx.tf.docker_options = {"name": "ubuntu-container"}
ctx.tf.testdata = "a \nb \nc \nd \ne \nf \n"
ctx.tf.lines = 3
ctx.tf.code = ctx.code
ctx.result = ctx.tf
ctx.result.mount("/tmp/result")
ctx.translate(force=True)
ctx.equilibrate()
print(ctx.result.value)
import seamless.core.execute
seamless.core.execute.DIRECT_PRINT = True
from seamless.highlevel import Context
from pprint import pprint
ctx = Context()
###ctx.mount("/tmp/mount-test")
ctx.a = 12
ctx.compute()
print(ctx.a.value)
print(ctx.a.schema) # None
def triple_it(a):
return 3 * a
def triple_it_b(a, b):
print("RUN!")
return 3 * a + b
ctx.transform = triple_it
ctx.transform.hash_pattern = {"*": "#"}
ctx.transform.debug = True
ctx.transform.a = 1
print("START")
ctx.compute()
from seamless.highlevel import Context, Cell
from pprint import pprint
ctx = Context()
###ctx.mount("/tmp/mount-test")
ctx.a = 0
ctx.translate()
ctx.a = 2
ctx.compute()
ctx.get_graph()
print(ctx.a.schema)
print(ctx.a.value)
print(ctx.a.exception)
ctx.a = 1
ctx.a.example = 0
ctx.compute()
print(ctx.a.schema)
print(ctx.a.value)
ctx.a = Cell()
ctx.compute()
print(ctx.a.schema)
print(ctx.a.value)
ctx.a.example = 50
print(ctx.a.value)
print(ctx.a.schema)
ctx.a.set(12)
from seamless.highlevel import Context, mylib
ctx = Context()
ctx.spam = "spam"
mylib.Test = ctx
mylib.Test.set_constructor(constructor=lambda ctx: ctx,
post_constructor=None,
args=[],
direct_library_access=True)
ctx = Context()
ctx.test = mylib.Test()
print(ctx.test.spam)
print(ctx.test.spam.value)
import math
from seamless.highlevel import Context, Cell
import json
import seamless
seamless.database_sink.connect()
ctx = Context()
ctx.pi = math.pi
ctx.doubleit = lambda a: 2 * a
ctx.doubleit.a = ctx.pi
ctx.twopi = ctx.doubleit
ctx.translate()
graph = ctx.get_graph()
json.dump(graph,
open("/tmp/twopi-database.seamless", "w"),
indent=2,
sort_keys=True)
import os
os.system("md5sum twopi.seamless /tmp/twopi-database.seamless")
def constructor(ctx, libctx, context_graph, inp, result):
m = ctx.m = Macro()
m.graph = context_graph
m.pins.result = {
"io": "output",
"celltype": "mixed",
"hash_pattern": {
"!": "#"
}
}
ctx.inp = Context()
ctx.cs_inp = Context()
inp_prefix = "INPUT_"
m.inp_prefix = inp_prefix
for key in inp:
c = Cell()
ctx.inp[key] = c
c.hash_pattern = {"!": "#"}
inp[key].connect(c)
ctx.cs_inp[key] = Cell("checksum")
ctx.cs_inp[key] = ctx.inp[key]
setattr(m, inp_prefix + key, ctx.cs_inp[key])
def map_list_N(ctx, inp_prefix, graph, **inp):
first_k = list(inp.keys())[0]
length = len(inp[first_k])
first_k = first_k[len(inp_prefix):]
for k0 in inp:
k = k0[len(inp_prefix):]
if len(inp[k0]) != length:
err = "all cells in inp must have the same length, but '{}' has length {} while '{}' has length {}"
raise ValueError(err.format(k, len(inp[k0]), first_k, length))
from seamless.core import Cell as CoreCell
from seamless.core.unbound_context import UnboundContext
pseudo_connections = []
ctx.result = cell("mixed", hash_pattern={"!": "#"})
ctx.sc_data = cell("mixed", hash_pattern={"!": "#"})
ctx.sc_buffer = cell("mixed", hash_pattern={"!": "#"})
ctx.sc = StructuredCell(data=ctx.sc_data,
buffer=ctx.sc_buffer,
inchannels=[(n, ) for n in range(length)],
outchannels=[()],
hash_pattern={"!": "#"})
for n in range(length):
hc = HighLevelContext(graph)
subctx = "subctx%d" % (n + 1)
setattr(ctx, subctx, hc)
if not hasattr(hc, "inp"):
raise TypeError(
"map_list_N context must have a subcontext called 'inp'")
hci = hc.inp
if not isinstance(hci, UnboundContext):
raise TypeError(
"map_list_N context must have an attribute 'inp' that is a context, not a {}"
.format(type(hci)))
for k0 in inp:
k = k0[len(inp_prefix):]
if not hasattr(hci, k):
raise TypeError(
"map_list_N context must have a cell called inp.'{}'".
format(k))
if isinstance(hci[k], StructuredCell):
raise TypeError(
"map_list_N context has a cell called inp.'{}', but its celltype must be mixed, not structured"
.format(k))
if not isinstance(hci[k], CoreCell):
raise TypeError(
"map_list_N context must have an attribute inp.'{}' that is a cell, not a {}"
.format(k, type(hci[k])))
if hci[k].celltype != "mixed":
raise TypeError(
"map_list_N context has a cell called inp.'{}', but its celltype must be mixed, not {}"
.format(k, hci[k].celltype))
con = [".." + k], ["ctx", subctx, "inp", k]
pseudo_connections.append(con)
cs = inp[k0][n]
hci[k].set_checksum(cs)
resultname = "result%d" % (n + 1)
setattr(ctx, resultname, cell("int"))
c = getattr(ctx, resultname)
hc.result.connect(c)
c.connect(ctx.sc.inchannels[(n, )])
con = ["ctx", subctx, "result"], ["..result"]
pseudo_connections.append(con)
ctx.sc.outchannels[()].connect(ctx.result)
ctx._pseudo_connections = pseudo_connections
m.code = map_list_N
ctx.result = Cell()
ctx.result.hash_pattern = {"!": "#"}
ctx.result = m.result
result.connect_from(ctx.result)
from seamless.highlevel import Context
ctx = Context()
ctx.code = "head -$lines testdata"
ctx.code.celltype = "text"
ctx.code.mount("/tmp/test.bash")
ctx.tf = lambda lines, testdata: None
ctx.tf.language = "bash"
ctx.tf.testdata = "a \nb \nc \nd \ne \nf \n"
ctx.tf.lines = 3
ctx.tf.code = ctx.code
ctx.result = ctx.tf
ctx.result.mount("/tmp/result")
ctx.translate(force=True)
ctx.equilibrate()
print(ctx.result.value)
ctx.code = "tar czf test.tgz testdata; cat test.tgz"
ctx.equilibrate()
print(ctx.result.value)
ctx.code = "python3 -c 'import numpy as np; np.save(\"test\",np.arange(12)*3)'; cat test.npy"
ctx.equilibrate()
print(ctx.result.value)
from seamless.highlevel import Context, Reactor, stdlib
from seamless.highlevel import set_resource
ctx = Context()
b = ctx.browser = Reactor()
b.code_start = set_resource("cell-browser.py")
b.code_update = set_resource("cell-browser_UPDATE.py")
b.code_stop = "widget.destroy()"
b.mimetype = "text"
b.charset = "UTF-8" #for text/ ; otherwise, the browser must figure it out
b.title = "Seamless browser"
b.val = "Hello world!"
ctx.equilibrate()
if __name__ == "__main__":
print(b._get_rc().status())
print(b._get_rc().io.outchannels)
print(b._get_rc().io.value)
print(b.val.mimetype)
pass
else:
stdlib.browser = ctx
from seamless.highlevel import Context
import json
ctx = Context()
###ctx.mount("/tmp/mount-test")
ctx.a = 12
def triple_it(a):
print("triple", a)
return 3 * a
ctx.transform = triple_it
ctx.transform.a = ctx.a
ctx.transform.debug = True
ctx.myresult = ctx.transform
ctx.compute()
print(ctx.myresult.value)
ctx2 = Context()
ctx2.sub = ctx
ctx2.sub2 = ctx
ctx2.translate()
print(ctx2.sub.myresult.value)
ctx2.compute()
print(ctx2.sub.myresult.value)
ctx2.sub.a = 3
ctx2.sub2.a = 5
from seamless.highlevel import Context
# 0
ctx = Context()
ctx.mount("/tmp/mount-test")
# 1
ctx.a = 10
print(ctx.a.value)
# 1a
ctx.a = 12
ctx.translate()
print(ctx.a.value)
# 2
def double_it(a):
return 2 * a
ctx.transform = double_it
ctx.transform.a = ctx.a
ctx.myresult = ctx.transform
ctx.equilibrate()
print(ctx.myresult.value)
# 3
ctx.a = 12
ctx.equilibrate()
print(ctx.myresult.value)
# 4
from seamless.highlevel import Context, stdlib
ctx = Context()
ctx.a = 12
def triple_it(a, **kwargs):
print("triple", a, kwargs)
return 3 * a
ctx.transform = triple_it
ctx.transform.a = ctx.a
ctx.myresult = ctx.transform
###ctx.equilibrate()
print(ctx.myresult.value)
ctx.transform.b = 777
ctx.translate(force=True)
stdlib.triple_it = ctx
print("START")
ctx2 = Context()
ctx2.sub = stdlib.triple_it
ctx2.sub2 = stdlib.triple_it
ctx2.equilibrate()
print(ctx2.sub.myresult.value)
print(ctx2.sub2.myresult.value)
print(ctx2.sub.transform.a.value)
print(ctx2.sub.transform.b.value)
print("UPDATE...")
Since Seamless is purely functional,
there is no such thing as NextFlow queue channels
Work in progress!
"""
from cffi import api
from seamless.core.transformer import Transformer
from attr import has
from seamless.highlevel import Context, Cell
from seamless.highlevel.library import LibraryContainer
from silk.Silk import Silk
mylib = LibraryContainer("mylib")
ctx0 = Context()
def fromPath(self, pattern, is_text):
# Warning: loads everything into memory! Make version that takes a cell...
import glob
if not hasattr(self, "state"):
self.state = {}
operators = getattr(self.state, "operators", [])
if len(operators) or hasattr(self.state, "startvalue"):
raise ValueError("fromPath must be the first operator")
filenames = glob.glob(pattern)
if not len(filenames):
raise ValueError("No files found")
if is_text:
mode = "r"
from seamless.highlevel import Context, Cell
ctx = Context()
ctx.txt = "not OK"
ctx.txt.celltype = "text"
ctx.txt.mount("mount.txt", authority="file")
ctx.compute()
print(ctx.txt.value)
###ctx.mount("mount-test", persistent=False)
ctx.intcell = 780
ctx.intcell.celltype = "int"
ctx.intcell.mount("/tmp/intcell.txt")
ctx.cpp_cell = """
#include <iostream>
using namespace std;
int main()
{
cout << "Hello, World!";
return 0;
}
"""
ctx.cpp_cell.celltype = "code"
ctx.cpp_cell.language = "cpp"
ctx.cpp_cell.mount("/tmp/cpp_cell.cpp")
ctx.txt2 = Cell()
ctx.txt2.celltype = "text"
ctx.link(ctx.txt2, ctx.txt)
ctx.compute()
from seamless.highlevel import Context
ctx = Context()
ctx.a = 12
def triple_it(a):
import sys, pdb
class ForkedPdb(pdb.Pdb):
"""A Pdb subclass that may be used
from a forked multiprocessing child
"""
def interaction(self, *args, **kwargs):
_stdin = sys.stdin
try:
sys.stdin = open('/dev/stdin')
super().interaction(*args, **kwargs)
finally:
sys.stdin = _stdin
#from pdb_clone.pdb import set_trace
#from pdb import set_trace
#from ipdb import set_trace
#set_trace = ForkedPdb().set_trace
from seamless.pdb import set_trace
set_trace()
return 3 * a
import math
from seamless.highlevel import Context, Cell
import json
import seamless
redis_sink = seamless.RedisSink()
ctx = Context()
ctx.pi = math.pi
ctx.doubleit = lambda a: 2 * a
ctx.doubleit.a = ctx.pi
ctx.twopi = ctx.doubleit
ctx.translate()
graph = ctx.get_graph()
json.dump(graph, open("twopi.seamless", "w"), indent=2, sort_keys=True)