def start_status_thread(host):
client = capnp.TwoPartyClient(host)
cap = client.ez_restore('example').cast_as(thread_capnp.Example)
subscriber = StatusSubscriber()
promise = cap.subscribeStatus(subscriber)
promise.wait()
def connect(self, sturdy_ref, cast_as=None):
# we assume that a sturdy ref url looks always like capnp://hash-digest-or-insecure@host:port/sturdy-ref-token
if sturdy_ref[:8] == "capnp://":
rest = sturdy_ref[8:]
hash_digest, rest = rest.split("@") if "@" in rest else (None,
rest)
host, rest = rest.split(":")
port, sr_token = rest.split("/") if "/" in rest else (rest, None)
host_port = "{}:{}".format(host, port)
if host_port in self._connections:
bootstrap_cap = self._connections[host_port]
else:
bootstrap_cap = capnp.TwoPartyClient(host_port).bootstrap()
self._connections[host_port] = bootstrap_cap
if sr_token:
restorer = bootstrap_cap.cast_as(persistence_capnp.Restorer)
dyn_obj_reader = restorer.restore(sr_token).wait().cap
return dyn_obj_reader.as_interface(
cast_as) if cast_as else dyn_obj_reader
else:
return bootstrap_cap.cast_as(
cast_as) if cast_as else bootstrap_cap
def test_using_threads():
capnp.remove_event_loop(True)
capnp.create_event_loop(True)
read, write = socket.socketpair(socket.AF_UNIX)
def run_server():
restorer = SimpleRestorer()
server = capnp.TwoPartyServer(write, restorer)
capnp.wait_forever()
server_thread = threading.Thread(target=run_server)
server_thread.daemon = True
server_thread.start()
client = capnp.TwoPartyClient(read)
ref = test_capability_capnp.TestSturdyRefObjectId.new_message(
tag='testInterface')
cap = client.restore(ref)
cap = cap.cast_as(test_capability_capnp.TestInterface)
remote = cap.foo(i=5)
response = remote.wait()
assert response.x == '125'
def main():
# start = [Process, Consumer, Producer]
config = {
"process_port": "10001",
"consumer_port": "10002",
"server": "*",
"start": "Process"
}
# read commandline args only if script is invoked directly from commandline
if len(sys.argv) > 1 and __name__ == "__main__":
for arg in sys.argv[1:]:
k, v = arg.split("=")
if k in config:
config[k] = v
cs = config["start"]
if cs == "Process":
consumer = capnp.TwoPartyClient(
"localhost:" + config["consumer_port"]).bootstrap().cast_as(
fbp_capnp.FBP.Input)
server = capnp.TwoPartyServer(config["server"] + ":" +
config["process_port"],
bootstrap=Process(consumer))
server.run_forever()
elif cs == "Consumer":
server = capnp.TwoPartyServer(config["server"] + ":" +
config["consumer_port"],
bootstrap=Consumer())
server.run_forever()
elif cs == "Producer":
produce(config)
async def produce(config):
reader, writer = await asyncio.open_connection("localhost", config["process_port"])
# Start TwoPartyClient using TwoWayPipe (takes no arguments in this mode)
client = capnp.TwoPartyClient()
# Assemble reader and writer tasks, run in the background
coroutines = [myreader(client, reader), mywriter(client, writer)]
asyncio.gather(*coroutines, return_exceptions=True)
process = client.bootstrap().cast_as(fbp_capnp.FBP.Input)
start = time.time()
outer = 100000
sub = 50
l = []
for i in range(1, outer+1):
l.append(process.input("data " + str(i)).a_wait())
if i % sub == 0:
await asyncio.gather(*l)
l = []
#await process.input("data " + str(i)).a_wait()
print("sent","data " + str(i))
end = time.time()
print("sent", outer*sub, "messages in", end-start, "s")
def soil_service_cap_async(start_soil_service_async):
soil_service = capnp.TwoPartyClient(
"localhost:" + str(SOIL_SERVICE_PORT)).bootstrap().cast_as(
soil_data_capnp.Service)
#client = await async_helpers.connect_to_server(SOIL_SERVICE_PORT)
#soil_service = client.bootstrap().cast_as(soil_data_capnp.Service)
return soil_service
def __init__(self, address: str) -> None:
self.address = address
if os.getuid() == 0:
self.client = capnp.TwoPartyClient('unix:/run/metac/' + address +
'/socket')
self.node_admin = self.client.bootstrap().cast_as(
metac_capnp.NodeAdmin)
self.node = self.node_admin.getUnprivilegedNode().wait()
else:
self.client = capnp.TwoPartyClient(address + ':901')
self.node_admin = None
self.node = self.client.bootstrap().cast_as(metac_capnp.Node)
# for CastToLocal
self.local_objects = {}
def __init__(self, address, port):
self._rpc_client = capnp.TwoPartyClient("{}:{}".format(address, port))
bootstrap = self._rpc_client.bootstrap().cast_as(
rpc.server.ServerBootstrap)
registration = bootstrap.registerAsClient(CLIENT_PROTOCOL_VERSION)
self._service = registration.wait().service
def main():
config = {"port": "6003", "server": "localhost"}
# read commandline args only if script is invoked directly from commandline
if len(sys.argv) > 1 and __name__ == "__main__":
for arg in sys.argv[1:]:
k, v = arg.split("=")
if k in config:
config[k] = v
test_climate_service()
#test_registry()
return
#s = capnp.TwoPartyServer("*:11002", bootstrap=csv_based.TimeSeries.from_csv_file("data/climate/climate-iso.csv", header_map={}, pandas_csv_config={}))
#s.run_forever()
#del s
#x()
for i in range(1):
csv_timeseries_cap = capnp.TwoPartyClient(
"localhost:11002").bootstrap().cast_as(
climate_data_capnp.TimeSeries)
header = csv_timeseries_cap.header().wait().header
data = csv_timeseries_cap.data().wait().data
print("i:", i, "header:", header)
"""
def main():
#address = parse_args().address
runtime_available = False
while not runtime_available:
try:
# runtime = capnp.TwoPartyClient("localhost:9000").bootstrap().cast_as(
runtime = capnp.TwoPartyClient(
"10.10.24.186:9000").bootstrap().cast_as(
cluster_admin_service_capnp.Cluster.Runtime)
runtime_available = True
except:
# time.sleep(1)
pass
monicaFactory = SlurmMonicaInstanceFactory({
"port":
10000,
"path_to_monica_binaries":
"C:/Users/berg.ZALF-AD/GitHub/monica/_cmake_vs2019_win64/Debug/"
})
registered_factory = False
while not registered_factory:
try:
unreg = runtime.registerModelInstanceFactory(
"monica_v2.1", monicaFactory).wait().unregister
registered_factory = True
except capnp.KjException as e:
print(e)
time.sleep(1)
#server = capnp.TwoPartyServer("*:8000", bootstrap=DataServiceImpl("/home/berg/archive/data/"))
server = capnp.TwoPartyServer("*:10000", bootstrap=monicaFactory)
server.run_forever()
def __init__(self, address, subworker_id, task_path, stage_path):
self.task_path = task_path
self.stage_path = stage_path
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
# Protection against long filenames, socket names are limitted
backup = os.getcwd()
try:
os.chdir(os.path.dirname(address))
sock.connect(os.path.basename(address))
finally:
os.chdir(backup)
self.rpc_client = capnp.TwoPartyClient(sock)
upstream = self.rpc_client.bootstrap().cast_as(
rpc_subworker.SubworkerUpstream)
self.upstream = upstream
control = ControlImpl(self)
register = upstream.register_request()
register.version = SUBWORKER_PROTOCOL_VERSION
register.subworkerId = subworker_id
register.subworkerType = "py"
register.control = control
register.send().wait()
def get_bridge_cap():
# Connect to the socket exposed by sandstorm-http-bridge
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.connect("/tmp/sandstorm-api")
client = capnp.TwoPartyClient(sock)
bridge_cap = client.bootstrap().cast_as(bridge.SandstormHttpBridge)
return bridge_cap
def rpc(self):
self.dpi = capnp.load(
os.path.join(cosimDir, "cosim_dpi_server", "esi_cosim_dpi.capnp"))
hostname = os.uname()[1]
self.rpc_client = capnp.TwoPartyClient(f"{hostname}:1111")
self.cosim = self.rpc_client.bootstrap().cast_as(
self.dpi.CosimDpiServer)
return self.cosim
async def main(host):
host = host.split(':')
addr = host[0]
port = host[1]
# Setup SSL context
ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=os.path.join(this_dir, 'selfsigned.cert'))
# Handle both IPv4 and IPv6 cases
try:
print("Try IPv4")
reader, writer = await asyncio.open_connection(
addr, port,
ssl=ctx,
family=socket.AF_INET
)
except OSError:
print("Try IPv6")
try:
reader, writer = await asyncio.open_connection(
addr, port,
ssl=ctx,
family=socket.AF_INET6
)
except OSError:
return False
# Start TwoPartyClient using TwoWayPipe (takes no arguments in this mode)
client = capnp.TwoPartyClient()
cap = client.bootstrap().cast_as(thread_capnp.Example)
# Start watcher to restart socket connection if it is lost
overalltasks = []
watcher = [watch_connection(cap)]
overalltasks.append(asyncio.gather(*watcher, return_exceptions=True))
# Assemble reader and writer tasks, run in the background
coroutines = [myreader(client, reader), mywriter(client, writer)]
overalltasks.append(asyncio.gather(*coroutines, return_exceptions=True))
# Start background task for subscriber
tasks = [background(cap)]
overalltasks.append(asyncio.gather(*tasks, return_exceptions=True))
# Run blocking tasks
print('main: {}'.format(time.time()))
await cap.longRunning().a_wait()
print('main: {}'.format(time.time()))
await cap.longRunning().a_wait()
print('main: {}'.format(time.time()))
await cap.longRunning().a_wait()
print('main: {}'.format(time.time()))
for task in overalltasks:
task.cancel()
return True
async def main(host):
host = host.split(':')
addr = host[0]
port = host[1]
# Setup SSL context
ctx = ssl.SSLContext()
# Handle both IPv4 and IPv6 cases
try:
print("Try IPv4")
reader, writer = await asyncio.open_connection(
addr, port,
ssl=ctx,
)
except OSError:
print("Try IPv6")
try:
reader, writer = await asyncio.open_connection(
addr, port,
ssl=ctx,
family=socket.AF_INET6
)
except OSError:
return False
# Start TwoPartyClient using TwoWayPipe (takes no arguments in this mode)
client = capnp.TwoPartyClient()
cap = client.bootstrap().cast_as(echo_capnp.Echo)
# Start watcher to restart socket connection if it is lost
overalltasks = []
watcher = [watch_connection(cap)]
overalltasks.append(asyncio.gather(*watcher, return_exceptions=True))
# Assemble reader and writer tasks, run in the background
coroutines = [myreader(client, reader), mywriter(client, writer)]
overalltasks.append(asyncio.gather(*coroutines, return_exceptions=True))
# Run blocking tasks
await asyncio.sleep(1)
print('main: {}'.format(time.time()))
print(await cap.echo("test").a_wait())
await asyncio.sleep(1)
print('main: {}'.format(time.time()))
print(await cap.echo("test1").a_wait())
await asyncio.sleep(1)
print('main: {}'.format(time.time()))
print(await cap.echo("test2").a_wait())
await asyncio.sleep(1)
print('main: {}'.format(time.time()))
print(await cap.echo("test3").a_wait())
for task in overalltasks:
task.cancel()
return True
def task(ftps_host,
ftps_user,
ftps_pwd,
import_host="localhost",
import_port="15000",
specific_dates=None):
print("Running import task at", datetime.datetime.now())
ds = [datetime.date.today()] if specific_dates is None else specific_dates
ftps = ftplib.FTP_TLS(ftps_host, user=ftps_user, passwd=ftps_pwd)
ftps.prot_p()
ftps.cwd("dwd")
cap = capnp.TwoPartyClient(import_host + ":" +
import_port).bootstrap().cast_as(
dwd_service_capnp.DWLABImport)
dates = [f"{d:%Y%m%d}" for d in ds]
files = defaultdict(lambda: defaultdict(dict))
for entry in filter(lambda e: e[1]["type"] == "file",
ftps.mlsd(facts=["type"])):
date = entry[0][9:17]
type = entry[0][4:8]
time = int(entry[0][18:22])
if date in dates and type in ["DWLA", "DWLB"]:
files[date][type][time] = entry[0]
dates = list(files.keys())
dates.sort()
for date in dates:
type_to_times = files[date]
def retrieve(type):
times = list(type_to_times[type].keys())
if len(times) > 0:
times.sort()
time = times[-1]
with io.BytesIO() as f:
ftps.retrbinary("RETR " + type_to_times[type][time],
f.write)
return f.getvalue().decode("cp1252")
dwla = retrieve("DWLA")
dwla_comp = zlib.compress(dwla.encode("cp1252"))
#print("DWLA:\n", dwla)
dwlb = retrieve("DWLB")
dwlb_comp = zlib.compress(dwlb.encode("cp1252"))
#print("DWLB:\n", dwlb)
d = datetime.datetime.strptime(date, "%Y%m%d")
#print("len(dwla)=",len(dwla), " len(dwlb)=",len(dwlb))
#print("len(dwla_comp)=",len(dwla_comp), " len(dwlb_comp)=",len(dwlb_comp))
success = cap.importData(f"{d:%Y-%m-%d}", dwla_comp, dwlb_comp).wait()
print("Import succeeded?", success)
async def bootstrap_cap_at(server, port, capnp_interface):
reader, writer = await asyncio.open_connection(server, port)
# Start TwoPartyClient using TwoWayPipe (takes no arguments in this mode)
client = capnp.TwoPartyClient()
# Assemble reader and writer tasks, run in the background
coroutines = [myreader(client, reader), mywriter(client, writer)]
asyncio.gather(*coroutines, return_exceptions=True)
return client.bootstrap().cast_as(capnp_interface)
def threaded_object(self, func, interface):
a, b = socket.socketpair()
def run_server():
capnp.create_event_loop()
server = capnp.TwoPartyServer(b, bootstrap=func())
server.on_disconnect().wait()
threading.Thread(target=run_server).start()
client = capnp.TwoPartyClient(a).bootstrap().cast_as(interface)
return client
def test_simple_rpc_bootstrap():
read, write = socket.socketpair()
_ = capnp.TwoPartyServer(write, bootstrap=Server(100))
client = capnp.TwoPartyClient(read)
cap = client.bootstrap()
cap = cap.cast_as(test_capability_capnp.TestInterface)
remote = cap.foo(i=5)
response = remote.wait()
assert response.x == "125"
async def connect(self):
log.info("Connect to host: {0} at {1}:{2}".format(
self.name, self.hostname, self.port))
self.client = capnp.TwoPartyClient("{0}:{1}".format(
self.hostname, self.port))
self.server = self.client.bootstrap().cast_as(S.server.PcpdServer)
has_facade = await self.server.hasFacade().a_wait()
if has_facade:
result = await self.server.getFacade().a_wait()
self.facade = Facade(server=result.value)
log.info("{0} attached to Facade".format(self.name))
await self.facade.connect()
def test_simple_rpc_with_options():
read, write = socket.socketpair()
_ = capnp.TwoPartyServer(write, bootstrap=Server())
# This traversal limit is too low to receive the response in, so we expect
# an exception during the call.
client = capnp.TwoPartyClient(read, traversal_limit_in_words=1)
with pytest.raises(capnp.KjException):
cap = client.bootstrap().cast_as(test_capability_capnp.TestInterface)
remote = cap.foo(i=5)
_ = remote.wait()
def main(server_host, server_port):
sock = socket.socket()
sock.connect((server_host, int(server_port)))
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
sock.setblocking(False)
client = capnp.TwoPartyClient(sock)
helloworld = client.bootstrap().cast_as(HelloWorld)
request = HelloWorld.HelloRequest.new_message(name="World")
print("Sending request...")
result = helloworld.sayHello(request=request).wait()
print(f"Got reply: {result.reply.message}")
def test_simple_rpc_restore_func():
read, write = socket.socketpair(socket.AF_UNIX)
server = capnp.TwoPartyServer(write, restore_func)
client = capnp.TwoPartyClient(read)
ref = test_capability_capnp.TestSturdyRefObjectId.new_message(tag='testInterface')
cap = client.restore(ref)
cap = cap.cast_as(test_capability_capnp.TestInterface)
remote = cap.foo(i=5)
response = remote.wait()
assert response.x == '125'
def _reconnect(self):
if self._sock is not None:
self._sock.close()
if self._proc is not None:
self._proc.wait()
self._sock, remote = socket.socketpair()
with closing(remote):
with open(os.devnull, 'r+') as devnull:
self._proc = subprocess.Popen(
[self._backend, str(remote.fileno())],
stdin=devnull,
pass_fds=[remote.fileno()])
self._sock_client = capnp.TwoPartyClient(self._sock)
self._client = self._sock_client.bootstrap().cast_as(
_passacre_capnp.Toplevel)
def main(host):
client = capnp.TwoPartyClient(host)
cap = client.ez_restore('example').cast_as(thread_capnp.Example)
status_thread = threading.Thread(target=start_status_thread, args=(host, ))
status_thread.daemon = True
status_thread.start()
print('main: {}'.format(time.time()))
cap.longRunning().wait()
print('main: {}'.format(time.time()))
cap.longRunning().wait()
print('main: {}'.format(time.time()))
cap.longRunning().wait()
print('main: {}'.format(time.time()))
def main(server_host, server_port, server_cert):
context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
context.load_verify_locations(server_cert)
sock = socket.socket()
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
conn = context.wrap_socket(sock, server_hostname="localhost")
conn.connect((server_host, int(server_port)))
conn.setblocking(False)
client = capnp.TwoPartyClient(conn)
helloworld = client.bootstrap().cast_as(HelloWorld)
request = HelloWorld.HelloRequest.new_message(name="World")
print("Sending request...")
result = helloworld.sayHello(request=request).wait()
print(f"Got reply: {result.reply.message}")
async def connect(self, sturdy_ref, cast_as = None):
# we assume that a sturdy ref url looks always like capnp://hash-digest-or-insecure@host:port/sturdy-ref-token
try:
if sturdy_ref[:8] == "capnp://":
rest = sturdy_ref[8:]
hash_digest, rest = rest.split("@") if "@" in rest else (None, rest)
host, rest = rest.split(":")
port, sr_token = rest.split("/") if "/" in rest else (rest, None)
host_port = "{}:{}".format(host, port)
if host_port in self.connections:
bootstrap_cap = self.connections[host_port]
else:
# Handle both IPv4 and IPv6 cases
try:
reader, writer = await asyncio.open_connection(
host, port,
family=socket.AF_INET
)
except Exception:
print("Tried open connection via IPv4, now trying IPv6.")
reader, writer = await asyncio.open_connection(
host, port,
family=socket.AF_INET6
)
# Start TwoPartyClient using TwoWayPipe (takes no arguments in this mode)
client = capnp.TwoPartyClient()
# Assemble reader and writer tasks, run in the background
coroutines = [self.socket_reader(client, reader), self.socket_writer(client, writer)]
self.alltasks.append(asyncio.gather(*coroutines, return_exceptions=True))
bootstrap_cap = client.bootstrap()
self.connections[host_port] = bootstrap_cap
if sr_token:
restorer = bootstrap_cap.cast_as(persistence_capnp.Restorer)
dyn_obj_reader = (await restorer.restore(sr_token).a_wait()).cap
return dyn_obj_reader.as_interface(cast_as) if cast_as else dyn_obj_reader
else:
return bootstrap_cap.cast_as(cast_as) if cast_as else bootstrap_cap
except Exception as e:
print(e)
return None
def test_simple_rpc_with_options():
read, write = socket.socketpair(socket.AF_UNIX)
restorer = SimpleRestorer()
server = capnp.TwoPartyServer(write, restorer)
# This traversal limit is too low to receive the response in, so we expect
# an exception during the call.
client = capnp.TwoPartyClient(read, traversal_limit_in_words=1)
ref = test_capability_capnp.TestSturdyRefObjectId.new_message(
tag='testInterface')
cap = client.restore(ref)
cap = cap.cast_as(test_capability_capnp.TestInterface)
remote = cap.foo(i=5)
with pytest.raises(capnp.KjException):
response = remote.wait()
def main():
host = sys.argv[1]
client = capnp.TwoPartyClient(host)
city_server = client.bootstrap().cast_as(city_server_capnp.CityServer)
pong_promise = city_server.ping()
pong = pong_promise.wait()
print(pong.pong)
uid = "704-655-624"
session = city_server.getSession(uid).wait().session
session_id = session.sessionId
print(session)
if len(sys.argv) > 2:
arg = sys.argv[2]
new_state = city_server.updateState(session_id, arg).wait().newState
print(new_state)
def main(bind, profile):
CLIENT_ID = '9Hawa4Z_Ki9Z8f5gPgIqR4qspx40'
CLIENT_SECRET = 'slKjOo1YYSDEN22ZrJU904_8EHdq'
capnp_client = capnp.TwoPartyClient(bind)
storeFactory = capnp_client.bootstrap().cast_as(schema.StoreFactory)
print("get store")
jwt = schema.JWT.new_message(payload=get_jwt(CLIENT_ID, CLIENT_SECRET))
promise = storeFactory.createStore(jwt)
store = promise.wait().store
print("validation test:")
validate(store)
print("\nperformances: ")
for n in [1000, 10000, 50000]:
duration = benchark(store, n)
print("set/get {} objects in {} sec".format(n, duration))
