def fbp(config, service : grid_capnp.Grid):
conman = common.ConnectionManager()
inp = conman.try_connect(config["in_sr"], cast_as=common_capnp.Channel.Reader, retry_secs=1)
outp = conman.try_connect(config["out_sr"], cast_as=common_capnp.Channel.Writer, retry_secs=1)
try:
if inp and outp and service:
while True:
in_msg = inp.read().wait()
if in_msg.which() == "done":
break
in_ip = in_msg.value.as_struct(common_capnp.IP)
attr = common.get_fbp_attr(in_ip, config["from_attr"])
if attr:
coord = attr.as_struct(geo_capnp.LatLonCoord)
else:
coord = in_ip.content.as_struct(geo_capnp.LatLonCoord)
val = service.closestValueAt(coord).wait().val
out_ip = common_capnp.IP.new_message()
if not config["to_attr"]:
out_ip.content = val
common.copy_fbp_attr(in_ip, out_ip, config["to_attr"], val)
outp.write(value=out_ip).wait()
outp.write(done=None).wait()
except Exception as e:
print("ascii_grid.py ex:", e)
print("ascii_grid.py: exiting FBP component")
def writer():
conMan = common.ConnectionManager()
writer = conMan.try_connect(
"capnp://[email protected]:43513/668ce2c1-f256-466d-99ce-30b01fd2b21b",
cast_as=common_capnp.Writer)
#channel = conMan.try_connect(channel_sr, cast_as=common_capnp.Channel)
#writer = channel.writer().wait().w.as_interface(common_capnp.Writer)
for i in range(1000):
time.sleep(random())
writer.write(value=common_capnp.X.new_message(t="hello_" +
str(i))).wait()
#writer.write(value="hello_" + str(i)).wait()
print("wrote: hello_" + str(i))
def main(serve_bootstrap=True, host="*", port=10000, reg_sturdy_ref=None):
config = {
"port": port,
"host": host,
"reg_sturdy_ref": reg_sturdy_ref,
"serve_bootstrap": str(serve_bootstrap),
"reg_category": "climate",
}
# read commandline args only if script is invoked directly from commandline
if len(sys.argv) > 1 and __name__ == "__main__":
for arg in sys.argv[1:]:
k, v = arg.split("=")
if k in config:
config[k] = v
print(config)
conMan = common.ConnectionManager()
service = Factory()
admin = serv.Admin(service)
service.admin = admin
restorer = common.Restorer()
if config["reg_sturdy_ref"]:
registrator = conMan.try_connect(config["reg_sturdy_ref"],
cast_as=reg_capnp.Registrator)
if registrator:
unreg = registrator.register(
ref=service, categoryId=config["reg_category"]).wait()
print("Registered ", config["name"], "climate service.")
#await unreg.unregister.unregister().a_wait()
else:
print("Couldn't connect to registrator at sturdy_ref:",
config["reg_sturdy_ref"])
addr = config["host"] + (
(":" + str(config["port"])) if config["port"] else "")
if config["serve_bootstrap"].lower() == "true":
server = capnp.TwoPartyServer(addr, bootstrap=restorer)
restorer.port = port if port else server.port
admin_sr = restorer.save(admin)
service_sr = restorer.save(service)
print("admin_sr:", admin_sr)
print("service_sr:", service_sr)
print("restorer_sr:", restorer.sturdy_ref())
else:
capnp.wait_forever()
server.run_forever()
def fbp(config, service: soil_capnp.Service):
conman = common.ConnectionManager()
inp = conman.try_connect(config["in_sr"],
cast_as=common_capnp.Channel.Reader,
retry_secs=1)
outp = conman.try_connect(config["out_sr"],
cast_as=common_capnp.Channel.Writer,
retry_secs=1)
mandatory = json.loads(config["mandatory"])
try:
if inp and outp and service:
while True:
in_msg = inp.read().wait()
if in_msg.which() == "done":
break
in_ip = in_msg.value.as_struct(common_capnp.IP)
attr = common.get_fbp_attr(in_ip, config["from_attr"])
if attr:
coord = attr.as_struct(geo_capnp.LatLonCoord)
else:
coord = in_ip.content.as_struct(geo_capnp.LatLonCoord)
profiles = service.profilesAt(coord, {
"mandatory": mandatory,
"onlyRawData": False
}).wait().profiles
if len(profiles) > 0:
profile = profiles[0]
out_ip = common_capnp.IP.new_message()
if not config["to_attr"]:
out_ip.content = profile
common.copy_fbp_attr(in_ip, out_ip, config["to_attr"],
profile)
outp.write(value=out_ip).wait()
outp.write(done=None).wait()
except Exception as e:
print("sqlite_soil_data_service.py ex:", e)
print("dwd_germany_service.py: exiting FBP component")
def test_climate_service():
conMan = common.ConnectionManager()
#restorer = conMan.try_connect("capnp://[email protected]:10000", cast_as=persistence_capnp.Restorer)
#service = conMan.try_connect("capnp://[email protected]:10000/6feaf299-d620-430b-9189-36dfccf48b3a", cast_as=climate_data_capnp.CSVTimeSeriesFactory)
service = conMan.try_connect(
"capnp://[email protected]:36541/7555fd1a-e413-4ec7-be5d-8f3a94825b3c",
cast_as=climate_data_capnp.Service)
#timeseries = conMan.try_connect("capnp://[email protected]:10000/8e7961c5-bd16-4c1d-86fd-8347dc46185e", cast_as=climate_data_capnp.TimeSeries)
#unsave = conMan.try_connect("capnp://[email protected]:10000/ac544d7b-1f82-4bf8-9adb-cf586ae46287", cast_as=common_capnp.Action)
#4e4fe3fb-791a-4a26-9ae1-1ce52093bda5' row: 340/col: 288
try:
print(service.info().wait())
except Exception as e:
print(e)
p = psutil.Process()
for ds in service.getAvailableDatasets().wait().datasets:
ds_data = ds.data
print(ds_data.info().wait())
print(ds.meta)
#tss = []
for lat in range(45, 55):
for lon in range(8, 15):
ts = ds_data.closestTimeSeriesAt({
"lat": lat,
"lon": lon
}).wait().timeSeries
ts.data().wait()
#tss.append(ts)
print(ts.info().wait())
#print(p.memory_percent())
#unsave = conMan.try_connect("capnp://[email protected]:10000/49ec71b8-a525-4c38-b137-58e1eafc0c1c", cast_as=common_capnp.Action)
#unsave.do().wait()
with open("../../data/climate/climate-iso.csv", "r") as _:
csv_data = _.read()
res = service.create(csvData=csv_data, config={}).wait()
print()
common_capnp = capnp.load(str(PATH_TO_CAPNP_SCHEMAS / "common.capnp"), imports=abs_imports)
geo_capnp = capnp.load(str(PATH_TO_CAPNP_SCHEMAS / "geo.capnp"), imports=abs_imports)
#------------------------------------------------------------------------------
config = {
"from_name": "utm32n",
"to_name": "latlon",
"to_attr": None, #"latlon",
"from_attr": None,
"in_sr": None, # geo.LatLonCoord | geo.UTMCoord | geo.GKCoord
"out_sr": None # geo.LatLonCoord | geo.UTMCoord | geo.GKCoord
}
common.update_config(config, sys.argv, print_config=True, allow_new_keys=False)
conman = common.ConnectionManager()
inp = conman.try_connect(config["in_sr"], cast_as=common_capnp.Channel.Reader, retry_secs=1)
outp = conman.try_connect(config["out_sr"], cast_as=common_capnp.Channel.Writer, retry_secs=1)
from_type = geo.name_to_struct_type(config["from_name"])
try:
if inp and outp:
while True:
msg = inp.read().wait()
# check for end of data from in port
if msg.which() == "done":
break
in_ip = msg.value.as_struct(common_capnp.IP)
attr = common.get_fbp_attr(in_ip, config["from_attr"])
def init_and_run_fbp_component(component_cap,
name_to_in_ports={},
name_to_out_ports={},
host="*",
port=None,
serve_bootstrap=True,
restorer=None,
conman=None,
run_before_enter_eventloop=None,
eventloop_wait_forever=True):
host = host if host else "*"
if not conman:
conman = common.ConnectionManager()
if not restorer:
restorer = common.Restorer()
def connect_ports(side, name_to_ports, port_type):
for name, data in name_to_ports.items():
# only connect via sr if port is not already containing a capability
if "sr" in data and data["port"] is None:
try:
print("trying to connect", side, "port:", name, " via sr:",
data["sr"])
data["port"] = conman.try_connect(data["sr"],
cast_as=port_type,
retry_secs=1)
except Exception as e:
print("Error connecting to", side, "port:", name,
". Exception:", e)
addr = host + ((":" + str(port)) if port else "")
if serve_bootstrap:
server = capnp.TwoPartyServer(addr, bootstrap=restorer)
restorer.port = port if port else server.port
connect_ports("in", name_to_in_ports, common_capnp.Reader)
connect_ports("out", name_to_out_ports, common_capnp.Writer)
else:
connect_ports("in", name_to_in_ports, common_capnp.Reader)
connect_ports("out", name_to_out_ports, common_capnp.Writer)
if run_before_enter_eventloop:
run_before_enter_eventloop()
if eventloop_wait_forever:
capnp.wait_forever()
ips = []
for name, pd in name_to_in_ports.items():
ops.append({"name": name, "port": pd["port"]})
ops = []
for name, pd in name_to_out_ports.items():
ops.append({"name": name, "port": pd["port"]})
component_cap.setupPorts(inPorts=ips, outPorts=ops).wait()
if run_before_enter_eventloop:
run_before_enter_eventloop()
if eventloop_wait_forever:
server.run_forever()
def test_registry():
conMan = common.ConnectionManager()
#monica = conMan.try_connect("capnp://[email protected]:40867/16e65116-76d5-44ae-b3e6-65738bcb3b31", cast_as=model_capnp.EnvInstance)
#print(monica.info().wait())
#restorer = conMan.try_connect("capnp://[email protected]:10000", cast_as=persistence_capnp.Restorer)
#climate = conMan.try_connect("capnp://[email protected]:37203/6b57e75b-dee3-4882-90ae-731a679a3653", cast_as=climate_data_capnp.Service)
#registry = conMan.try_connect("capnp://[email protected]:38955/5681fdaa-4d7f-4ea9-876e-e0566255486c", cast_as=registry_capnp.Registry)
#registrar = conMan.try_connect("capnp://[email protected]:36501/2d997d95-95ac-400e-aeb7-c439b58b77a5", cast_as=registry_capnp.Registrar)
#admin = conMan.try_connect("capnp://[email protected]:37087/6a9dea44-d00e-4a48-adf3-6c83b24cc5fa", cast_as=service_capnp.Admin)
#print(service_admin.info().wait())
#grid = conMan.try_connect("capnp://[email protected]:39875/2fa47702-f112-4628-b72c-7754e457d3a2", cast_as=grid_capnp.Grid)
#value = grid.valueAt(coord={'lat': 50.02045903295569, 'lon': 8.449222632820296}).wait()
writer = conMan.try_connect(
"capnp://[email protected]:44735/2c7554bc-5112-461d-a478-e95cbc905d37",
cast_as=common_capnp.Writer)
writer.write(value=monica_state_capnp.ICData.new_message(
init={"syncDate": {
"day": 1,
"month": 2,
"year": 2022
}})).wait()
print("bla")
# test channel
#channel_sr = "capnp://[email protected]:37505/6c25454e-4ef9-4659-9c94-341bdd827df5"
def writer():
conMan = common.ConnectionManager()
writer = conMan.try_connect(
"capnp://[email protected]:43513/668ce2c1-f256-466d-99ce-30b01fd2b21b",
cast_as=common_capnp.Writer)
#channel = conMan.try_connect(channel_sr, cast_as=common_capnp.Channel)
#writer = channel.writer().wait().w.as_interface(common_capnp.Writer)
for i in range(1000):
time.sleep(random())
writer.write(value=common_capnp.X.new_message(t="hello_" +
str(i))).wait()
#writer.write(value="hello_" + str(i)).wait()
print("wrote: hello_" + str(i))
#writer.write(value=common_capnp.X.new_message(t="world")).wait()
#print("wrote value:", "hello", "world")
Thread(target=writer).start()
reader = conMan.try_connect(
"capnp://[email protected]:34307/40daafc3-490b-4b27-a84f-6ee4c111b352",
cast_as=common_capnp.Reader)
#channel = conMan.try_connect(channel_sr, cast_as=common_capnp.Channel)
#reader = channel.reader().wait().r.as_interface(common_capnp.Reader)
for i in range(1000):
time.sleep(random())
print("read:", reader.read().wait().value.as_struct(common_capnp.X).t)
#print("read:", reader.read().wait().value.as_text())
#print(reader.read().wait().value.as_struct(common_capnp.X).t)
#print("read value:", value)
soil = conMan.try_connect(
"capnp://[email protected]:39341/9c15ad6f-0778-4bea-b91e-b015453188b9",
cast_as=soil_data_capnp.Service)
ps = soil.profilesAt(coord={
'lat': 50.02045903295569,
'lon': 8.449222632820296
},
query={
"mandatory":
["soilType", "organicCarbon", "rawDensity"]
}).wait()
print(ps)
#cap_holder = common.IdentifiableHolder(service, lambda: print("deleted capholder"))
#res = registrar.register(cap=cap_holder, regName="dwd-klima", categoryId="climate").wait()
res = registrar.register(cap=service,
regName="dwd-klima",
categoryId="climate").wait()
climate_entries = registry.entries("climate").wait().entries
for ent in climate_entries:
print(ent.name, " ref.info:") #, ent.ref.info().wait())
service_again = ent.ref.cast_as(climate_data_capnp.Service)
return
try:
print(registrar.info().wait())
except Exception as e:
print(e)
#unsave = conMan.try_connect("capnp://[email protected]:10000/49ec71b8-a525-4c38-b137-58e1eafc0c1c", cast_as=common_capnp.Action)
#unsave.do().wait()
with open("../../data/climate/climate-iso.csv", "r") as _:
csv_data = _.read()
res = service.create(csvData=csv_data, config={}).wait()
print()
def fbp(config : dict, service : ccdi.Service):
conman = common.ConnectionManager()
inp = conman.try_connect(config["in_sr"], cast_as=common_capnp.Channel.Reader, retry_secs=1)
outp = conman.try_connect(config["out_sr"], cast_as=common_capnp.Channel.Writer, retry_secs=1)
mode = config["mode"]
def iso_to_cdate(iso_date_str):
ds = iso_date_str.split("-")
return {"year": int(ds[0]), "month": int(ds[1]), "day": int(ds[2])}
try:
if inp and outp and service:
dataset : csv_based.Dataset = service.getAvailableDatasets().wait().datasets[0].data
while True:
in_msg = inp.read().wait()
if in_msg.which() == "done":
break
in_ip = in_msg.value.as_struct(common_capnp.IP)
attr = common.get_fbp_attr(in_ip, config["latlon_attr"])
if attr:
coord = attr.as_struct(geo_capnp.LatLonCoord)
else:
coord = in_ip.content.as_struct(geo_capnp.LatLonCoord)
start_date = common.get_fbp_attr(in_ip, config["start_date_attr"]).as_text()
end_date = common.get_fbp_attr(in_ip, config["end_date_attr"]).as_text()
timeseries_p : csv_based.TimeSeries = dataset.closestTimeSeriesAt(coord).timeSeries
timeseries = timeseries_p.subrange(iso_to_cdate(start_date), iso_to_cdate(end_date)).wait().timeSeries
res = timeseries
if mode == "sturdyref":
res = timeseries.save().wait()
elif mode == "capability":
res = timeseries
elif mode == "data":
res = climate_capnp.TimeSeriesData.new_message()
res.isTransposed = False
header = timeseries.header().header
se_date = timeseries.range()
resolution = timeseries.resolution().resolution
res.data = timeseries.data().wait().data
res.header = header
se_date = se_date
res.startDate = se_date.startDate
res.endDate = se_date.endDate
res.resolution = resolution
#print(res.data().wait())
out_ip = common_capnp.IP.new_message()
if not config["to_attr"]:
out_ip.content = res
common.copy_fbp_attr(in_ip, out_ip, config["to_attr"], res)
outp.write(value=out_ip).wait()
outp.write(done=None).wait()
except Exception as e:
print("dwd_germany_service.py ex:", e)
print("dwd_germany_service.py: exiting FBP component")