def main():
import getopt
from webpie import HTTPServer
Usage = "python router.py [-p <port>] <config.yaml>"
if not sys.argv[1:] or sys.argv[1] in ("-h", "--help", "-?"):
print(Usage)
sys.exit(2)
opts, args = getopt.getopt(sys.argv[1:], "p:")
opts = dict(opts)
if not args:
print(Usage)
sys.exit(2)
config_file = args[0]
port = opts.get("-p")
if not port:
config = yaml.load(open(config_file, 'r'), Loader=yaml.SafeLoader)
port = config.get("port")
if not port:
print(Usage)
sys.exit(2)
port = int(port)
HTTPServer(port, create_application(config_file)).run()
def reconfigure(self):
#print("MultiServerSubprocess.reconfigure()...")
self.ReconfiguredTime = os.path.getmtime(self.ConfigFile)
self.Config = config = expand(
yaml.load(open(self.ConfigFile, 'r'), Loader=yaml.SafeLoader))
templates = config.get("templates", {})
services = config.get("services", [])
service_list = []
assert isinstance(services, list)
for svc_cfg in services:
#print("svc_cfg:", svc_cfg)
svc = None
if "template" in svc_cfg:
template = templates.get(svc_cfg.get("template", "*"))
if template is not None:
c = {}
c.update(template)
c.update(svc_cfg)
svc_cfg = expand(c)
names = svc_cfg.get("names", [svc_cfg.get("name")])
for name in names:
c = svc_cfg.copy()
c["name"] = name
svc = Service(expand(c), self.Logger)
if svc.Initialized:
service_list.append(svc)
#print("Service", svc, "created and added to the list")
else:
self.log(
f'service "{svc.ServiceName}" failed to initialize - removing from service list'
)
else:
#print("MultiServerSubprocess.reconfigure: svc_cfg:", svc_cfg)
#print("MultiServerSubprocess.reconfigure: expanded:", expand(svc_cfg))
svc = Service(expand(svc_cfg), self.Logger)
if not svc.Initialized:
#print("service not initialzed")
self.log(
f'service "{svc.ServiceName}" failed to initialize - removing from service list'
)
else:
service_list.append(svc)
#print("Service", svc, "created and added to the list")
#print("--------")
names = ",".join(s.Name for s in service_list)
if self.Server is None:
self.Server = HTTPServer.from_config(self.Config,
service_list,
logger=self.Logger)
self.log(f"server created with services: {names}")
else:
self.Server.setServices(service_list)
self.log(f"server reconfigured with services: {names}")
self.Services = service_list
self.log("reconfigured")
class Monitor(object):
def __init__(self, fn, plot_attrs={}):
#
# plot attributes:
#
# label ->
self.FileName = fn
self.Labels = set()
self.Data = [] # [(t, data_dict),]
self.SaveInterval = 1
self.NextSave = self.SaveInterval
self.Server = None
self.PlotAttributes = plot_attrs
def start_server(self, port):
app = App(self, static_location="static", enable_static=True)
self.Server = HTTPServer(port, app)
self.Server.start()
return self.Server
def add(self, t, data=None, **data_args):
if data is None: data = data_args
self.Data.append((t, data.copy()))
for k in data.keys():
self.Labels.add(k)
self.NextSave -= 1
if self.NextSave <= 0:
self.save()
self.NextSave = self.SaveInterval
def data_as_table(self):
labels = list(self.Labels)
rows = []
for t, row in self.Data:
rows.append([t] + [row.get(l) for l in labels])
return ['t'] + labels, rows
def save(self):
labels, rows = self.data_as_table()
with open(self.FileName, "w") as f:
writer = csv.writer(f)
writer.writerow(labels)
for row in rows:
writer.writerow(row)
def reconfigure(self):
self.log("reconfigure...")
self.ReconfiguredTime = os.path.getmtime(self.ConfigFile)
self.Config = config = expand(yaml.load(open(self.ConfigFile, 'r'), Loader=yaml.SafeLoader))
templates = config.get("templates", {})
new_servers = config["servers"]
new_ports = {cfg["port"] for cfg in new_servers}
to_stop = []
for p, s in list(self.ServersByPort.items()):
if not p in new_ports:
to_stop.append(s)
new_lst = []
for cfg in new_servers:
port = cfg["port"]
apps = []
for app_cfg in cfg["apps"]:
if "template" in app_cfg:
template = templates.get(app_cfg.get("template", "*"))
if template is not None:
c = {}
c.update(template)
c.update(app_cfg)
app_cfg = expand(c)
instances = app_cfg.get("instances", [app_cfg.get("instance")])
for instance in instances:
c = app_cfg.copy()
c["instance"] = instance
apps.append(QueuedApplication(expand(c), self.Logger))
else:
apps.append(QueuedApplication(expand(app_cfg), self.Logger))
app_list = ",".join(a.Instance for a in apps)
srv = self.ServersByPort.get(port)
if srv is None:
srv = HTTPServer.from_config(cfg, apps, logger=self.Logger)
srv.start()
self.log(f"server {srv.Port} started with apps: {app_list}")
else:
srv.reconfigureApps(apps)
self.log(f"server {srv.Port} reconfigured with apps: {app_list}")
new_lst.append(srv)
self.Servers = new_lst
self.ServersByPort = {srv.Port:srv for srv in self.Servers}
for s in to_stop:
s.stop()
self.log(f"server {s.Port} stopped")
self.debug("reconfigured")
class Monitor(object):
def __init__(self, fn, title="Monitor", plots=[], metadata={}):
#
# plot_list: [ plot_desc, ... ]
# plot_desc: [ series, ... ]
# series:
# {
# label: "label",
# secondary_axis: <bool>, default: false
# line_width: <float> or 0.0, default: 1.0
# color: <string> default: auto
# marker_style: <string> or null, default: null
# }
#
self.FileName = fn
self.Labels = set()
self.Data = [] # [(t, data_dict), ...]
self.SaveInterval = 1
self.NextSave = self.SaveInterval
self.Server = None
self.PlotDesc = plots
self.Title = title
self.Meta = metadata
def reset(self):
self.Labels = set()
self.Data = []
self.NextSave = self.SaveInterval
def start_server(self, port):
app = App(self, static_location="static", enable_static=True)
self.Server = HTTPServer(port, app, logging=False)
self.Server.start()
return self.Server
def add(self, t, data=None, **data_args):
if data is None: data = data_args
self.Data.append((t, data.copy()))
for k in data.keys():
self.Labels.add(k)
self.NextSave -= 1
if self.NextSave <= 0:
self.save()
self.NextSave = self.SaveInterval
def data_as_columns(self):
labels = list(self.Labels)
n = len(self.Data)
prescale = 1.0 if n < 10000 else 0.1
scaler = 0.0
columns = {c: [] for c in labels}
columns['t'] = []
for i, (t, row) in enumerate(self.Data):
scaler += prescale
if scaler >= 1.0 or i == n - 1:
scaler -= 1.0
columns['t'].append(t)
for l in labels:
columns[l].append(row.get(l))
return columns
def data_as_rows(self):
out = []
for t, dct in self.Data:
d = dct.copy()
d["t"] = t
out.append(d)
return out
def save(self):
return
labels, rows = self.data_as_columns()
with open(self.FileName, "w") as f:
writer = csv.writer(f)
writer.writerow(labels)
for row in rows:
writer.writerow(row)
def start_server(self, port):
app = App(self, static_location="static", enable_static=True)
self.Server = HTTPServer(port, app, logging=False)
self.Server.start()
return self.Server
max_jobs = config.MaxJobs
queue_capacity = config.QueueCapacity
port = config.Port
host = config.Host
registry_port = config.RegistryPort
data_server_url = config.DataServerURL # required
log_file_dir = config.LogFileDir
log_file = None
if log_file_dir:
log_file = LogFile(log_file_dir + "/job_server.log")
source_archive = config.SourceArchive
rs = RegistryServer(registry_port)
authenticator = Authenticator(config)
js = JobServer(host, port, rs, authenticator, data_server_url,
config.BulkTransportPort, queue_capacity, max_jobs,
source_archive, log_file_dir)
application = WebServiceApp(config, rs, authenticator, js)
web_server = HTTPServer(config.WebServerPort, application)
rs.start()
js.start()
web_server.start()
print("Web server port:", config.WebServerPort)
print("Job server port:", config.Port)
with open("job_server.pid", "w") as pidfile:
pidfile.write("%d" % (os.getpid(), ))
web_server.join()
WPHandler.__init__(self, request, app)
def get(self, request, name):
cell = self.App.Cell
value = cell.get(name)
return value if value is not None else 404
def set(self, request, name, value=None):
cell = self.App.Cell
value = cell.set(name, value)
return 200
class App(WPApp):
def __init__(self, cell):
WPApp.__init__(self, Handler)
self.Cell = cell
opts, args = getopt.getopt(sys.argv[1:], "c:")
opts = dict(opts)
config = yaml.load(open(opts["-c"], "r"), Loader=yaml.SafeLoader)
port = int(config["server"]["port"])
link = EtherLink(config["ring"])
cell = MemoryCell(link)
link.init(cell)
link.start()
server = HTTPServer(port, App(cell))
server.start()
server.join()