def check_configured(self):
"""Check if compute package has been configured and that and that the
state of the ReducerControl is not in setup otherwise render setup template.
Check if initial model has been configured, otherwise render setup_model template.
:return: Rendered html template or None
"""
if not self.check_compute_context():
return render_template('setup.html',
client=self.name,
state=ReducerStateToString(
self.control.state()),
logs=None,
refresh=False,
message='Please set the compute package')
if self.control.state() == ReducerState.setup:
return render_template(
'setup.html',
client=self.name,
state=ReducerStateToString(self.control.state()),
logs=None,
refresh=True,
message=
'Warning. Reducer is not base-configured. please do so with config file.'
)
if not self.check_initial_model():
return render_template('setup_model.html',
message="Please set the initial model.")
return None
def check_configured(self):
if not self.control.get_compute_context():
return render_template('setup.html',
client=self.name,
state=ReducerStateToString(
self.control.state()),
logs=None,
refresh=False,
message='')
if self.control.state() == ReducerState.setup:
return render_template(
'setup.html',
client=self.name,
state=ReducerStateToString(self.control.state()),
logs=None,
refresh=True,
message=
'Warning. Reducer is not base-configured. please do so with config file.'
)
if not self.control.get_latest_model():
return render_template('setup_model.html',
message="Please set the initial model.")
return None
def run(self):
threading.Thread(target=self.rest.run, daemon=True).start()
import time
from datetime import datetime
try:
old_state = self.control.state()
t1 = datetime.now()
while True:
time.sleep(1)
if old_state != self.control.state():
delta = datetime.now() - t1
print(
"Reducer in state {} for {} seconds. Entering {} state"
.format(ReducerStateToString(old_state), delta.seconds,
ReducerStateToString(self.control.state())),
flush=True)
t1 = datetime.now()
old_state = self.control.state()
self.control.monitor()
except (KeyboardInterrupt, SystemExit):
print("Exiting..", flush=True)
def transition(self, state):
old_state = self.state.find_one({'state': 'current_state'})
if old_state != state:
return self.state.update({'state': 'current_state'},
{'state': ReducerStateToString(state)},
True)
else:
print("Not updating state, already in {}".format(
ReducerStateToString(state)))
def index():
# logs_fancy = str()
# for log in self.logs:
# logs_fancy += "<p>" + log + "</p>\n"
client = self.name
state = ReducerStateToString(self.control.state())
logs = None
refresh = True
if self.control.state() == ReducerState.setup:
return render_template(
'setup.html',
client=client,
state=state,
logs=logs,
refresh=refresh,
dashboardhost=os.environ["FEDN_DASHBOARD_HOST"],
dashboardport=os.environ["FEDN_DASHBOARD_PORT"])
return render_template(
'index.html',
client=client,
state=state,
logs=logs,
refresh=refresh,
dashboardhost=os.environ["FEDN_DASHBOARD_HOST"],
dashboardport=os.environ["FEDN_DASHBOARD_PORT"])
def index():
client = self.name
state = ReducerStateToString(self.control.state())
logs = None
refresh = True
if self.current_compute_context == None or self.current_compute_context == '':
return render_template(
'setup.html',
client=client,
state=state,
logs=logs,
refresh=False,
message=
'Warning. No compute context is set. please set one with <a href="/context">/context</a>'
)
if self.control.state() == ReducerState.setup:
return render_template(
'setup.html',
client=client,
state=state,
logs=logs,
refresh=refresh,
message=
'Warning. Reducer is not base-configured. please do so with config file.'
)
return render_template('index.html',
client=client,
state=state,
logs=logs,
refresh=refresh)
def start():
if request.method == 'POST':
timeout = request.form.get('timeout', 180)
rounds = int(request.form.get('rounds', 1))
task = (request.form.get('task', ''))
active_clients = request.form.get('active_clients', 2)
clients_required = request.form.get('clients_required', 2)
clients_requested = request.form.get('clients_requested', 8)
latest_model_id = self.control.get_latest_model()
config = {'round_timeout': timeout, 'model_id': latest_model_id,
'rounds': rounds, 'active_clients': active_clients, 'clients_required': clients_required,
'clients_requested': clients_requested, 'task': task}
self.control.instruct(config)
return redirect(url_for('index', message="Sent execution plan."))
else:
# Select rounds UI
rounds = range(1, 100)
latest_model_id = self.control.get_latest_model()
return render_template('index.html', round_options=rounds, latest_model_id=latest_model_id)
client = self.name
state = ReducerStateToString(self.control.state())
logs = None
refresh = False
return render_template('index.html', client=client, state=state, logs=logs, refresh=refresh)
def run(self):
threading.Thread(target=self.rest.run, daemon=True).start()
import time
try:
while True:
time.sleep(1)
print("Reducer in {} state".format(
ReducerStateToString(self.control.state())),
flush=True)
self.control.monitor()
except (KeyboardInterrupt, SystemExit):
print("Exiting..", flush=True)
def index():
not_configured = self.check_configured()
if not_configured:
return not_configured
events = self.control.get_events()
message = request.args.get('message', None)
message_type = request.args.get('message_type', None)
return render_template('events.html',
client=self.name,
state=ReducerStateToString(
self.control.state()),
events=events,
logs=None,
refresh=True,
configured=True,
message=message,
message_type=message_type)
def control():
not_configured = self.check_configured()
if not_configured:
return not_configured
client = self.name
state = ReducerStateToString(self.control.state())
logs = None
refresh = True
try:
self.current_compute_context = self.control.get_compute_context(
)
except:
self.current_compute_context = None
if self.current_compute_context == None or self.current_compute_context == '':
return render_template(
'setup.html',
client=client,
state=state,
logs=logs,
refresh=False,
message=
'No compute context is set. Please set one here <a href="/context">/context</a>'
)
if self.control.state() == ReducerState.setup:
return render_template(
'setup.html',
client=client,
state=state,
logs=logs,
refresh=refresh,
message=
'Warning. Reducer is not base-configured. please do so with config file.'
)
if self.control.state() == ReducerState.monitoring:
return redirect(
url_for('index',
state=state,
refresh=refresh,
message="Reducer is in monitoring state"))
if request.method == 'POST':
timeout = float(request.form.get('timeout', 180))
rounds = int(request.form.get('rounds', 1))
task = (request.form.get('task', ''))
clients_required = request.form.get('clients_required', 1)
clients_requested = request.form.get('clients_requested', 8)
# checking if there are enough clients connected to start!
clients_available = 0
try:
for combiner in self.control.network.get_combiners():
if combiner.allowing_clients():
combiner_state = combiner.report()
nac = combiner_state['nr_active_clients']
clients_available = clients_available + int(nac)
except Exception as e:
pass
if clients_available < clients_required:
return redirect(
url_for(
'index',
state=state,
message=
"Not enough clients available to start rounds.",
message_type='warning'))
validate = request.form.get('validate', False)
if validate == 'False':
validate = False
helper_type = request.form.get('helper', 'keras')
# self.control.statestore.set_framework(helper_type)
latest_model_id = self.control.get_latest_model()
config = {
'round_timeout': timeout,
'model_id': latest_model_id,
'rounds': rounds,
'clients_required': clients_required,
'clients_requested': clients_requested,
'task': task,
'validate': validate,
'helper_type': helper_type
}
import threading
threading.Thread(target=self.control.instruct,
args=(config, )).start()
# self.control.instruct(config)
return redirect(
url_for('index',
state=state,
refresh=refresh,
message="Sent execution plan.",
message_type='SUCCESS'))
else:
seed_model_id = None
latest_model_id = None
try:
seed_model_id = self.control.get_first_model()[0]
latest_model_id = self.control.get_latest_model()
except Exception as e:
pass
return render_template(
'index.html',
latest_model_id=latest_model_id,
compute_package=self.current_compute_context,
seed_model_id=seed_model_id,
helper=self.control.statestore.get_framework(),
validate=True,
configured=True)
client = self.name
state = ReducerStateToString(self.control.state())
logs = None
refresh = False
return render_template('index.html',
client=client,
state=state,
logs=logs,
refresh=refresh,
configured=True)
def status():
return {'state': ReducerStateToString(self.control.state())}
def start():
client = self.name
state = ReducerStateToString(self.control.state())
logs = None
refresh = True
try:
self.current_compute_context = self.control.get_compute_context(
)
except:
self.current_compute_context = None
if self.current_compute_context == None or self.current_compute_context == '':
return render_template(
'setup.html',
client=client,
state=state,
logs=logs,
refresh=False,
message=
'No compute context is set. Please set one here <a href="/context">/context</a>'
)
if self.control.state() == ReducerState.setup:
return render_template(
'setup.html',
client=client,
state=state,
logs=logs,
refresh=refresh,
message=
'Warning. Reducer is not base-configured. please do so with config file.'
)
if request.method == 'POST':
timeout = float(request.form.get('timeout'))
rounds = int(request.form.get('rounds', 1))
task = (request.form.get('task', ''))
clients_required = request.form.get('clients_required', 1)
clients_requested = request.form.get('clients_requested', 8)
#TODO: Enable in UI
validate = request.form.get('validate', True)
latest_model_id = self.control.get_latest_model()
config = {
'round_timeout': timeout,
'model_id': latest_model_id,
'rounds': rounds,
'clients_required': clients_required,
'clients_requested': clients_requested,
'task': task,
'validate': validate
}
self.control.instruct(config)
return redirect(
url_for('index', message="Sent execution plan."))
else:
# Select rounds UI
rounds = range(1, 200)
latest_model_id = self.control.get_latest_model()
return render_template(
'index.html',
round_options=rounds,
latest_model_id=latest_model_id,
compute_package=self.current_compute_context,
helper=self.control.statestore.get_framework(),
validate=True)
client = self.name
state = ReducerStateToString(self.control.state())
logs = None
refresh = False
return render_template('index.html',
client=client,
state=state,
logs=logs,
refresh=refresh)
def control():
""" Main page for round control. Configure, start and stop global training rounds. """
not_configured = self.check_configured()
if not_configured:
return not_configured
client = self.name
state = ReducerStateToString(self.control.state())
logs = None
refresh = True
if self.remote_compute_context:
try:
self.current_compute_context = self.control.get_compute_context(
)
except:
self.current_compute_context = None
else:
self.current_compute_context = "None:Local"
if self.control.state() == ReducerState.monitoring:
return redirect(
url_for('index',
state=state,
refresh=refresh,
message="Reducer is in monitoring state"))
if request.method == 'POST':
timeout = float(request.form.get('timeout', 180))
rounds = int(request.form.get('rounds', 1))
task = (request.form.get('task', ''))
clients_required = request.form.get('clients_required', 1)
clients_requested = request.form.get('clients_requested', 8)
# checking if there are enough clients connected to start!
clients_available = 0
for combiner in self.control.network.get_combiners():
try:
combiner_state = combiner.report()
nac = combiner_state['nr_active_clients']
clients_available = clients_available + int(nac)
except Exception as e:
pass
if clients_available < clients_required:
return redirect(
url_for(
'index',
state=state,
message=
"Not enough clients available to start rounds! "
"check combiner client capacity",
message_type='warning'))
validate = request.form.get('validate', False)
if validate == 'False':
validate = False
helper_type = request.form.get('helper', 'keras')
# self.control.statestore.set_framework(helper_type)
latest_model_id = self.control.get_latest_model()
config = {
'round_timeout': timeout,
'model_id': latest_model_id,
'rounds': rounds,
'clients_required': clients_required,
'clients_requested': clients_requested,
'task': task,
'validate': validate,
'helper_type': helper_type
}
import threading
threading.Thread(target=self.control.instruct,
args=(config, )).start()
# self.control.instruct(config)
return redirect(
url_for('index',
state=state,
refresh=refresh,
message="Sent execution plan.",
message_type='SUCCESS'))
else:
seed_model_id = None
latest_model_id = None
try:
seed_model_id = self.control.get_first_model()[0]
latest_model_id = self.control.get_latest_model()
except Exception as e:
pass
return render_template(
'index.html',
latest_model_id=latest_model_id,
compute_package=self.current_compute_context,
seed_model_id=seed_model_id,
helper=self.control.statestore.get_framework(),
validate=True,
configured=True)
client = self.name
state = ReducerStateToString(self.control.state())
logs = None
refresh = False
return render_template('index.html',
client=client,
state=state,
logs=logs,
refresh=refresh,
configured=True)