def __init__(self, config):
if config['secure']:
raise Exception(
"NYI - Monitor does not support secure mode yet. Use builtin monitor in combiner mode"
)
self.name = config['name']
channel = grpc.insecure_channel(config['host'] + ":" +
str(config['port']))
self.connection = rpc.ConnectorStub(channel)
print("Client: {} connected to {}:{}".format(self.name, config['host'],
config['port']),
flush=True)
# Connect to MongoDB
try:
self.mdb = connect_to_mongodb()
self.collection = self.mdb['status']
except Exception as e:
print("FAILED TO CONNECT TO MONGO, {}".format(e), flush=True)
self.collection = None
raise
threading.Thread(target=self.__listen_to_status_stream,
daemon=True).start()
def _connect(self, client_config):
"""Connect to assigned combiner.
Parameters
----------
client_config : dict
A dictionary with connection information and settings
for the assigned combiner.
"""
# TODO use the client_config['certificate'] for setting up secure comms'
if client_config['certificate']:
import base64
cert = base64.b64decode(client_config['certificate']) # .decode('utf-8')
credentials = grpc.ssl_channel_credentials(root_certificates=cert)
channel = grpc.secure_channel("{}:{}".format(client_config['host'], str(client_config['port'])),
credentials)
else:
channel = grpc.insecure_channel("{}:{}".format(client_config['host'], str(client_config['port'])))
self.channel = channel
self.connection = rpc.ConnectorStub(channel)
self.orchestrator = rpc.CombinerStub(channel)
self.models = rpc.ModelServiceStub(channel)
print("Client: {} connected {} to {}:{}".format(self.name,
"SECURED" if client_config['certificate'] else "INSECURE",
client_config['host'], client_config['port']), flush=True)
print("Client: Using {} compute package.".format(client_config["package"]))
def allowing_clients(self):
"""
:return:
"""
channel = Channel(self.address, self.port,
self.certificate).get_channel()
connector = rpc.ConnectorStub(channel)
request = fedn.ConnectionRequest()
try:
response = connector.AcceptingClients(request)
except grpc.RpcError as e:
if e.code() == grpc.StatusCode.UNAVAILABLE:
raise CombinerUnavailableError
else:
raise
if response.status == fedn.ConnectionStatus.NOT_ACCEPTING:
return False
if response.status == fedn.ConnectionStatus.ACCEPTING:
return True
if response.status == fedn.ConnectionStatus.TRY_AGAIN_LATER:
return False
return False
def allowing_clients(self):
channel = Channel(self.address, self.port, self.certificate).get_channel()
connector = rpc.ConnectorStub(channel)
request = fedn.ConnectionRequest()
response = connector.AcceptingClients(request)
if response.status == fedn.ConnectionStatus.NOT_ACCEPTING:
return False
if response.status == fedn.ConnectionStatus.ACCEPTING:
return True
if response.status == fedn.ConnectionStatus.TRY_AGAIN_LATER:
return False
return False
def __init__(self, config):
self.state = None
self.error_state = False
from fedn.common.net.connect import ConnectorClient, Status
self.connector = ConnectorClient(config['discover_host'],
config['discover_port'],
config['token'],
config['name'],
config['preferred_combiner'],
config['client_id'],
secure=config['secure'],
preshared_cert=config['preshared_cert'],
verify_cert=config['verify_cert'])
self.name = config['name']
import time
dirname = time.strftime("%Y%m%d-%H%M%S")
self.run_path = os.path.join(os.getcwd(), dirname)
os.mkdir(self.run_path)
from fedn.utils.logger import Logger
self.logger = Logger(to_file=config['logfile'],file_path=self.run_path)
self.started_at = datetime.now()
self.logs = []
client_config = {}
print("Asking for assignment",flush=True)
import time
while True:
status, response = self.connector.assign()
if status == Status.TryAgain:
time.sleep(5)
continue
if status == Status.Assigned:
client_config = response
break
time.sleep(5)
print(".", end=' ', flush=True)
print("Got assigned!", flush=True)
# TODO use the client_config['certificate'] for setting up secure comms'
if client_config['certificate']:
import base64
cert = base64.b64decode(client_config['certificate']) # .decode('utf-8')
credentials = grpc.ssl_channel_credentials(root_certificates=cert)
channel = grpc.secure_channel("{}:{}".format(client_config['host'], str(client_config['port'])),
credentials)
else:
channel = grpc.insecure_channel("{}:{}".format(client_config['host'], str(client_config['port'])))
self.connection = rpc.ConnectorStub(channel)
self.orchestrator = rpc.CombinerStub(channel)
self.models = rpc.ModelServiceStub(channel)
print("Client: {} connected {} to {}:{}".format(self.name,
"SECURED" if client_config['certificate'] else "INSECURE",
client_config['host'], client_config['port']), flush=True)
if config['remote_compute_context']:
from fedn.common.control.package import PackageRuntime
pr = PackageRuntime(os.getcwd(), os.getcwd())
retval = None
tries = 10
while tries > 0:
retval = pr.download(config['discover_host'], config['discover_port'], config['token'])
if retval:
break
time.sleep(60)
print("No compute package available... retrying in 60s Trying {} more times.".format(tries),flush=True)
tries -= 1
if retval:
if not 'checksum' in config:
print("\nWARNING: Skipping security validation of local package!, make sure you trust the package source.\n",flush=True)
else:
checks_out = pr.validate(config['checksum'])
if not checks_out:
print("Validation was enforced and invalid, client closing!")
self.error_state = True
return
if retval:
pr.unpack()
self.dispatcher = pr.dispatcher(self.run_path)
try:
print("Running Dispatcher for entrypoint: startup", flush=True)
self.dispatcher.run_cmd("startup")
except KeyError:
pass
else:
# TODO: Deprecate
dispatch_config = {'entry_points':
{'predict': {'command': 'python3 predict.py'},
'train': {'command': 'python3 train.py'},
'validate': {'command': 'python3 validate.py'}}}
dispatch_dir = os.getcwd()
from_path = os.path.join(os.getcwd(),'client')
from distutils.dir_util import copy_tree
copy_tree(from_path, run_path)
self.dispatcher = Dispatcher(dispatch_config, self.run_path)
self.lock = threading.Lock()
if 'model_type' in client_config.keys():
self.helper = get_helper(client_config['model_type'])
if not self.helper:
print("Failed to retrive helper class settings! {}".format(client_config),flush=True)
threading.Thread(target=self._send_heartbeat, daemon=True).start()
threading.Thread(target=self.__listen_to_model_update_request_stream, daemon=True).start()
threading.Thread(target=self.__listen_to_model_validation_request_stream, daemon=True).start()
self.state = ClientState.idle
def __init__(self, config):
from fedn.common.net.connect import ConnectorClient, Status
self.connector = ConnectorClient(config['discover_host'],
config['discover_port'],
config['token'],
config['name'],
config['preferred_combiner'],
config['client_id'],
secure=config['secure'],
preshared_cert=['preshared_cert'],
verify_cert=config['verify_cert'])
self.name = config['name']
self.started_at = datetime.now()
self.logs = []
client_config = {}
print("Asking for assignment", flush=True)
import time
while True:
status, response = self.connector.assign()
if status == Status.TryAgain:
time.sleep(5)
continue
if status == Status.Assigned:
client_config = response
break
time.sleep(5)
print(".", end=' ', flush=True)
print("Got assigned!", flush=True)
# TODO use the client_config['certificate'] for setting up secure comms'
if client_config['certificate']:
import base64
cert = base64.b64decode(
client_config['certificate']) # .decode('utf-8')
credentials = grpc.ssl_channel_credentials(root_certificates=cert)
channel = grpc.secure_channel(
"{}:{}".format(client_config['host'],
str(client_config['port'])), credentials)
else:
channel = grpc.insecure_channel("{}:{}".format(
client_config['host'], str(client_config['port'])))
self.connection = rpc.ConnectorStub(channel)
self.orchestrator = rpc.CombinerStub(channel)
self.models = rpc.ModelServiceStub(channel)
print("Client: {} connected {} to {}:{}".format(
self.name,
"SECURED" if client_config['certificate'] else "INSECURE",
client_config['host'], client_config['port']),
flush=True)
if config['remote_compute_context']:
from fedn.common.control.package import PackageRuntime
pr = PackageRuntime(os.getcwd(), os.getcwd())
retval = None
tries = 10
while tries > 0:
retval = pr.download(config['discover_host'],
config['discover_port'], config['token'])
if retval:
break
time.sleep(60)
print(
"No compute package availabe... retrying in 60s Trying {} more times."
.format(tries),
flush=True)
tries -= 1
if retval:
pr.unpack()
self.dispatcher = pr.dispatcher()
try:
self.dispatcher.run_cmd("startup")
except KeyError:
print("No startup code present. skipping")
else:
# TODO: Deprecate
dispatch_config = {
'entry_points': {
'predict': {
'command': 'python3 predict.py'
},
'train': {
'command': 'python3 train.py'
},
'validate': {
'command': 'python3 validate.py'
}
}
}
dispatch_dir = os.getcwd()
self.dispatcher = Dispatcher(dispatch_config, dispatch_dir)
self.lock = threading.Lock()
if 'model_type' in client_config.keys():
self.helper = get_helper(client_config['model_type'])
if not self.helper:
print("Failed to retrive helper class settings! {}".format(
client_config),
flush=True)
threading.Thread(target=self._send_heartbeat, daemon=True).start()
threading.Thread(target=self.__listen_to_model_update_request_stream,
daemon=True).start()
threading.Thread(
target=self.__listen_to_model_validation_request_stream,
daemon=True).start()
self.state = ClientState.idle
def __init__(self, config):
from fedn.common.net.connect import ConnectorClient, Status
self.connector = ConnectorClient(config['discover_host'],
config['discover_port'],
config['token'],
config['name'],
config['client_id'],
secure=config['secure'],
preshared_cert=['preshared_cert'],
verify_cert=config['verify_cert'])
self.name = config['name']
self.started_at = datetime.now()
self.logs = []
client_config = {}
print("Asking for assignment")
import time
while True:
status, response = self.connector.assign()
#print(status,response,flush=True)
if status == Status.TryAgain:
time.sleep(5)
continue
if status == Status.Assigned:
client_config = response
break
time.sleep(5)
print(".", end=' ', flush=True)
# print("try to reconnect to REDUCER", flush=True)
# connect_config = None
print("Got assigned!", flush=True)
tries = 180
# while True:
# connect_config = {'host': 'combiner', 'port': 12080}
# TODO REMOVE ONLY FOR TESTING (only used for partial restructuring)
# import os
repo_config = {
'storage_access_key': os.environ['FEDN_MINIO_ACCESS_KEY'],
'storage_secret_key': os.environ['FEDN_MINIO_SECRET_KEY'],
'storage_bucket': 'models',
'storage_secure_mode': False,
'storage_hostname': os.environ['FEDN_MINIO_HOST'],
'storage_port': int(os.environ['FEDN_MINIO_PORT'])
}
# repo_config, _ = self.controller.get_config()
self.bucket_name = repo_config['storage_bucket']
# TODO use the client_config['certificate'] for setting up secure comms'
if client_config['certificate']:
import base64
cert = base64.b64decode(
client_config['certificate']) # .decode('utf-8')
credentials = grpc.ssl_channel_credentials(root_certificates=cert)
channel = grpc.secure_channel(
"{}:{}".format(client_config['host'],
str(client_config['port'])), credentials)
else:
channel = grpc.insecure_channel("{}:{}".format(
client_config['host'], str(client_config['port'])))
self.connection = rpc.ConnectorStub(channel)
self.orchestrator = rpc.CombinerStub(channel)
self.models = rpc.ModelServiceStub(channel)
print("Client: {} connected {} to {}:{}".format(
self.name,
"SECURED" if client_config['certificate'] else "INSECURE",
client_config['host'], client_config['port']),
flush=True)
# TODO REMOVE OVERRIDE WITH CONTEXT FETCHED
dispatch_config = {
'entry_points': {
'predict': {
'command': 'python3 predict.py'
},
'train': {
'command': 'python3 train.py'
},
'validate': {
'command': 'python3 validate.py'
}
}
}
# TODO REMOVE OVERRIDE WITH CONTEXT FETCHED
dispatch_dir = os.getcwd()
self.dispatcher = Dispatcher(dispatch_config, dispatch_dir)
self.lock = threading.Lock()
threading.Thread(target=self._send_heartbeat, daemon=True).start()
threading.Thread(target=self.__listen_to_model_update_request_stream,
daemon=True).start()
threading.Thread(
target=self.__listen_to_model_validation_request_stream,
daemon=True).start()
self.state = ClientState.idle