def RF_train_model(self, experiment_id, task_id):
"""Computes local histogram data for given information. Assumes RF_fetch_model is previously called
and that the following fields have been set by the server process in the model-configuration-file:
- current_condition_list
- current_feature_list
- random_state
This function then writes the result into the local model under the attribute model_update
NOTE: Function assumes positive-label=1, negative-label=0, need to incorporate how we can pass this information to the worker.
"""
logging.info("Training...%s", self.experiment_id)
batch = self.batch
histograms = utils.RF_create_histograms(batch, self.config, self.model)
self.model.model_update = histograms # store as string
_, self.stub, _ = grpc_util.get_grpc_connection(
grpc_function='train_model_response',
request=globalserver_pb2.DefaultRequest(
client=self.client,
secret=self.secret,
task_id=task_id,
experiment_id=experiment_id))
logging.info("Training finished. %s", self.experiment_id)
gc.collect()
return True
def P2P_train_model(self, experiment_id, task_id):
logging.info("Training...%s", self.experiment_id)
generator = self.data_generator(
"train",
preprocessing=self.preprocessing_function,
config=self.config,
client=self.client)
train_X, train_y = next(generator)
train_data_local = xgb.DMatrix(train_X, label=train_y)
train_params_dict = self.config['compile']['model_params'].copy()
train_params_dict['nthread'] = self.config['training'].get(
'nthread', -1)
train_params_dict['verbosity'] = self.config['training'].get(
'verbosity', 0)
self.model = xgb.train(
train_params_dict,
train_data_local,
num_boost_round=self.config['training']['client_steps_per_round'],
xgb_model=self.model)
_, self.stub, _ = grpc_util.get_grpc_connection(
grpc_function='train_model_response',
request=globalserver_pb2.DefaultRequest(
client=self.client,
secret=self.secret,
task_id=task_id,
experiment_id=experiment_id))
logging.info("Training finished. %s", self.experiment_id)
gc.collect()
return True
def fetch_model(self, experiment_id, task_id, protocol):
logging.info(f"Parsing Model...%s", experiment_id)
_, self.stub, responses = grpc_util.get_grpc_connection(
grpc_function='fetch_model_request',
request=globalserver_pb2.DefaultRequest(
client=self.client,
task_id=task_id,
secret=self.secret,
experiment_id=experiment_id))
for row in responses: # pseudo stream
load_model = getattr(
self, protocol + "_load_model"
) #todo split into load config and compile model. In load config set custom config!
load_model(model=row)
logging.info(f"Model parsed...%s", experiment_id)
self._set_dataset()
self._set_custom_training_config() #todo allow to change everything
self._set_preprocessing()
tf.keras.backend.clear_session()
gc.collect()
return True
def start_working(self):
server_ok = True
while server_ok:
server_ok, self.stub, response = grpc_util.get_grpc_connection(
grpc_function='get_task_request',
request=globalserver_pb2.TaskRequest(
client=self.client,
secret=self.secret,
experiment_id=self.experiment_id),
stub=self.stub)
# bad response from server
if not response.ok:
logging.error(f"Message <{response.message}>")
raise Exception("Bad Response From Server")
# good response from server
elif response.task in config['VALID_TASKS']:
self.run_task(response.task,
experiment_id=self.experiment_id,
task_id=response.task_id,
protocol=response.protocol)
# No task to do or unknown task
else:
# logging.debug(f"Nothing to do with <{response.task}> ...")
time.sleep(config['CONTROLLER_IDLE_SLEEP'])
def P2P_send_model(self, experiment_id, task_id):
logging.info("Sending model...%s", self.experiment_id)
iterator = utils.stream_model_P2P(experiment_id,
self.model,
self.client,
self.secret,
task_id=task_id)
_, self.stub, _ = grpc_util.get_grpc_connection(
grpc_function='send_model_update_response', request=iterator)
logging.info("Sending model finished%s", self.experiment_id)
gc.collect()
return True
def __init__(self, client, secret, experiment_id):
logging.info(f"Initialize worker...")
self.experiment_id = experiment_id
self.client = client
self.secret = secret
self.model = None
self.global_weights = None
self.config = {}
self.data_generator = None
self.preprocessing_function = None
# This is only needed until memory leak is fixed, to read memory usage
self.process = psutil.Process(os.getpid())
self.batch = None # todo ugly
_, self.stub, _ = grpc_util.get_grpc_connection(
grpc_function='test_connection',
request=globalserver_pb2.DefaultRequest(
client=client, secret=secret, experiment_id=experiment_id))
self.dataset_metadata = {}
logging.info(f"GRPC Connection established...")
def NN_train_model(self, experiment_id, task_id):
logging.info("Training...%s", self.experiment_id)
self.model.fit(
self.data_generator("train",
preprocessing=self.preprocessing_function,
config=self.config,
client=self.client),
epochs=self.config['training'].get("epochs", 1),
verbose=self.config['training'].get("verbose", 0),
callbacks=self.config['training'].get("callback", []),
shuffle=self.config['training'].get("shuffle", True),
class_weight={
int(key): value
for key, value in self.config['training'].get(
"class_weight").items()
} if self.config['training'].get("class_weight", None) else None,
initial_epoch=self.config['training'].get("initial_epoch", 0),
steps_per_epoch=self.config['training'].get("steps_per_epoch", 12),
max_queue_size=self.config['training'].get("max_queue_size", 10),
workers=1, # self.config['training'].get("workers", 1),
use_multiprocessing=self.config['training'].get(
"use_multiprocessing", False),
)
_, self.stub, _ = grpc_util.get_grpc_connection(
grpc_function='train_model_response',
request=globalserver_pb2.DefaultRequest(
client=self.client,
secret=self.secret,
task_id=task_id,
experiment_id=experiment_id))
logging.info("Training finished. %s", self.experiment_id)
tf.keras.backend.clear_session()
gc.collect()
return True
def _send_loss(self, experiment_id, task_id, protocol, data_type):
predict = getattr(self, f"{protocol}_predict")
y_pred, y_true = predict(data_type)
performance = utils.get_loss(
y_pred=y_pred,
y_true=y_true,
tf_metrics=self.config['training'].get('tfmetrics', []),
sk_metrics=self.config['training'].get('skmetrics', []))
logging.info(f"Client: {performance}, %s", self.experiment_id)
_, self.stub, _ = grpc_util.get_grpc_connection(
grpc_function=f'send_{data_type}_loss_response',
request=globalserver_pb2.Loss(client=self.client,
secret=self.secret,
task_id=task_id,
experiment_id=experiment_id,
loss=json.dumps(
performance,
default=utils.default)))
logging.info("Loss sent...%s", self.experiment_id)
tf.keras.backend.clear_session()
gc.collect()
return True