def receive_model(self, message, verbose=True):
msg = json.loads(message['data'])
if (msg is not None):
if (msg['type'] == 'transact'):
return utils.ipfs2keras(msg['model_addr']), msg
elif (msg['type'] == 'log'):
if (verbose):
output = "Worker:" + msg['worker_id'][-5:]
output += " - Epoch " + str(
msg['epoch_id']) + " of " + str(msg['num_epochs'])
output += " - Valid Loss: " + str(msg['eval_loss'])[0:8]
print(output)
# Figure out of we should tell this worker to quit.
parent_model = msg['parent_model']
worker_id = msg['worker_id']
num_epochs = msg['num_epochs']
epoch_id = msg['epoch_id']
progress = self.update_progress(parent_model, worker_id,
num_epochs, epoch_id)
max_progress = self.max_progress(parent_model)
if progress < max_progress * 0.75:
quit = {}
quit['op_code'] = 'quit'
self.publish(self.spec['train_channel'] + ':' + worker_id,
quit)
def fit_worker(self, message):
decoded = json.loads(message['data'])
if(decoded['framework'] == 'keras'):
model = utils.ipfs2keras(decoded['model_addr'])
try:
np_strings = json.loads(self.api.cat(decoded['data_addr']))
except NotImplementedError:
raise NotImplementedError("The IPFS API only supports Python 3.6. Please modify your environment.")
input, target, valid_input, valid_target = list(map(lambda x: self.deserialize_numpy(x),np_strings))
train_channel = decoded['train_channel']
self.learner_callback = OutputPipe(
id=self.id,
publisher=self.publish,
channel=train_channel,
epochs=decoded['epochs'],
model_addr=decoded['model_addr'],
model=model
)
args = (self.train_meta, train_channel + ':' + self.id)
monitor_thread = threading.Thread(target=self.listen_to_channel,
args=args)
monitor_thread.start()
print('training model')
model.fit(
input,
target,
batch_size=decoded['batch_size'],
validation_data=(valid_input, valid_target),
verbose=False,
epochs=decoded['epochs'],
callbacks=[self.learner_callback]
)
print('done')
else:
raise NotImplementedError("Only compatible with Keras at the moment")
def added_local_data_model(self, info):
task_addr = info['task']
task_info = self.api.get_json(task_addr)
task_name = info['name']
model_addr = info['model']
data_dir = task_info['data_dir']
name = task_info['name']
creator = info['creator']
print(
f'FOUND NEW MODEL: {task_addr}, {model_addr}, {data_dir}, {name}, {creator}'
)
if os.path.exists(f'data/{data_dir}') and creator != self.id:
model = utils.ipfs2keras(model_addr)
input = None
target = None
for filename in os.listdir(f'data/{data_dir}'):
temp_data = np.load(f'data/{data_dir}/{filename}')
temp_input = temp_data['x_train']
temp_target = temp_data['y_train']
if input is None:
input = temp_input
else:
input = np.append(input, temp_input)
input = np.reshape(input, (-1, 28, 28))
if target is None:
target = temp_target
else:
target = np.append(target, temp_target)
# TODO specifically mnist?!?!?!?!?!?
input = input.reshape(input.shape[0], 784)
input = input.astype('float32')
input /= 255
target = keras.utils.to_categorical(target, 10)
self.train_model(model, input, name, taraget, task_name, task_addr)
else:
print("Can't train your own model so soon!!!!!")
def added_adapter_model(self, info):
task_addr = info['task']
task_info = self.api.get_json(task_addr)
task_name = info['name']
model_addr = info['model']
adapter = task_info['adapter']
name = task_info['name']
creator = info['creator']
model = utils.ipfs2keras(model_addr)
utils.save_adapter(adapter)
import grid.adapters.adapter as grid_adapter
print('load next input')
n_test, n_target = grid_adapter.next_input()
self.train_model(model, n_test, n_target, name, task_name, task_addr)
def added_model(self, info):
info = self.api.get_json(info['data'])
task_addr = info['task']
task_name = info['name']
model_addr = info['model']
task_info = self.api.get_json(task_addr)
data_dir = task_info['data_dir']
name = task_info['name']
creator = info['creator']
print(f'FOUND NEW MODEL: {task_addr}, {model_addr}, {data_dir}, {name}, {creator}')
if os.path.exists(f'data/{data_dir}') and creator != self.id:
model = utils.ipfs2keras(model_addr)
input = None
target = None
for filename in os.listdir(f'data/{data_dir}'):
temp_data = np.load(f'data/{data_dir}/{filename}')
temp_input = temp_data['x_train']
temp_target = temp_data['y_train']
if input is None:
input = temp_input
else:
input = np.append(input, temp_input)
input = np.reshape(input, (-1, 28, 28))
if target is None:
target = temp_target
else:
target = np.append(target, temp_target)
# TODO specifically mnist?!?!?!?!?!?
input = input.reshape(input.shape[0], 784)
input = input.astype('float32')
input /= 255
target = keras.utils.to_categorical(target, 10)
hist = model.fit(
input,
target,
batch_size=100, # TODO config?!?!?!?!
verbose=True,
epochs=10, # TODO config?!??!??!?
validation_split=0.1 # TODO config??!?!?!?!?!?
)
loss = hist.history.get('loss')[-1]
print(f'{Fore.GREEN}Finished training {Fore.YELLOW} -- {loss}{Style.RESET_ALL}')
my_best_model = utils.best_model_for_task(task_name, return_model=True)
best_loss = 100000000
if not my_best_model == None:
best_loss = my_best_model.evaluate(input, target, batch_size=100)[0]
print(f'{Fore.YELLOW}Best Evaluated at: {best_loss}{Style.RESET_ALL}')
if best_loss < loss:
print(f'{Fore.RED}Trained model worse than best trained. Ignoring.{Style.RESET_ALL}')
return
if loss < best_loss:
print(f'New best loss of {Fore.GREEN}{loss}{Style.RESET_ALL} for task {Fore.GREEN}{task_name}{Style.RESET_ALL}')
utils.save_best_model_for_task(task_name, model)
self.add_model(name, model, parent=task_addr)
else:
print("Can't train your own model so soon!!!!!")
def load_model(self, addr):
return utils.ipfs2keras(addr)
