def receive_model(self, message, verbose=True): msg = json.loads(message['data']) if (msg is not None): if (msg['type'] == 'transact'): return utils.ipfs2keras(msg['model_addr']), msg elif (msg['type'] == 'log'): if (verbose): output = "Worker:" + msg['worker_id'][-5:] output += " - Epoch " + str( msg['epoch_id']) + " of " + str(msg['num_epochs']) output += " - Valid Loss: " + str(msg['eval_loss'])[0:8] print(output) # Figure out of we should tell this worker to quit. parent_model = msg['parent_model'] worker_id = msg['worker_id'] num_epochs = msg['num_epochs'] epoch_id = msg['epoch_id'] progress = self.update_progress(parent_model, worker_id, num_epochs, epoch_id) max_progress = self.max_progress(parent_model) if progress < max_progress * 0.75: quit = {} quit['op_code'] = 'quit' self.publish(self.spec['train_channel'] + ':' + worker_id, quit)
def fit_worker(self, message): decoded = json.loads(message['data']) if(decoded['framework'] == 'keras'): model = utils.ipfs2keras(decoded['model_addr']) try: np_strings = json.loads(self.api.cat(decoded['data_addr'])) except NotImplementedError: raise NotImplementedError("The IPFS API only supports Python 3.6. Please modify your environment.") input, target, valid_input, valid_target = list(map(lambda x: self.deserialize_numpy(x),np_strings)) train_channel = decoded['train_channel'] self.learner_callback = OutputPipe( id=self.id, publisher=self.publish, channel=train_channel, epochs=decoded['epochs'], model_addr=decoded['model_addr'], model=model ) args = (self.train_meta, train_channel + ':' + self.id) monitor_thread = threading.Thread(target=self.listen_to_channel, args=args) monitor_thread.start() print('training model') model.fit( input, target, batch_size=decoded['batch_size'], validation_data=(valid_input, valid_target), verbose=False, epochs=decoded['epochs'], callbacks=[self.learner_callback] ) print('done') else: raise NotImplementedError("Only compatible with Keras at the moment")
def added_local_data_model(self, info): task_addr = info['task'] task_info = self.api.get_json(task_addr) task_name = info['name'] model_addr = info['model'] data_dir = task_info['data_dir'] name = task_info['name'] creator = info['creator'] print( f'FOUND NEW MODEL: {task_addr}, {model_addr}, {data_dir}, {name}, {creator}' ) if os.path.exists(f'data/{data_dir}') and creator != self.id: model = utils.ipfs2keras(model_addr) input = None target = None for filename in os.listdir(f'data/{data_dir}'): temp_data = np.load(f'data/{data_dir}/{filename}') temp_input = temp_data['x_train'] temp_target = temp_data['y_train'] if input is None: input = temp_input else: input = np.append(input, temp_input) input = np.reshape(input, (-1, 28, 28)) if target is None: target = temp_target else: target = np.append(target, temp_target) # TODO specifically mnist?!?!?!?!?!? input = input.reshape(input.shape[0], 784) input = input.astype('float32') input /= 255 target = keras.utils.to_categorical(target, 10) self.train_model(model, input, name, taraget, task_name, task_addr) else: print("Can't train your own model so soon!!!!!")
def added_adapter_model(self, info): task_addr = info['task'] task_info = self.api.get_json(task_addr) task_name = info['name'] model_addr = info['model'] adapter = task_info['adapter'] name = task_info['name'] creator = info['creator'] model = utils.ipfs2keras(model_addr) utils.save_adapter(adapter) import grid.adapters.adapter as grid_adapter print('load next input') n_test, n_target = grid_adapter.next_input() self.train_model(model, n_test, n_target, name, task_name, task_addr)
def added_model(self, info): info = self.api.get_json(info['data']) task_addr = info['task'] task_name = info['name'] model_addr = info['model'] task_info = self.api.get_json(task_addr) data_dir = task_info['data_dir'] name = task_info['name'] creator = info['creator'] print(f'FOUND NEW MODEL: {task_addr}, {model_addr}, {data_dir}, {name}, {creator}') if os.path.exists(f'data/{data_dir}') and creator != self.id: model = utils.ipfs2keras(model_addr) input = None target = None for filename in os.listdir(f'data/{data_dir}'): temp_data = np.load(f'data/{data_dir}/{filename}') temp_input = temp_data['x_train'] temp_target = temp_data['y_train'] if input is None: input = temp_input else: input = np.append(input, temp_input) input = np.reshape(input, (-1, 28, 28)) if target is None: target = temp_target else: target = np.append(target, temp_target) # TODO specifically mnist?!?!?!?!?!? input = input.reshape(input.shape[0], 784) input = input.astype('float32') input /= 255 target = keras.utils.to_categorical(target, 10) hist = model.fit( input, target, batch_size=100, # TODO config?!?!?!?! verbose=True, epochs=10, # TODO config?!??!??!? validation_split=0.1 # TODO config??!?!?!?!?!? ) loss = hist.history.get('loss')[-1] print(f'{Fore.GREEN}Finished training {Fore.YELLOW} -- {loss}{Style.RESET_ALL}') my_best_model = utils.best_model_for_task(task_name, return_model=True) best_loss = 100000000 if not my_best_model == None: best_loss = my_best_model.evaluate(input, target, batch_size=100)[0] print(f'{Fore.YELLOW}Best Evaluated at: {best_loss}{Style.RESET_ALL}') if best_loss < loss: print(f'{Fore.RED}Trained model worse than best trained. Ignoring.{Style.RESET_ALL}') return if loss < best_loss: print(f'New best loss of {Fore.GREEN}{loss}{Style.RESET_ALL} for task {Fore.GREEN}{task_name}{Style.RESET_ALL}') utils.save_best_model_for_task(task_name, model) self.add_model(name, model, parent=task_addr) else: print("Can't train your own model so soon!!!!!")
def load_model(self, addr): return utils.ipfs2keras(addr)