def training(self): """ Does the actual training of the model, running it on game data. Endless. """ self.compile_model() self.filenames = deque(get_game_data_filenames(self.config.resource)) shuffle(self.filenames) total_steps = self.config.trainer.start_total_steps cnt = 0 while True: if cnt % 2 == 0: print('load latest play data.') import gc del self.dataset gc.collect() self.dataset = deque(), deque(), deque() self.filenames = deque( get_game_data_filenames(self.config.resource)) shuffle(self.filenames) self.fill_queue() steps = self.train_epoch(self.config.trainer.epoch_to_checkpoint) total_steps += steps self.save_current_model() a, b, c = self.dataset while len(a) > self.config.trainer.dataset_size / 2: a.popleft() b.popleft() c.popleft() cnt += 1
def remove_play_data(self): return files = get_game_data_filenames(self.config.resource) if len(files) < self.config.play_data.max_file_num: return for i in range(len(files) - self.config.play_data.max_file_num): os.remove(files[i])
def remove_play_data(self, all=False): files = get_game_data_filenames(self.config.resource) if (all): for path in files: os.remove(path) else: while len(files) > self.config.play_data.max_file_num: os.remove(files[0]) del files[0]
def load_play_data(self): filenames = get_game_data_filenames(self.config.resource) updated = False for filename in filenames: if filename in self.loaded_filenames: continue self.load_data_from_file(filename) updated = True for filename in (self.loaded_filenames - set(filenames)): self.unload_data_of_file(filename) updated = True if updated: logger.debug("updating training dataset") self.dataset = self.collect_all_loaded_data()
def training(self): self.compile_model() self.filenames = deque(get_game_data_filenames(self.config.resource)) shuffle(self.filenames) total_steps = self.config.trainer.start_total_steps while True: self.fill_queue() steps = self.train_epoch(self.config.trainer.epoch_to_checkpoint) total_steps += steps self.save_current_model() a, b, c = self.dataset while len(a) > self.config.trainer.dataset_size / 2: a.popleft() b.popleft() c.popleft()
def load_play_data(self): new_filenames = set(get_game_data_filenames(self.config.resource)[-self.config.trainer.max_num_files_in_memory:]) for filename in self.loaded_filenames - new_filenames: logger.debug(f"removing data {filename} from training set") self.loaded_filenames.remove(filename) del self.loaded_data[filename] with ProcessPoolExecutor(max_workers=self.config.trainer.cleaning_processes) as executor: futures = {executor.submit(load_data_from_file, filename, self.config.model.t_history):filename for filename in new_filenames - self.loaded_filenames} for future in as_completed(futures): filename = futures[future] logger.debug(f"loading data from {filename}") self.loaded_filenames.add(filename) self.loaded_data[filename] = future.result() self.dataset = self.collect_all_loaded_data()
def training(self): """ Does the actual training of the model, running it on game data. Endless. """ self.compile_model() self.filenames = deque(get_game_data_filenames(self.config.resource)) shuffle(self.filenames) total_steps = self.config.trainer.start_total_steps while True: self.fill_queue() steps = self.train_epoch(self.config.trainer.epoch_to_checkpoint) total_steps += steps self.save_current_model() a, b, c = self.dataset while len(a) > self.config.trainer.dataset_size/2: a.popleft() b.popleft() c.popleft()
def training(self): """ Does the actual training of the model, running it on game data. Endless. """ self.compile_model() self.filenames = deque(get_game_data_filenames(self.config.resource)) shuffle(self.filenames) total_steps = self.config.trainer.start_total_steps start_time = time() while True: self.fill_queue() steps = self.train_epoch(self.config.trainer.epoch_to_checkpoint) total_steps += steps self.save_current_model() a, b, c = self.dataset print(f"time={time() - start_time:5.1f}s ") exit() #train for only one fit while len(a) > self.config.trainer.dataset_size / 2: a.popleft() b.popleft() c.popleft()
def training(self): self.compile_model() total_steps = self.config.trainer.start_total_steps bef_files = [] last_file = None while True: files = get_game_data_filenames(self.config.resource) #if last_file is not None: # print(files.index(last_file)) # print(len(bef_files)) if (len(files)*self.config.play_data.nb_game_in_file < 1000 \ or ((last_file is not None) and files.index(last_file)+3 > len(files))): print('waiting for enough data 600s, ' + str(len(files) * self.config.play_data.nb_game_in_file) + ' vs ' + str(self.config.trainer.min_games_to_begin_learn) + ' games') time.sleep(600) continue else: last_file = files[-1] bef_files = files if (len(files) > 100): files = files[-100:] self.filenames = deque(files) shuffle(self.filenames) self.fill_queue() if len(self.dataset[0]) > self.config.trainer.batch_size: steps = self.train_epoch( self.config.trainer.epoch_to_checkpoint) total_steps += steps self.save_current_model() a, b, c = self.dataset a.clear() b.clear() c.clear()
def training(self): self.compile_model() self.filenames = deque(get_game_data_filenames(self.config.resource)) shuffle(self.filenames) last_load_data_step = last_save_step = total_steps = self.config.trainer.start_total_steps while True: self.fill_queue() # if self.dataset_size < self.config.trainer.min_data_size_to_learn: # logger.info(f"dataset_size={self.dataset_size} is less than {self.config.trainer.min_data_size_to_learn}") # sleep(60) # self.fill_queue() # continue #self.update_learning_rate(total_steps) steps = self.train_epoch(self.config.trainer.epoch_to_checkpoint) total_steps += steps #if last_save_step + self.config.trainer.save_model_steps < total_steps: self.save_current_model() last_save_step = total_steps a, b, c = self.dataset while len(a) > self.config.trainer.dataset_size / 2: a.popleft() b.popleft() c.popleft()