def training(self):
"""
Does the actual training of the model, running it on game data. Endless.
"""
self.compile_model()
self.filenames = deque(get_game_data_filenames(self.config.resource))
shuffle(self.filenames)
total_steps = self.config.trainer.start_total_steps
cnt = 0
while True:
if cnt % 2 == 0:
print('load latest play data.')
import gc
del self.dataset
gc.collect()
self.dataset = deque(), deque(), deque()
self.filenames = deque(
get_game_data_filenames(self.config.resource))
shuffle(self.filenames)
self.fill_queue()
steps = self.train_epoch(self.config.trainer.epoch_to_checkpoint)
total_steps += steps
self.save_current_model()
a, b, c = self.dataset
while len(a) > self.config.trainer.dataset_size / 2:
a.popleft()
b.popleft()
c.popleft()
cnt += 1
def remove_play_data(self):
return
files = get_game_data_filenames(self.config.resource)
if len(files) < self.config.play_data.max_file_num:
return
for i in range(len(files) - self.config.play_data.max_file_num):
os.remove(files[i])
def remove_play_data(self, all=False):
files = get_game_data_filenames(self.config.resource)
if (all):
for path in files:
os.remove(path)
else:
while len(files) > self.config.play_data.max_file_num:
os.remove(files[0])
del files[0]
def load_play_data(self):
filenames = get_game_data_filenames(self.config.resource)
updated = False
for filename in filenames:
if filename in self.loaded_filenames:
continue
self.load_data_from_file(filename)
updated = True
for filename in (self.loaded_filenames - set(filenames)):
self.unload_data_of_file(filename)
updated = True
if updated:
logger.debug("updating training dataset")
self.dataset = self.collect_all_loaded_data()
def training(self):
self.compile_model()
self.filenames = deque(get_game_data_filenames(self.config.resource))
shuffle(self.filenames)
total_steps = self.config.trainer.start_total_steps
while True:
self.fill_queue()
steps = self.train_epoch(self.config.trainer.epoch_to_checkpoint)
total_steps += steps
self.save_current_model()
a, b, c = self.dataset
while len(a) > self.config.trainer.dataset_size / 2:
a.popleft()
b.popleft()
c.popleft()
def load_play_data(self):
new_filenames = set(get_game_data_filenames(self.config.resource)[-self.config.trainer.max_num_files_in_memory:])
for filename in self.loaded_filenames - new_filenames:
logger.debug(f"removing data {filename} from training set")
self.loaded_filenames.remove(filename)
del self.loaded_data[filename]
with ProcessPoolExecutor(max_workers=self.config.trainer.cleaning_processes) as executor:
futures = {executor.submit(load_data_from_file, filename, self.config.model.t_history):filename for filename in new_filenames - self.loaded_filenames}
for future in as_completed(futures):
filename = futures[future]
logger.debug(f"loading data from {filename}")
self.loaded_filenames.add(filename)
self.loaded_data[filename] = future.result()
self.dataset = self.collect_all_loaded_data()
def training(self):
"""
Does the actual training of the model, running it on game data. Endless.
"""
self.compile_model()
self.filenames = deque(get_game_data_filenames(self.config.resource))
shuffle(self.filenames)
total_steps = self.config.trainer.start_total_steps
while True:
self.fill_queue()
steps = self.train_epoch(self.config.trainer.epoch_to_checkpoint)
total_steps += steps
self.save_current_model()
a, b, c = self.dataset
while len(a) > self.config.trainer.dataset_size/2:
a.popleft()
b.popleft()
c.popleft()
def training(self):
"""
Does the actual training of the model, running it on game data. Endless.
"""
self.compile_model()
self.filenames = deque(get_game_data_filenames(self.config.resource))
shuffle(self.filenames)
total_steps = self.config.trainer.start_total_steps
start_time = time()
while True:
self.fill_queue()
steps = self.train_epoch(self.config.trainer.epoch_to_checkpoint)
total_steps += steps
self.save_current_model()
a, b, c = self.dataset
print(f"time={time() - start_time:5.1f}s ")
exit()
#train for only one fit
while len(a) > self.config.trainer.dataset_size / 2:
a.popleft()
b.popleft()
c.popleft()
def training(self):
self.compile_model()
total_steps = self.config.trainer.start_total_steps
bef_files = []
last_file = None
while True:
files = get_game_data_filenames(self.config.resource)
#if last_file is not None:
# print(files.index(last_file))
# print(len(bef_files))
if (len(files)*self.config.play_data.nb_game_in_file < 1000 \
or ((last_file is not None) and files.index(last_file)+3 > len(files))):
print('waiting for enough data 600s, ' +
str(len(files) * self.config.play_data.nb_game_in_file) +
' vs ' +
str(self.config.trainer.min_games_to_begin_learn) +
' games')
time.sleep(600)
continue
else:
last_file = files[-1]
bef_files = files
if (len(files) > 100):
files = files[-100:]
self.filenames = deque(files)
shuffle(self.filenames)
self.fill_queue()
if len(self.dataset[0]) > self.config.trainer.batch_size:
steps = self.train_epoch(
self.config.trainer.epoch_to_checkpoint)
total_steps += steps
self.save_current_model()
a, b, c = self.dataset
a.clear()
b.clear()
c.clear()
def training(self):
self.compile_model()
self.filenames = deque(get_game_data_filenames(self.config.resource))
shuffle(self.filenames)
last_load_data_step = last_save_step = total_steps = self.config.trainer.start_total_steps
while True:
self.fill_queue()
# if self.dataset_size < self.config.trainer.min_data_size_to_learn:
# logger.info(f"dataset_size={self.dataset_size} is less than {self.config.trainer.min_data_size_to_learn}")
# sleep(60)
# self.fill_queue()
# continue
#self.update_learning_rate(total_steps)
steps = self.train_epoch(self.config.trainer.epoch_to_checkpoint)
total_steps += steps
#if last_save_step + self.config.trainer.save_model_steps < total_steps:
self.save_current_model()
last_save_step = total_steps
a, b, c = self.dataset
while len(a) > self.config.trainer.dataset_size / 2:
a.popleft()
b.popleft()
c.popleft()
