def remove_old_play_data(self):
with MkdirLockFile(self.config.resource.play_data_dir):
files = get_game_data_filenames(self.config.resource)
if len(files) < self.config.play_data.max_file_num:
return
for i in range(len(files) - self.config.play_data.max_file_num):
os.remove(files[i])
def save_play_data(self, buffer):
rc = self.config.resource
game_id = datetime.now().strftime("%Y%m%d-%H%M%S.%f")
with MkdirLockFile(rc.play_data_dir):
path = os.path.join(rc.play_data_dir, rc.play_data_filename_tmpl % game_id)
logger.info(f"save play data to {path}")
write_game_data_to_file(path, buffer)
def load(self, save_path):
with MkdirLockFile(save_path):
latest_checkpoint = tf.train.latest_checkpoint(save_path)
if latest_checkpoint is None:
return None
if self.latest_checkpoint == latest_checkpoint:
return None
logger.debug(f"loading model from {latest_checkpoint}")
self.saver.restore(self.sess, latest_checkpoint)
self.latest_checkpoint = latest_checkpoint
def set(self, key, response):
dirname = self.gen_directory_name(key)
try:
os.makedirs(dirname, self.dirmode)
except (IOError, OSError):
pass
with MkdirLockFile(dirname) as lock:
self.write(os.path.join(lock.path, 'headers'),
ujson.dumps(self.normalize_headers(response.headers)))
self.write(os.path.join(lock.path, 'data'), response.data)
def save(self, save_path, step):
logger.debug(f"save model to {save_path}")
with MkdirLockFile(save_path):
checkpoint_path = os.path.join(save_path,
'model-%s.ckpt' % self.model_name)
self.saver.save(self.sess,
checkpoint_path,
global_step=step,
write_meta_graph=False)
self.latest_checkpoint = checkpoint_path
logger.debug(f"saved model path {checkpoint_path}")
return checkpoint_path
def load_play_data(self):
with MkdirLockFile(self.config.resource.play_data_dir):
filenames = get_game_data_filenames(self.config.resource)
updated = False
for filename in filenames:
if filename in self.loaded_filenames:
continue
self.load_data_from_file(filename)
updated = True
for filename in (self.loaded_filenames - set(filenames)):
self.unload_data_of_file(filename)
updated = True
if updated:
self.dataset = DataSet(self.loaded_data)
logger.debug(f"updating training dataset size {self.dataset_size}")
