def train(processed_dir,
save_file=None,
epochs=10,
logdir=None,
checkpoint_freq=10000):
test_dataset = DataSet.read(os.path.join(processed_dir, "test.chunk.gz"))
train_chunk_files = [
os.path.join(processed_dir, fname)
for fname in os.listdir(processed_dir)
if TRAINING_CHUNK_RE.match(fname)
]
save_file = os.path.join(os.getcwd(), save_file)
n = PolicyNetwork()
try:
n.initialize_variables(save_file)
except:
n.initialize_variables(None)
if logdir is not None:
n.initialize_logging(logdir)
last_save_checkpoint = 0
for i in range(epochs):
random.shuffle(train_chunk_files)
for file in train_chunk_files:
print("Using %s" % file)
train_dataset = DataSet.read(file)
train_dataset.shuffle()
with timer("training"):
n.train(train_dataset)
n.save_variables(save_file)
if n.get_global_step() > last_save_checkpoint + checkpoint_freq:
with timer("test set evaluation"):
n.check_accuracy(test_dataset)
last_save_checkpoint = n.get_global_step()
def train(processed_dir="processed_data"):
checkpoint_freq = 10000
read_file = None
save_file = 'tmp2'
epochs = 10
logdir = 'logs2'
#
test_dataset = DataSet.read(os.path.join(processed_dir, "test.chunk.gz"))
train_chunk_files = [
os.path.join(processed_dir, fname)
for fname in os.listdir(processed_dir)
if TRAINING_CHUNK_RE.match(fname)
]
if read_file is not None:
read_file = os.path.join(os.getcwd(), save_file)
n = PolicyNetwork()
n.initialize_variables(read_file)
if logdir is not None:
n.initialize_logging(logdir)
last_save_checkpoint = 0
for i in range(epochs):
random.shuffle(train_chunk_files)
for file in train_chunk_files:
print("提取 %s" % file)
with timer("load dataset"):
train_dataset = DataSet.read(file)
with timer("training"):
n.train(train_dataset)
with timer("save model"):
n.save_variables(save_file)
if n.get_global_step() > last_save_checkpoint + checkpoint_freq:
with timer("test set evaluation"):
n.check_accuracy(test_dataset)
last_save_checkpoint = n.get_global_step()
def train(processed_dir,
read_file=None,
save_file=None,
epochs=10,
logdir=None,
checkpoint_freq=10000):
test_dataset = DataSet.read(os.path.join(processed_dir, "test.chunk.gz"))
train_chunk_files = [
os.path.join(processed_dir, fname)
for fname in os.listdir(processed_dir)
if TRAINING_CHUNK_RE.match(fname)
]
n = PolicyNetwork(DEFAULT_FEATURES.planes)
n.initialize_variables(read_file)
if logdir is not None:
n.initialize_logging(logdir)
last_save_checkpoint = 0
for i in range(epochs):
random.shuffle(train_chunk_files)
for file in train_chunk_files:
print("Using %s" % file)
train_dataset = DataSet.read(file)
n.train(train_dataset)
if save_file is not None and n.get_global_step(
) > last_save_checkpoint + checkpoint_freq:
n.check_accuracy(test_dataset)
print("Saving checkpoint to %s" % save_file, file=sys.stderr)
last_save_checkpoint = n.get_global_step()
n.save_variables(save_file)
if save_file is not None:
n.save_variables(save_file)
print("Finished training. New model saved to %s" % save_file,
file=sys.stderr)
def train(processed_dir, read_file=None, save_file=None, epochs=10, logdir=None, checkpoint_freq=10000):
test_dataset = DataSet.read(os.path.join(processed_dir, "test.chunk.gz"))
train_chunk_files = [os.path.join(processed_dir, fname)
for fname in os.listdir(processed_dir)
if TRAINING_CHUNK_RE.match(fname)]
if read_file is not None:
read_file = os.path.join(os.getcwd(), save_file)
n = PolicyNetwork()
n.initialize_variables(read_file)
if logdir is not None:
n.initialize_logging(logdir)
last_save_checkpoint = 0
for i in range(epochs):
random.shuffle(train_chunk_files)
for file in train_chunk_files:
print("Using %s" % file)
with timer("load dataset"):
train_dataset = DataSet.read(file)
with timer("training"):
n.train(train_dataset)
with timer("save model"):
n.save_variables(save_file)
if n.get_global_step() > last_save_checkpoint + checkpoint_freq:
with timer("test set evaluation"):
n.check_accuracy(test_dataset)
last_save_checkpoint = n.get_global_step()
def train(processed_dir,
read_file=None,
save_file=None,
epochs=10,
logdir=None,
checkpoint_freq=10000):
test_dataset = DataSet.read(os.path.join(processed_dir, 'test.chunk.gz'))
#print(test_dataset)
train_chunk_files = [
os.path.join(processed_dir, fname)
for fname in os.listdir(processed_dir)
if TRAINING_CHUNK_RE.match(fname)
]
print(train_chunk_files)
if read_file is not None:
read_file = os.path.join(os.getcwd(), save_file)
n = PolicyNetwork()
n.initialize_variables()
if logdir is not None:
n.initialize_logging(logdir)
last_save_checkpoint = 0
for i in range(epochs):
random.shuffle(train_chunk_files)
for file in tqdm.tqdm(train_chunk_files, desc='epochs ' + str(i)):
#print('Using %s' % file)
with timer('load dataset'):
train_dataset = DataSet.read(file)
with timer('training'):
n.train(train_dataset)
if n.get_global_step() > last_save_checkpoint + checkpoint_freq:
with timer('save model'):
n.save_variables(save_file)
with timer('test set evaluation'):
n.check_accuracy(test_dataset)
last_save_checkpoint = n.get_global_step()
with timer('test set evaluation'):
n.check_accuracy(test_dataset)