def test_testdata():
ds = df.DataFromList(list(zip(range(10), range(10, 0, -1))), shuffle=False)
ds.reset_state()
index = 0
for dp in ds.get_data():
assert index == dp[0]
assert 10 - index == dp[1]
index += 1
def build_eval_dataflow(self, policy=None, repeats=None):
"""
:param policy: policy to evaluate when mode == eval
:param repeats: repeat evaluation multiple times when mode == eval
:return: dataflow with evaluation results
"""
df = dataflow.DataFromList(self.filtered_samples, shuffle=False)
df = dataflow.RepeatedData(df, 1000000)
df = EvalDataFeed(df,
self.get_db,
self.domain,
policy=policy,
repeats=repeats)
return df
def build_dataflow(self,
batch_size,
step_size,
restart_limit=None,
cache=None):
"""
:param batch_size: batch size
:param step_size: number of steps for BPTT
:param restart_limit: restart after limit number of batches. Used for validation. If 0 (but not None) or larger
than an epoch its set to one epoch.
:param cache: preloaded cache
:return: dataflow with input data
"""
# update db wrapper function with shared cache
if cache is not None:
self.get_db = (
lambda: Database(filename=self.filename, cache=cache))
df = dataflow.DataFromList(self.filtered_samples,
shuffle=(self.mode == 'train'))
if restart_limit is None:
df = dataflow.RepeatedData(df,
1000000) # reshuffles on every repeat
df = DynamicTrajBatch(df,
batch_size=batch_size,
step_size=step_size,
traj_lens=self.filtered_samples[:, 4])
self.steps_in_epoch = df.steps_in_epoch()
if restart_limit is not None:
if restart_limit == 0 or restart_limit >= self.steps_in_epoch:
restart_limit = self.steps_in_epoch - 1
self.steps_in_epoch = restart_limit
df = OneShotData(df, size=restart_limit)
df = TrajDataFeed(df,
self.get_db,
self.domain,
batch_size=batch_size,
step_size=step_size)
# uncomment to test dataflow speed
# dataflow.TestDataSpeed(df, size=1000).start()
return df
def test_queue_input_infinite():
ds = df.DataFromList(list(zip(range(10), range(10, 0, -1))), shuffle=False)
ds.reset_state()
placeholders = [
tf.placeholder(tf.float32, shape=()),
tf.placeholder(tf.float32, shape=())
]
thread = QueueInput(ds, placeholders, repeat_infinite=True, queue_size=50)
with tf.Session() as sess:
assert 0 == sess.run(thread.queue_size())
thread.start(sess)
assert 0 < sess.run(thread.queue_size())
time.sleep(1.0)
assert 50 == sess.run(thread.queue_size())
def test_queue_input_infinite():
ds = df.DataFromList(list(zip(range(10), range(10, 0, -1))), shuffle=False)
ds.reset_state()
placeholders = [
tf.placeholder(tf.float32, shape=()),
tf.placeholder(tf.float32, shape=())
]
thread = QueueInput(ds, placeholders, repeat_infinite=True)
tensors = [2.0 * ph for ph in thread.tensors()]
with tf.Session() as sess:
thread.start(sess)
for i in range(30):
_ = sess.run(tensors)
def test_queue_input_monitored_session():
ds = df.DataFromList(list(zip(range(10), range(10, 0, -1))), shuffle=False)
ds.reset_state()
placeholders = [
tf.placeholder(tf.float32, shape=()),
tf.placeholder(tf.float32, shape=())
]
thread = QueueInput(ds, placeholders)
tensors = [2.0 * ph for ph in thread.tensors()]
with tf.train.MonitoredTrainingSession() as sess:
thread.start(sess)
for i in range(10):
dp = sess.run(tensors)
assert i * 2.0 == dp[0]
assert (10 - i) * 2.0 == dp[1]
def test_queue_input_terminated():
ds = df.DataFromList(list(zip(range(10), range(10, 0, -1))), shuffle=False)
ds.reset_state()
placeholders = [
tf.placeholder(tf.float32, shape=()),
tf.placeholder(tf.float32, shape=())
]
thread = QueueInput(ds, placeholders)
tensors = [2.0 * ph for ph in thread.tensors()]
with tf.Session() as sess:
thread.start(sess)
for i in range(10):
_ = sess.run(tensors)
with pytest.raises(tf.errors.OutOfRangeError) as excinfo:
sess.run(tensors)
def test_queue_input_multi_threads():
threads = []
for t in range(3):
ds = df.DataFromList(list(zip(range(10), range(10, 0, -1))),
shuffle=False)
ds.reset_state()
placeholders = [
tf.placeholder(tf.float32, shape=()),
tf.placeholder(tf.float32, shape=())
]
thread = QueueInput(ds, placeholders, repeat_infinite=True)
threads.append(thread)
with tf.Session() as sess:
for thread in threads:
thread.start(sess)
for i in range(10):
dp = sess.run([t.tensors() for t in threads])
assert [[i * 1.0, (10 - i) * 1.0]] * 3 == dp