def model_fn(data, train_config):
learning_rate = train_config['learning_rate']
reg_value = train_config['reg_param']
amp_factor = train_config['amp_factor']
win_size_mins = train_config['win_size_mins']
batch_size = train_config['batch_size']
pre_processed_data_freq = 10 #Hz
cnn_window_size = 10 # s
num_cnn_data_points = pre_processed_data_freq * cnn_window_size
win_size_cnn_outputs = win_size_mins * (60 // cnn_window_size)
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
dataset = tf.data.Dataset.from_generator(
lambda: window_generator(data, win_size_mins),
output_types=(tf.float32, tf.float32),
output_shapes=((win_size_cnn_outputs, num_cnn_data_points, 3),
(win_size_cnn_outputs))).prefetch(-1).batch(batch_size)
data_iterator = dataset.make_one_shot_iterator()
x, y = data_iterator.get_next()
local_devices = device_lib.list_local_devices()
gpu_names = [x.name for x in local_devices if x.device_type == 'GPU']
num_gpus = min(1, len(gpu_names))
# 10 seconds at 10 Hz
x = tf.reshape(x, [-1, 1, num_cnn_data_points, 3])
y = tf.reshape(y, [-1, win_size_cnn_outputs])
x_vals = tf.split(x, num_gpus, axis=0)
y_vals = tf.split(y, num_gpus, axis=0)
losses = []
predictions = []
actual_labels = []
for gpu_id in range(int(num_gpus)):
with tf.device(tf.DeviceSpec(device_type="GPU", device_index=gpu_id)):
with tf.variable_scope(tf.get_variable_scope(),
reuse=(gpu_id > 0)):
logits, pred = cnn_bi_lstm_model(x_vals[gpu_id], (gpu_id > 0),
amp_factor, win_size_mins)
predictions.append(pred)
l2 = reg_value * sum(
tf.nn.l2_loss(tf_var)
for tf_var in tf.trainable_variables())
actual_labels.append(y_vals[gpu_id])
losses.append(
tf.reduce_sum(
tf.nn.sigmoid_cross_entropy_with_logits(
labels=tf.cast(y_vals[gpu_id], tf.float32),
logits=logits)) + l2)
loss = tf.reduce_mean(losses)
prediction = tf.concat(predictions, axis=0)
label = tf.concat(actual_labels, axis=0)
error = 1 - tf.reduce_mean(tf.cast(tf.equal(prediction, label),
tf.float32))
train_step = optimizer.minimize(loss, colocate_gradients_with_ops=True)
return optimizer, loss, error
def __init__(self,
metadata,
eps=1,
delta=1e-5,
infer_ranges=False,
num_teachers=10,
n_iters=100,
batch_size=128,
learning_rate=1e-4,
multiprocess=False):
"""
:param metadata: dict: Attribute metadata describing the data domain of the synthetic target data
:param eps: float: Privacy parameter
:param delta: float: Privacy parameter
:param target: str: Name of the target variable for downstream classification tasks
:param num_teachers: int: Number of teacher discriminators
:param n_iters: int: Number of training iterations
"""
# Data description
self.metadata, self.attribute_list = self.read_meta(metadata)
self.datatype = DataFrame
self.nfeatures = self.get_num_features()
# Privacy params
self.epsilon = eps
self.delta = delta
self.infer_ranges = infer_ranges
# Training params
self.num_teachers = num_teachers
self.n_iters = n_iters
self.batch_size = batch_size
self.learning_rate = learning_rate
self.z_dim = int(self.nfeatures / 4)
self.h_dim = int(self.nfeatures)
# Configure device
device_name = tf.test.gpu_device_name()
if device_name is '':
self.device_spec = tf.DeviceSpec(device_type='CPU', device_index=0)
else:
self.device_spec = tf.DeviceSpec(device_type='GPU', device_index=0)
with tf.device(self.device_spec.to_string()):
# Variable init
# Feature matrix
self.X = tf.placeholder(tf.float32, shape=[None, self.nfeatures])
# Latent space
self.Z = tf.placeholder(tf.float32, shape=[None, self.z_dim])
# Noise variable
self.M = tf.placeholder(tf.float32, shape=[None, 1])
# Generator
self.GDist = None
self._generator()
# Discriminator
self._discriminator()
self.sess = tf.Session()
self.multiprocess = multiprocess
self.trained = False
self.__name__ = f'PateGanEps{self.epsilon}'
def _custom_parsing_context(self):
dev_spec = tf.DeviceSpec(
device_type=("GPU" if self.device_option.is_gpu() else "CPU"),
device_index=self.device_option.num)
return tf.device(dev_spec)
def main(_):
tf.logging.set_verbosity(tf.logging.INFO)
# If we have ps_hosts, then we'll assume that this is going to be a
# distributed training run. Configure the cluster appropriately. Otherwise,
# we just do everything in-process.
if FLAGS.ps_hosts:
cluster = tf.train.ClusterSpec({
'ps': FLAGS.ps_hosts.split(','),
'worker': FLAGS.worker_hosts.split(','),
})
if FLAGS.job_name == 'ps':
# Ignore the GPU if we're the parameter server. This let's the PS run on
# the same machine as a worker.
config = tf.ConfigProto(device_count={'GPU': 0})
elif FLAGS.job_name == 'worker':
config = tf.ConfigProto(gpu_options=tf.GPUOptions(
visible_device_list='%d' % FLAGS.gpu_device,
allow_growth=True))
else:
raise ValueError('unknown job name "%s"' % FLAGS.job_name)
server = tf.train.Server(
cluster,
job_name=FLAGS.job_name,
task_index=FLAGS.task_index,
config=config)
if FLAGS.job_name == 'ps':
return server.join()
device_setter = tf.train.replica_device_setter(
worker_device='/job:worker/task:%d' % FLAGS.task_index,
cluster=cluster)
else:
server = None
device_setter = tf.train.replica_device_setter(0)
# Build the graph.
with tf.Graph().as_default():
with tf.device(device_setter):
model = Model(FLAGS.input_base_path, FLAGS)
# If an eval path is present, then create eval operators and set up scalar
# summaries to report on the results. Run the evals on the CPU since
# the analogy eval requires a fairly enormous tensor to be allocated to
# do the nearest neighbor search.
if FLAGS.eval_base_path:
wordsim_filenames = glob.glob(
os.path.join(FLAGS.eval_base_path, '*.ws.tab'))
for filename in wordsim_filenames:
name = os.path.basename(filename).split('.')[0]
with tf.device(tf.DeviceSpec(device_type='CPU')):
op = model.wordsim_eval_op(filename, FLAGS.eval_word_prefix)
tf.summary.scalar(name, op)
analogy_filenames = glob.glob(
os.path.join(FLAGS.eval_base_path, '*.an.tab'))
for filename in analogy_filenames:
name = os.path.basename(filename).split('.')[0]
with tf.device(tf.DeviceSpec(device_type='CPU')):
op = model.analogy_eval_op(filename)
tf.summary.scalar(name, op)
tf.summary.scalar('loss', model.loss_op)
tf.summary.scalar('l2_loss', model.l2_loss_op)
tf.summary.scalar('sigmoid_loss', model.sigmoid_loss_op)
tf.summary.scalar('vector_regul_loss', model.vector_regul_loss_op)
# Train on, soldier.
supervisor = tf.train.Supervisor(
logdir=FLAGS.output_base_path,
is_chief=(FLAGS.task_index == 0),
save_summaries_secs=60,
recovery_wait_secs=5)
max_step = FLAGS.num_epochs * model.steps_per_epoch
master = server.target if server else ''
with supervisor.managed_session(master) as session:
local_step = 0
global_step = session.run(model.global_step)
while not supervisor.should_stop() and global_step < max_step:
global_step, loss, _ = session.run([
model.global_step, model.loss_op, model.train_op])
if not np.isfinite(loss):
raise ValueError('non-finite cost at step %d' % global_step)
local_step += 1
if local_step % 10 == 0:
tf.logging.info(
'local_step=%d global_step=%d loss=%.1f, %.1f%% complete',
local_step, global_step, loss, 100.0 * global_step / max_step)
if FLAGS.task_index == 0:
supervisor.saver.save(
session, supervisor.save_path, global_step=global_step)
model.write_embeddings(FLAGS, session)