def UpsampleConv(name, input_dim, output_dim, filter_size, inputs, he_init=True, biases=True): output = inputs output = lib.concat([output, output, output, output], axis=1) output = tf.transpose(output, [0,2,3,1]) output = tf.depth_to_space(output, 2) output = tf.transpose(output, [0,3,1,2]) output = lib.ops.conv2d.Conv2D(name, input_dim, output_dim, filter_size, output, he_init=he_init, biases=biases) return output
def UpsampleConv(name, input_dim, output_dim, filter_size, inputs, he_init=True, biases=True): output = inputs output = lib.concat([output, output, output, output], axis=1) output = tf.transpose(output, [0,2,3,1]) output = tf.depth_to_space(output, 2) output = tf.transpose(output, [0,3,1,2]) output = lib.ops.conv2d.Conv2D(name, input_dim, output_dim, filter_size, output, he_init=he_init, biases=biases) return output
# fake_data = tf.concat(fake_data_splits, axis=0) # fake_data_splits = tf.split(fake_data, len(DEVICES)) all_real_data = tf.reshape( 2 * ((tf.cast(all_real_data_conv, tf.float32) / 255.) - .5), [BATCH_SIZE, OUTPUT_DIM]) all_real_data_splits = tf.split(all_real_data, len(DEVICES) / 2) DEVICES_B = DEVICES[:len(DEVICES) / 2] DEVICES_A = DEVICES[len(DEVICES) / 2:] disc_costs = [] for i, device in enumerate(DEVICES_A): with tf.device(device): real_and_fake_data = lib.concat( [all_real_data_splits[i]] + [fake_data_splits[i]] + [fake_data_splits[len(DEVICES_A) + i]], axis=0) disc_all = Discriminator(real_and_fake_data) disc_real = disc_all[:BATCH_SIZE / len(DEVICES_A)] disc_fake = disc_all[BATCH_SIZE / len(DEVICES_A):] disc_costs.append( tf.reduce_mean(disc_fake) - tf.reduce_mean(disc_real)) for i, device in enumerate(DEVICES_B): with tf.device(device): real_data = tf.identity( all_real_data_splits[i]) # transfer from gpu0 fake_data__ = lib.concat([ fake_data_splits[i], fake_data_splits[len(DEVICES_A) + i] ], axis=0)
minval=0., maxval=1. / 128) # dequantize all_real_data_splits = lib.split(all_real_data, len(DEVICES), axis=0) DEVICES_B = DEVICES[:len(DEVICES) / 2] DEVICES_A = DEVICES[len(DEVICES) / 2:] disc_costs = [] disc_acgan_costs = [] disc_acgan_accs = [] disc_acgan_fake_accs = [] for i, device in enumerate(DEVICES_A): with tf.device(device): real_and_fake_data = lib.concat([ all_real_data_splits[i], all_real_data_splits[len(DEVICES_A) + i], fake_data_splits[i], fake_data_splits[len(DEVICES_A) + i] ], axis=0) real_and_fake_labels = lib.concat([ labels_splits[i], labels_splits[len(DEVICES_A) + i], labels_splits[i], labels_splits[len(DEVICES_A) + i] ], axis=0) disc_all, disc_all_acgan = Discriminator( real_and_fake_data, real_and_fake_labels) disc_real = disc_all[:BATCH_SIZE / len(DEVICES_A)] disc_fake = disc_all[BATCH_SIZE / len(DEVICES_A):] disc_costs.append( tf.reduce_mean(disc_fake) - tf.reduce_mean(disc_real)) if ACGAN: disc_acgan_costs.append(
fake_data_splits = [] for device in DEVICES: with tf.device(device): fake_data_splits.append(Generator(BATCH_SIZE/len(DEVICES))) all_real_data = tf.reshape(2*((tf.cast(all_real_data_int, tf.float32)/255.)-.5), [BATCH_SIZE, OUTPUT_DIM]) all_real_data_splits = lib.split(all_real_data, len(DEVICES)/2, axis=0) DEVICES_B = DEVICES[:len(DEVICES)/2] DEVICES_A = DEVICES[len(DEVICES)/2:] disc_costs = [] for i, device in enumerate(DEVICES_A): with tf.device(device): real_and_fake_data = lib.concat([all_real_data_splits[i]] + [fake_data_splits[i]] + [fake_data_splits[len(DEVICES_A)+i]], axis=0) disc_all = Discriminator(real_and_fake_data) disc_real = disc_all[:BATCH_SIZE/len(DEVICES_A)] disc_fake = disc_all[BATCH_SIZE/len(DEVICES_A):] disc_costs.append(tf.reduce_mean(disc_fake) - tf.reduce_mean(disc_real)) for i, device in enumerate(DEVICES_B): with tf.device(device): real_data = tf.identity(all_real_data_splits[i]) # transfer from gpu0 fake_data = lib.concat([fake_data_splits[i], fake_data_splits[len(DEVICES_A)+i]], axis=0) alpha = tf.random_uniform( shape=[BATCH_SIZE/len(DEVICES_A),1], minval=0., maxval=1. ) differences = fake_data - real_data
# outputs.append(ngram_output) # features.append(ngram_features) # if SETTINGS['rnn_discrim']: # outputs.append(output) # return tf.concat(0, outputs), language_model_output, features # we apply the sigmoid later _iteration = tf.placeholder(tf.int32, shape=None) real_inputs_discrete = tf.placeholder(tf.int32, shape=[BATCH_SIZE, SEQ_LEN]) real_inputs = tf.one_hot(real_inputs_discrete, len(charmap)) fake_inputs, _ = Generator(BATCH_SIZE) fake_inputs_discrete = tf.argmax(fake_inputs, fake_inputs.get_shape().ndims-1) disc_real, disc_real_lm, disc_real_features = Discriminator(real_inputs) disc_fake, disc_fake_lm, disc_fake_features = Discriminator(fake_inputs) # disc_out = Discriminator(tf.concat([real_inputs, fake_inputs], 0))[0] disc_out = Discriminator(lib.concat([real_inputs, fake_inputs], 0))[0] disc_real = disc_out[:BATCH_SIZE] disc_fake = disc_out[BATCH_SIZE:] # Gen objective: push D(fake) to one if SETTINGS['feature_matching']: gen_costs = [tf.reduce_mean((tf.reduce_mean(real_features, reduction_indices=[0]) - tf.reduce_mean(fake_features, reduction_indices=[0]))**2) for real_features, fake_features in zip(disc_real_features, disc_fake_features)] gen_cost = 0. for gc in gen_costs: gen_cost = gen_cost + gc elif SETTINGS['wgan']: gen_cost = -tf.reduce_mean(Discriminator(Generator(GEN_BS_MULTIPLE*BATCH_SIZE)[0])[0]) else: if SETTINGS['one_sided_label_smoothing']: raise Exception('check this implementation')
for device in DEVICES: with tf.device(device): fake_data_splits.append(Generator(BATCH_SIZE/len(DEVICES))) # fake_data = tf.concat(fake_data_splits, axis=0) # fake_data_splits = tf.split(fake_data, len(DEVICES)) all_real_data = tf.reshape(2*((tf.cast(all_real_data_conv, tf.float32)/255.)-.5), [BATCH_SIZE, OUTPUT_DIM]) all_real_data_splits = tf.split(all_real_data, len(DEVICES)/2) DEVICES_B = DEVICES[:len(DEVICES)/2] DEVICES_A = DEVICES[len(DEVICES)/2:] disc_costs = [] for i, device in enumerate(DEVICES_A): with tf.device(device): real_and_fake_data = lib.concat([all_real_data_splits[i]] + [fake_data_splits[i]] + [fake_data_splits[len(DEVICES_A)+i]], axis=0) disc_all = Discriminator(real_and_fake_data) disc_real = disc_all[:BATCH_SIZE/len(DEVICES_A)] disc_fake = disc_all[BATCH_SIZE/len(DEVICES_A):] disc_costs.append(tf.reduce_mean(disc_fake) - tf.reduce_mean(disc_real)) for i, device in enumerate(DEVICES_B): with tf.device(device): real_data = tf.identity(all_real_data_splits[i]) # transfer from gpu0 fake_data__ = lib.concat([fake_data_splits[i], fake_data_splits[len(DEVICES_A)+i]], axis=0) alpha = tf.random_uniform( shape=[BATCH_SIZE/len(DEVICES_A),1], minval=0., maxval=1. ) differences = fake_data__ - real_data
# features.append(ngram_features) # if SETTINGS['rnn_discrim']: # outputs.append(output) # return tf.concat(0, outputs), language_model_output, features # we apply the sigmoid later _iteration = tf.placeholder(tf.int32, shape=None) real_inputs_discrete = tf.placeholder(tf.int32, shape=[BATCH_SIZE, SEQ_LEN]) real_inputs = tf.one_hot(real_inputs_discrete, len(charmap)) fake_inputs, _ = Generator(BATCH_SIZE) fake_inputs_discrete = tf.argmax(fake_inputs, fake_inputs.get_shape().ndims - 1) disc_real, disc_real_lm, disc_real_features = Discriminator(real_inputs) disc_fake, disc_fake_lm, disc_fake_features = Discriminator(fake_inputs) # disc_out = Discriminator(tf.concat([real_inputs, fake_inputs], 0))[0] disc_out = Discriminator(lib.concat([real_inputs, fake_inputs], 0))[0] disc_real = disc_out[:BATCH_SIZE] disc_fake = disc_out[BATCH_SIZE:] # Gen objective: push D(fake) to one if SETTINGS['feature_matching']: gen_costs = [ tf.reduce_mean( (tf.reduce_mean(real_features, reduction_indices=[0]) - tf.reduce_mean(fake_features, reduction_indices=[0]))**2) for real_features, fake_features in zip(disc_real_features, disc_fake_features) ] gen_cost = 0. for gc in gen_costs: gen_cost = gen_cost + gc
all_real_data = tf.reshape(2*((tf.cast(all_real_data_int, tf.float32)/256.)-.5), [BATCH_SIZE, OUTPUT_DIM]) all_real_data += tf.random_uniform(shape=[BATCH_SIZE,OUTPUT_DIM],minval=0.,maxval=1./128) # dequantize all_real_data_splits = lib.split(all_real_data, len(DEVICES), axis=0) DEVICES_B = DEVICES[:len(DEVICES)/2] DEVICES_A = DEVICES[len(DEVICES)/2:] disc_costs = [] disc_acgan_costs = [] disc_acgan_accs = [] disc_acgan_fake_accs = [] for i, device in enumerate(DEVICES_A): with tf.device(device): real_and_fake_data = lib.concat([ all_real_data_splits[i], all_real_data_splits[len(DEVICES_A)+i], fake_data_splits[i], fake_data_splits[len(DEVICES_A)+i] ], axis=0) real_and_fake_labels = lib.concat([ labels_splits[i], labels_splits[len(DEVICES_A)+i], labels_splits[i], labels_splits[len(DEVICES_A)+i] ], axis=0) disc_all, disc_all_acgan = Discriminator(real_and_fake_data, real_and_fake_labels) disc_real = disc_all[:BATCH_SIZE/len(DEVICES_A)] disc_fake = disc_all[BATCH_SIZE/len(DEVICES_A):] disc_costs.append(tf.reduce_mean(disc_fake) - tf.reduce_mean(disc_real)) if ACGAN: disc_acgan_costs.append(tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits(logits=disc_all_acgan[:BATCH_SIZE/len(DEVICES_A)], labels=real_and_fake_labels[:BATCH_SIZE/len(DEVICES_A)])