Пример #1
0
    def make_sample_grid_and_save(self,
                                  ops,
                                  batch_size=10,
                                  random=12,
                                  interpolation=16,
                                  height=16,
                                  save_to_disk=True):
        # Gather images
        pool_size = random * height + 2 * height
        current_size = 0
        with tf.Graph().as_default():
            data_in = self.test_data.make_one_shot_iterator().get_next()
            with tf.Session() as sess_new:
                images = []
                labels = []
                while current_size < pool_size:
                    data_dict = sess_new.run(data_in)
                    images.append(data_dict["x"])
                    labels.append(data_dict["label"].argmax(1))
                    current_size += images[-1].shape[0]
                images = np.concatenate(images, axis=0)[:pool_size]
                labels = np.concatenate(labels, axis=0)[:pool_size]

        def batched_op(op, op_input, array):
            return np.concatenate(
                [
                    self.tf_sess.run(op, feed_dict={
                        op_input: array[x:x + batch_size]})
                    for x in range(0, array.shape[0], batch_size)
                ],
                axis=0)

        if random:
            random_x = images[:random * height]
            random_l = labels[:random * height]
            random_y = batched_op(ops.ae, ops.x, random_x)

            if ops.gap_weight is not None:
                encode_map = batched_op(ops.encode, ops.x, random_x)
                gap_weight = self.tf_sess.run(ops.gap_weight)  # 2x10

                encode_map, origin = utils.convert_to_heatmap(encode_map, gap_weight,
                                                              random_x.copy()[:,:,:, ::-1], random_l)

                fig_size = 32
                big_images = np.zeros(shape=[height * fig_size, random * fig_size, 3], dtype=np.uint8)
                big_encode = np.zeros(shape=[height * fig_size, random * fig_size, 3], dtype=np.uint8)
                big_encode_map = np.zeros(shape=[height * fig_size, random * 2 * fig_size, 3], dtype=np.uint8)
                for i in range(encode_map.shape[0]):
                    col = i % random
                    row = (i - col) // random
                    big_encode_map[row * fig_size: (row + 1) * fig_size,
                    col * fig_size * 2: (col + 1) * fig_size * 2] = \
                        np.concatenate((origin[i], encode_map[i]), axis=1)
                    big_images[row * fig_size: (row + 1) * fig_size,
                    col * fig_size: (col + 1) * fig_size] = origin[i]
                    big_encode[row * fig_size: (row + 1) * fig_size,
                    col * fig_size: (col + 1) * fig_size] = encode_map[i]


                encode_map = big_encode_map

                cv2.imwrite(os.path.join(self.image_dir, "heatmap%d.png" % (self.cur_nimg >> 10)),
                            encode_map)
                cv2.imwrite(os.path.join(self.image_dir, "heatmap_origin%d.png" % (self.cur_nimg >> 10)),
                            big_images)
                cv2.imwrite(os.path.join(self.image_dir, "heatmap_map%d.png" % (self.cur_nimg >> 10)),
                            big_encode)

            randoms = np.concatenate([random_x, random_y], axis=2)
            image_random = utils.images_to_grid(
                randoms.reshape((height, random) + randoms.shape[1:]))
        else:
            image_random = None

        # Interpolations
        interpolation_x = images[-2 * height:]
        latent_x = batched_op(ops.encode, ops.x, interpolation_x)
        latents = []
        for x in range(interpolation):
            latents.append((latent_x[:height] * (interpolation - x - 1) +
                            latent_x[height:] * x) / float(interpolation - 1))
        latents = np.concatenate(latents, axis=0)
        interpolation_y = batched_op(ops.decode, ops.h, latents)
        interpolation_y = interpolation_y.reshape(
            (interpolation, height) + interpolation_y.shape[1:])
        interpolation_y = interpolation_y.transpose(1, 0, 2, 3, 4)
        image_interpolation = utils.images_to_grid(interpolation_y)

        # Interpolations
        latents_slerp = []
        dots = np.sum(latent_x[:height] * latent_x[height:],
                      tuple(range(1, len(latent_x.shape))),
                      keepdims=True)
        norms = np.sum(latent_x * latent_x,
                       tuple(range(1, len(latent_x.shape))),
                       keepdims=True)
        cosine_dist = dots / np.sqrt(norms[:height] * norms[height:])
        omega = np.arccos(cosine_dist)
        for x in range(interpolation):
            t = x / float(interpolation - 1)
            latents_slerp.append(
                np.sin((1 - t) * omega) / np.sin(omega) * latent_x[:height] +
                np.sin(t * omega) / np.sin(omega) * latent_x[height:])
        latents_slerp = np.concatenate(latents_slerp, axis=0)
        interpolation_y_slerp = batched_op(ops.decode, ops.h, latents_slerp)
        interpolation_y_slerp = interpolation_y_slerp.reshape(
            (interpolation, height) + interpolation_y_slerp.shape[1:])
        interpolation_y_slerp = interpolation_y_slerp.transpose(1, 0, 2, 3, 4)
        image_interpolation_slerp = utils.images_to_grid(interpolation_y_slerp)

        ## generate synthetic images
        random_latents = np.random.standard_normal(latents.shape)

        samples_y = batched_op(ops.decode, ops.h, random_latents)
        samples_y = samples_y.reshape((interpolation, height) + samples_y.shape[1:])
        samples_y = samples_y.transpose(1, 0, 2, 3, 4)
        image_samples = utils.images_to_grid(samples_y)

        if random:
            image = np.concatenate(
                [image_random, image_interpolation, image_interpolation_slerp,
                 image_samples], axis=1)
        else:
            image = np.concatenate(
                [image_interpolation, image_interpolation_slerp,
                 image_samples], axis=1)
        if save_to_disk:
            utils.save_images(utils.to_png(image), self.image_dir,
                              self.cur_nimg)
        return (image_random, image_interpolation, image_interpolation_slerp,
                image_samples)
Пример #2
0
    def make_sample_grid_and_save(self,
                                  ops,
                                  batch_size=10,
                                  random=4,
                                  interpolation=16,
                                  height=16,
                                  save_to_disk=True):
        # Gather images
        pool_size = random * height + 2 * height
        current_size = 0
        with tf.Graph().as_default():
            data_in = self.test_data.make_one_shot_iterator().get_next()
            with tf.Session() as sess_new:
                images = []
                while current_size < pool_size:
                    images.append(sess_new.run(data_in)['x'])
                    current_size += images[-1].shape[0]
                images = np.concatenate(images, axis=0)[:pool_size]

        def batched_op(op, op_input, array):
            return np.concatenate([
                self.tf_sess.run(op,
                                 feed_dict={op_input: array[x:x + batch_size]})
                for x in range(0, array.shape[0], batch_size)
            ],
                                  axis=0)

        def batched_op_list(op, op_input, array):
            return np.concatenate([
                self.tf_sess.run(op,
                                 feed_dict={
                                     op_input[0]: array[0][x:x + batch_size],
                                     op_input[1]: array[1][x:x + batch_size],
                                     op_input[2]: array[2][x:x + batch_size]
                                 })
                for x in range(0, array[0].shape[0], batch_size)
            ],
                                  axis=0)

        # Random reconstructions
        if random:
            random_x = images[:random * height]
            random_y = batched_op(ops.ae, ops.x, random_x)
            randoms = np.concatenate([random_x, random_y], axis=2)
            image_random = utils.images_to_grid(
                randoms.reshape((height, random) + randoms.shape[1:]))
        else:
            image_random = None

        # Linear interpolations
        interpolation_x = images[-2 * height:]
        latent_x = batched_op(ops.encode, ops.x, interpolation_x)
        latents = []
        for x in range(interpolation):
            latents.append((latent_x[:height] *
                            (interpolation - x - 1) + latent_x[height:] * x) /
                           float(interpolation - 1))
        latents = np.concatenate(latents, axis=0)
        interpolation_y = batched_op(ops.decode, ops.h, latents)
        interpolation_y = interpolation_y.reshape((interpolation, height) +
                                                  interpolation_y.shape[1:])
        interpolation_y = interpolation_y.transpose(1, 0, 2, 3, 4)
        image_interpolation = utils.images_to_grid(interpolation_y)

        # Free interpolation
        interpolation_x = images[-2 * height:]
        latent_x = batched_op(ops.encode, ops.x, interpolation_x)
        latents = []
        for x in range(interpolation):
            alpha = x / float(interpolation - 1) * np.ones(
                shape=[height, 1, 1, 1], dtype=np.float32)
            interp_latent = batched_op_list(
                ops.my_encode_mix, [ops.h, ops.h_b, ops.alpha_h],
                [latent_x[height:], latent_x[:height], alpha])
            latents.append(interp_latent)
        latents = np.concatenate(latents, axis=0)
        interpolation_y = batched_op(ops.decode, ops.h, latents)
        interpolation_y = interpolation_y.reshape((interpolation, height) +
                                                  interpolation_y.shape[1:])
        interpolation_y = interpolation_y.transpose(1, 0, 2, 3, 4)
        oppreserve_interpolation2 = utils.images_to_grid(interpolation_y)

        # generate synthetic images
        random_latents = np.random.standard_normal(latents.shape)

        samples_y = batched_op(ops.decode, ops.h, random_latents)
        samples_y = samples_y.reshape((interpolation, height) +
                                      samples_y.shape[1:])
        samples_y = samples_y.transpose(1, 0, 2, 3, 4)
        image_samples = utils.images_to_grid(samples_y)

        if random:
            image = np.concatenate([
                image_random, image_interpolation, oppreserve_interpolation2,
                image_samples
            ],
                                   axis=1)
        else:
            image = np.concatenate([
                image_interpolation, oppreserve_interpolation2, image_samples
            ],
                                   axis=1)
        if save_to_disk:
            utils.save_images(utils.to_png(image), self.image_dir,
                              self.cur_nimg)

        return (image_random, image_interpolation, oppreserve_interpolation2,
                image_samples)
Пример #3
0
    def make_sample_grid_and_save(self,
                                  ops,
                                  batch_size=16,
                                  random=4,
                                  interpolation=16,
                                  height=16,
                                  save_to_disk=True):
        # Gather images
        pool_size = random * height + 2 * height
        current_size = 0
        with tf.Graph().as_default():
            data_in = self.test_data.make_one_shot_iterator().get_next()
            with tf.Session() as sess_new:
                images = []
                while current_size < pool_size:
                    images.append(sess_new.run(data_in)['x'])
                    current_size += images[-1].shape[0]
                images = np.concatenate(images, axis=0)[:pool_size]

        def batched_op(op, op_input, array):
            return np.concatenate(
                [
                    self.tf_sess.run(op, feed_dict={
                        op_input: array[x:x + batch_size]})
                    for x in range(0, array.shape[0], batch_size)
                ],
                axis=0)

        # Random reconstructions
        if random:
            random_x = images[:random * height]
            random_y = batched_op(ops.ae, ops.x, random_x)
            randoms = np.concatenate([random_x, random_y], axis=2)
            image_random = utils.images_to_grid(
                randoms.reshape((height, random) + randoms.shape[1:]))
        else:
            image_random = None

        # Interpolations
        interpolation_x = images[-2 * height:]
        latent_x = batched_op(ops.encode, ops.x, interpolation_x)
        latents = []
        for x in range(interpolation):
            latents.append((latent_x[:height] * (interpolation - x - 1) +
                            latent_x[height:] * x) / float(interpolation - 1))
        latents = np.concatenate(latents, axis=0)
        interpolation_y = batched_op(ops.decode, ops.h, latents)
        interpolation_y = interpolation_y.reshape(
            (interpolation, height) + interpolation_y.shape[1:])
        interpolation_y = interpolation_y.transpose(1, 0, 2, 3, 4)
        image_interpolation = utils.images_to_grid(interpolation_y)

        latents_slerp = []
        dots = np.sum(latent_x[:height] * latent_x[height:],
                      tuple(range(1, len(latent_x.shape))),
                      keepdims=True)
        norms = np.sum(latent_x * latent_x,
                       tuple(range(1, len(latent_x.shape))),
                       keepdims=True)
        cosine_dist = dots / np.sqrt(norms[:height] * norms[height:])
        omega = np.arccos(cosine_dist)
        for x in range(interpolation):
            t = x / float(interpolation - 1)
            latents_slerp.append(
                np.sin((1 - t) * omega) / np.sin(omega) * latent_x[:height] +
                np.sin(t * omega) / np.sin(omega) * latent_x[height:])
        latents_slerp = np.concatenate(latents_slerp, axis=0)
        interpolation_y_slerp = batched_op(ops.decode, ops.h, latents_slerp)
        interpolation_y_slerp = interpolation_y_slerp.reshape(
            (interpolation, height) + interpolation_y_slerp.shape[1:])
        interpolation_y_slerp = interpolation_y_slerp.transpose(1, 0, 2, 3, 4)
        image_interpolation_slerp = utils.images_to_grid(interpolation_y_slerp)

        random_latents = np.random.standard_normal(latents.shape)
        samples_y = batched_op(ops.decode, ops.h, random_latents)
        samples_y = samples_y.reshape(
            (interpolation, height) + samples_y.shape[1:])
        samples_y = samples_y.transpose(1, 0, 2, 3, 4)
        image_samples = utils.images_to_grid(samples_y)

        if random:
            image = np.concatenate(
                [image_random, image_interpolation, image_interpolation_slerp,
                 image_samples], axis=1)
        else:
            image = np.concatenate(
                [image_interpolation, image_interpolation_slerp,
                 image_samples], axis=1)
        if save_to_disk:
            utils.save_images(utils.to_png(image), self.image_dir,
                              self.cur_nimg)
        return (image_random, image_interpolation, image_interpolation_slerp,
                image_samples)
Пример #4
0
    def make_sample_grid_and_save(self,
                                  ops,
                                  batch_size=16,
                                  random=4,
                                  interpolation=16,
                                  height=16,
                                  save_to_disk=True):
        """

        :param ops: AEops class, including train_op
        :param batch_size:
        :param random: number of reconstructed images = random * height
        :param interpolation: number of interpolation, namely the row number of compositive image
        :param height: number of hight
        :param save_to_disk:
        :return: recon, inter, slerp, samples
        """
        # Gather images
        pool_size = random * height + 2 * height  # 96
        current_size = 0

        with tf.Graph().as_default():
            data_in = self.test_data.make_one_shot_iterator().get_next()
            with tf.Session() as sess_new:
                images = []
                while current_size < pool_size:
                    images.append(sess_new.run(data_in)['x'])
                    current_size += images[-1].shape[0]
                images = np.concatenate(images,
                                        axis=0)[:pool_size]  # [96, 32, 32, 1]

        def batched_op(op, op_input, array):
            return np.concatenate([
                self.tf_sess.run(op,
                                 feed_dict={op_input: array[x:x + batch_size]})
                for x in range(0, array.shape[0], batch_size)
            ],
                                  axis=0)

        # 1. Random reconstructions
        if random:  # not zero
            random_x = images[:random * height]  # [64, 32, 32, 1]
            random_y = batched_op(ops.ae, ops.x, random_x)
            randoms = np.concatenate(
                [random_x, random_y],
                axis=2)  # ae output: [64, 32, 32, 1] => [64, 32, 64, 1]
            image_random = utils.images_to_grid(  # [16, 4, 32, 64, 1] => [512, 256, 1]
                randoms.reshape((height, random) + randoms.shape[1:]))
        else:
            image_random = None

        # 2. Interpolations
        interpolation_x = images[-2 * height:]  # [32, 32, 32, 1]
        latent_x = batched_op(ops.encode, ops.x,
                              interpolation_x)  # [32, 4, 4, 16]
        latents = []
        for x in range(interpolation):
            latents.append((latent_x[:height] *
                            (interpolation - x - 1) + latent_x[height:] * x) /
                           float(interpolation - 1))
        latents = np.concatenate(latents, axis=0)  # [256, 4, 4, 16]
        interpolation_y = batched_op(ops.decode, ops.h,
                                     latents)  # [256, 32, 32, 1]
        interpolation_y = interpolation_y.reshape(  # [16, 16, 32, 32, 1]
            (interpolation, height) + interpolation_y.shape[1:])
        interpolation_y = interpolation_y.transpose(1, 0, 2, 3, 4)
        image_interpolation = utils.images_to_grid(
            interpolation_y)  # [512, 512, 1]

        # 3. Interpolation by slerp
        latents_slerp = []
        dots = np.sum(latent_x[:height] * latent_x[height:],
                      tuple(range(1, len(latent_x.shape))),
                      keepdims=True)  # [16, 1, 1, 1]
        norms = np.sum(latent_x * latent_x,
                       tuple(range(1, len(latent_x.shape))),
                       keepdims=True)  # [32, 1, 1, 1]
        cosine_dist = dots / np.sqrt(
            norms[:height] * norms[height:])  # [16, 1, 1, 1]
        omega = np.arccos(cosine_dist)
        for x in range(interpolation):
            t = x / float(interpolation - 1)
            latents_slerp.append(
                np.sin((1 - t) * omega) / np.sin(omega) * latent_x[:height] +
                np.sin(t * omega) / np.sin(omega) * latent_x[height:])
        latents_slerp = np.concatenate(
            latents_slerp, axis=0)  # 16 of[16, 4, 4, 16] => [256, 4, 4, 16]
        interpolation_y_slerp = batched_op(ops.decode, ops.h,
                                           latents_slerp)  # [256, 32, 32, 1]
        interpolation_y_slerp = interpolation_y_slerp.reshape(
            (interpolation, height) +
            interpolation_y_slerp.shape[1:])  # [16, 16, 32, 32, 1]
        interpolation_y_slerp = interpolation_y_slerp.transpose(1, 0, 2, 3, 4)
        image_interpolation_slerp = utils.images_to_grid(
            interpolation_y_slerp)  # [512, 512, 1]

        # 4. get decoder by random normal dist of hidden h
        random_latents = np.random.standard_normal(
            latents.shape)  # [256, 4, 4, 16]
        samples_y = batched_op(ops.decode, ops.h, random_latents)
        samples_y = samples_y.reshape((interpolation, height) +
                                      samples_y.shape[1:])
        samples_y = samples_y.transpose(1, 0, 2, 3, 4)
        image_samples = utils.images_to_grid(samples_y)  # [512, 512, 1]

        if random:  # [512, 256+512+512+512, 1]
            image = np.concatenate([
                image_random, image_interpolation, image_interpolation_slerp,
                image_samples
            ],
                                   axis=1)
        else:
            image = np.concatenate([
                image_interpolation, image_interpolation_slerp, image_samples
            ],
                                   axis=1)
        if save_to_disk:
            utils.save_images(utils.to_png(image), self.image_dir,
                              self.cur_nimg)

        return image_random, image_interpolation, image_interpolation_slerp, image_samples